Fix improper cache key on write for IPRO of resources with PageSpeed params

We need to be writing with the base name, with params stripped.
This was causing a test to "flake" under valgrind, since with it we got
to the test showing the bug slowly enough for the entry written out by
an earlier test under proper URL to expire.

README.md

@@ -37,21 +37,22 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/release-1.5.27.3-beta.zip
-   $ unzip release-1.5.27.3-beta.zip # or unzip release-1.5.27.3-beta
-   $ cd ngx_pagespeed-release-1.5.27.3-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.5.27.3.tar.gz
-   $ tar -xzvf 1.5.27.3.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.4-beta.zip
+   $ unzip v1.7.30.4-beta.zip # or unzip v1.7.30.4-beta
+   $ cd ngx_pagespeed-1.7.30.4-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
+   $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
 
    ```bash
+   $ cd ~
    $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.1.tar.gz
-   $ tar -xvzf nginx-1.4.1.tar.gz
-   $ cd nginx-1.4.1/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-release-1.5.27.3-beta
+   $ wget http://nginx.org/download/nginx-1.4.6.tar.gz
+   $ tar -xvzf nginx-1.4.6.tar.gz
+   $ cd nginx-1.4.6/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.4-beta
    $ make
    $ sudo make install
    ```
@@ -72,9 +73,7 @@ In your `nginx.conf`, add to the main or server block:
 
 ```nginx
 pagespeed on;
-
-# needs to exist and be writable by nginx
-pagespeed FileCachePath /var/ngx_pagespeed_cache;
+pagespeed FileCachePath /var/ngx_pagespeed_cache;  # Use tmpfs for best results.
 ```
 
 In every server block where pagespeed is enabled add:
@@ -86,7 +85,9 @@ location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""
 location ~ "^/ngx_pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
 location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
+location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
 location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
+location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```
 
 To confirm that the module is loaded, fetch a page and check that you see the
@@ -94,33 +95,14 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.5.27.3-...
+X-Page-Speed: 1.7.30.4-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as
 urls being replaced with new ones like `yellow.css.pagespeed.ce.lzJ8VcVi1l.css`.
 
-When reading the [mod_pagespeed
-documentation](https://developers.google.com/speed/docs/mod_pagespeed/using_mod),
-keep in mind that you need to make a small adjustment to configuration
-directives: replace **ModPagespeed** with **pagespeed**:
-
-    mod_pagespeed.conf:
-      ModPagespeedEnableFilters collapse_whitespace,add_instrumentation
-      ModPagespeedRunExperiment on
-      ModPagespeedExperimentSpec id=3;percent=50;default
-      ModPagespeedExperimentSpec id=4;percent=50
-
-    ngx_pagespeed.conf:
-      pagespeed EnableFilters collapse_whitespace,add_instrumentation;
-      pagespeed RunExperiment on;
-      pagespeed ExperimentSpec "id=3;percent=50;default";
-      pagespeed ExperimentSpec "id=4;percent=50";
-
-For more configuration details, see the [differences from mod_pagespeed
-configuration](https://github.com/pagespeed/ngx_pagespeed/wiki/Configuration-differences-from-mod_pagespeed)
-and <a href="https://github.com/pagespeed/ngx_pagespeed/wiki/Known-Issues">known
-issues</a> wiki pages.
+For complete documentation, see [Using
+PageSpeed](https://developers.google.com/speed/pagespeed/module/using).
 
 There are extensive system tests which cover most of ngx_pagespeed's
 functionality.  Consider [testing your
@@ -133,3 +115,20 @@ the progress of the project:
   list](https://groups.google.com/forum/#!forum/ngx-pagespeed-discuss)
 - [ngx-pagespeed-announce mailing
   list](https://groups.google.com/forum/#!forum/ngx-pagespeed-announce)
+
+Note: The
+[canonicalize_javascript_libraries](https://developers.google.com/speed/pagespeed/module/filter-canonicalize-js)
+depends on `pagespeed_libraries.conf` which is distributed in Apache's format.
+To convert it to the Nginx format, run:
+
+```bash
+$ scripts/pagespeed_libraries_generator.sh > ~/pagespeed_libraries.conf
+$ sudo mv ~/pagespeed_libraries.conf /etc/nginx/
+```
+
+And then include it in your Nginx configuration by reference:
+
+```nginx
+include pagespeed_libraries.conf;
+pagespeed EnableFilters canonicalize_javascript_libraries;
+```

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.5.27.3.tar.gz"
-    echo "     $ tar -xzvf 1.5.27.3.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz"
+    echo "     $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -83,6 +83,32 @@ else
   buildtype=Release
 fi
 
+# The compiler needs to know that __sync_add_and_fetch_4 is ok,
+# and this requires an instruction that didn't exist on i586 or i386.
+if [ "$uname_arch" = "i686" ]; then
+  FLAG_MARCH='-march=i686'
+fi
+
+CFLAGS="$CFLAGS $FLAG_MARCH"
+
+case "$NGX_GCC_VER" in
+  4.8*)
+    # On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs.  This breaks
+    # on VerifySizesAreEqual in bit_cast in chromium/src/base/basictypes.h which
+    # has a typedef that is intentionally unused.
+    CFLAGS="$CFLAGS -Wno-unused-local-typedefs"
+
+    # On GCC 4.8 and above, we get the following compiler warning:
+    # chromium/src/base/memory/scoped_ptr.h:133:7: warning: declaration of ‘class scoped_ptr<C>’ [enabled by default]
+    # Based on discussion at http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54055,
+    # this is invalid code, but hasn't been fixed yet in chromium.
+    # Unfortunately, there also does not appear to be a flag for just disabling
+    # that warning, so we add Wno-error to override nginx's default -Werror
+    # option.
+    CFLAGS="$CFLAGS -Wno-error"
+  ;;
+esac
+
 pagespeed_include="\
   $mod_pagespeed_dir \
   $mod_pagespeed_dir/third_party/chromium/src \
@@ -91,6 +117,7 @@ pagespeed_include="\
   $mod_pagespeed_dir/third_party/protobuf/src \
   $mod_pagespeed_dir/third_party/re2/src \
   $mod_pagespeed_dir/out/$buildtype/obj/gen \
+  $mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
   $mod_pagespeed_dir/third_party/apr/src/include \
   $mod_pagespeed_dir/third_party/aprutil/src/include \
   $mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
@@ -99,17 +126,10 @@ ngx_feature_path="$pagespeed_include"
 
 if $build_from_source ; then
   psol_library_binaries="\
-    $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a \
-    $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/serf/libserf.a \
-    $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/aprutil/libaprutil.a \
-    $mod_pagespeed_dir/out/$buildtype/obj.target/third_party/apr/libapr.a"
+    $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
 else
   psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
-  psol_library_binaries="\
-    $psol_library_dir/pagespeed_automatic.a \
-    $psol_library_dir/libserf.a \
-    $psol_library_dir/libaprutil.a \
-    $psol_library_dir/libapr.a"
+  psol_library_binaries="$psol_library_dir/pagespeed_automatic.a"
 fi
 
 pagespeed_libs="-lstdc++ $psol_library_binaries -lrt -pthread -lm"
@@ -139,34 +159,30 @@ if [ $ngx_found = yes ]; then
   ps_src="$ngx_addon_dir/src"
   ngx_addon_name=ngx_pagespeed
   NGX_ADDON_DEPS="$NGX_ADDON_DEPS \
-    $ps_src/ngx_pagespeed.h \
-    $ps_src/ngx_fetch.h \
-    $ps_src/ngx_url_async_fetcher.h \
+    $ps_src/log_message_handler.h \
     $ps_src/ngx_base_fetch.h \
-    $ps_src/ngx_server_context.h \
-    $ps_src/ngx_rewrite_options.h \
-    $ps_src/ngx_rewrite_driver_factory.h \
-    $ps_src/ngx_thread_system.h \
+    $ps_src/ngx_caching_headers.h \
+    $ps_src/ngx_fetch.h \
+    $ps_src/ngx_list_iterator.h \
     $ps_src/ngx_message_handler.h \
-    $ps_src/pthread_shared_mem.h \
-    $ps_src/ngx_request_context.h \
-    $ps_src/log_message_handler.h"
+    $ps_src/ngx_pagespeed.h \
+    $ps_src/ngx_rewrite_driver_factory.h \
+    $ps_src/ngx_rewrite_options.h \
+    $ps_src/ngx_server_context.h \
+    $ps_src/ngx_url_async_fetcher.h"
   NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
+    $ps_src/log_message_handler.cc \
+    $ps_src/ngx_base_fetch.cc \
+    $ps_src/ngx_caching_headers.cc \
+    $ps_src/ngx_fetch.cc \
+    $ps_src/ngx_list_iterator.cc \
+    $ps_src/ngx_message_handler.cc \
     $ps_src/ngx_pagespeed.cc \
     $ps_src/ngx_rewrite_driver_factory.cc \
     $ps_src/ngx_rewrite_options.cc \
     $ps_src/ngx_server_context.cc \
-    $ps_src/ngx_fetch.cc \
-    $ps_src/ngx_url_async_fetcher.cc \
-    $ps_src/ngx_base_fetch.cc \
-    $ps_src/ngx_thread_system.cc \
-    $ps_src/ngx_message_handler.cc \
-    $ps_src/pthread_shared_mem.cc \
-    $ps_src/ngx_request_context.cc \
-    $ps_src/log_message_handler.cc \
-    $mod_pagespeed_dir/net/instaweb/apache/add_headers_fetcher.cc \
-    $mod_pagespeed_dir/net/instaweb/apache/loopback_route_fetcher.cc \
-    $mod_pagespeed_dir/net/instaweb/apache/serf_url_async_fetcher.cc"
+    $ps_src/ngx_url_async_fetcher.cc"
+
   # Make pagespeed run immediately before gzip.
   HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
     sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
@@ -175,9 +191,11 @@ if [ $ngx_found = yes ]; then
     sed "s/$HTTP_GZIP_FILTER_MODULE/ngx_pagespeed_etag_filter $HTTP_GZIP_FILTER_MODULE/")
   CORE_LIBS="$CORE_LIBS $pagespeed_libs"
   CORE_INCS="$CORE_INCS $pagespeed_include"
+  echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
 else
   cat << END
-$0: error: module ngx_pagespeed requires the pagespeed optimization library
+$0: error: module ngx_pagespeed requires the pagespeed optimization library.
+Look in obj/autoconf.err for more details.
 END
   exit 1
 fi

scripts/pagespeed_libraries_generator.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Converts pagespeed_libraries.conf from Apache-format to Nginx-format,
+# supporting the canonicalize_javascript_libraries filter.
+# Inspired by https://github.com/pagespeed/ngx_pagespeed/issues/532
+#
+# Usage:
+#   scripts/pagespeed_libraries_generator.sh > pagespeed_libraries.conf
+#
+# Then have nginx include that configuration file and enable the filter
+# canonicalize_javascript_libraries.
+#
+# Author: vid@zippykid.com (Vid Luther)
+#         jefftk@google.com (Jeff Kaufman)
+
+URL="https://modpagespeed.googlecode.com/svn/trunk/src/"
+URL+="net/instaweb/genfiles/conf/pagespeed_libraries.conf"
+curl -s "$URL" \
+    | grep ModPagespeedLibrary \
+    | while read library size hash url ; do
+  echo "  pagespeed Library $size $hash $url;"
+done

scripts/prepare_psol.sh

@@ -63,11 +63,16 @@ rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \
   --include="apr_memcache2.c" \
   --include="loopback_route_fetcher.cc" \
   --include="add_headers_fetcher.cc" \
+  --include="console_css_out.cc" \
+  --include="console_out.cc" \
   --include="dense_hash_map" \
   --include="dense_hash_set" \
   --include="sparse_hash_map" \
   --include="sparse_hash_set" \
   --include="sparsetable" \
+  --include="mod_pagespeed_console_out.cc" \
+  --include="mod_pagespeed_console_css_out.cc" \
+  --include="mod_pagespeed_console_html_out.cc" \
   --exclude='*'
 mkdir -p psol/lib/Debug/linux/ia32
 mkdir -p psol/lib/Debug/linux/x64

src/log_message_handler.cc

@@ -98,8 +98,6 @@ namespace net_instaweb {
 namespace log_message_handler {
 
 
-const int kDebugLogLevel = -2;
-
 void Install(ngx_log_t* log_in) {
   log = log_in;
   logging::SetLogMessageHandler(&LogMessageHandler);

src/ngx_base_fetch.cc

@@ -17,6 +17,7 @@
 // Author: jefftk@google.com (Jeff Kaufman)
 
 #include "ngx_base_fetch.h"
+#include "ngx_list_iterator.h"
 
 #include "ngx_pagespeed.h"
 
@@ -29,16 +30,18 @@ namespace net_instaweb {
 
 NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
                            NgxServerContext* server_context,
-                           const RequestContextPtr& request_ctx)
+                           const RequestContextPtr& request_ctx,
+                           PreserveCachingHeaders preserve_caching_headers)
     : AsyncFetch(request_ctx),
       request_(r),
       server_context_(server_context),
       done_called_(false),
       last_buf_sent_(false),
       pipe_fd_(pipe_fd),
-      references_(2) {
+      references_(2),
+      handle_error_(true),
+      preserve_caching_headers_(preserve_caching_headers) {
   if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
-  PopulateRequestHeaders();
 }
 
 NgxBaseFetch::~NgxBaseFetch() {
@@ -53,58 +56,6 @@ void NgxBaseFetch::Unlock() {
   pthread_mutex_unlock(&mutex_);
 }
 
-void NgxBaseFetch::PopulateRequestHeaders() {
-  CopyHeadersFromTable<RequestHeaders>(&request_->headers_in.headers,
-                                       request_headers());
-}
-
-void NgxBaseFetch::PopulateResponseHeaders() {
-  CopyHeadersFromTable<ResponseHeaders>(&request_->headers_out.headers,
-                                        response_headers());
-
-  response_headers()->set_status_code(request_->headers_out.status);
-
-  // Manually copy over the content type because it's not included in
-  // request_->headers_out.headers.
-  response_headers()->Add(
-      HttpAttributes::kContentType,
-      ngx_psol::str_to_string_piece(request_->headers_out.content_type));
-
-  // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
-  response_headers()->ComputeCaching();
-}
-
-template<class HeadersT>
-void NgxBaseFetch::CopyHeadersFromTable(ngx_list_t* headers_from,
-                                        HeadersT* headers_to) {
-  // http_version is the version number of protocol; 1.1 = 1001. See
-  // NGX_HTTP_VERSION_* in ngx_http_request.h
-  headers_to->set_major_version(request_->http_version / 1000);
-  headers_to->set_minor_version(request_->http_version % 1000);
-
-  // Standard nginx idiom for iterating over a list.  See ngx_list.h
-  ngx_uint_t i;
-  ngx_list_part_t* part = &headers_from->part;
-  ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(part->elts);
-
-  for (i = 0 ; /* void */; i++) {
-    if (i >= part->nelts) {
-      if (part->next == NULL) {
-        break;
-      }
-
-      part = part->next;
-      header = static_cast<ngx_table_elt_t*>(part->elts);
-      i = 0;
-    }
-
-    StringPiece key = ngx_psol::str_to_string_piece(header[i].key);
-    StringPiece value = ngx_psol::str_to_string_piece(header[i].value);
-
-    headers_to->Add(key, value);
-  }
-}
-
 bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
                                MessageHandler* handler) {
   Lock();
@@ -113,12 +64,18 @@ bool NgxBaseFetch::HandleWrite(const StringPiece& sp,
   return true;
 }
 
+// should only be called in nginx thread
 ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
-  if (done_called_ && last_buf_sent_) {
-    return NGX_DECLINED;
+  CHECK(!(done_called_ && last_buf_sent_))
+        << "CopyBufferToNginx() was called after the last buffer was sent";
+
+  // there is no buffer to send
+  if (!done_called_ && buffer_.empty()) {
+    *link_ptr = NULL;
+    return NGX_AGAIN;
   }
 
-  int rc = ngx_psol::string_piece_to_buffer_chain(
+  int rc = string_piece_to_buffer_chain(
       request_->pool, buffer_, link_ptr, done_called_ /* send_last_buf */);
   if (rc != NGX_OK) {
     return rc;
@@ -129,31 +86,33 @@ ngx_int_t NgxBaseFetch::CopyBufferToNginx(ngx_chain_t** link_ptr) {
 
   if (done_called_) {
     last_buf_sent_ = true;
+    return NGX_OK;
   }
 
-  return NGX_OK;
+  return NGX_AGAIN;
 }
 
 // There may also be a race condition if this is called between the last Write()
 // and Done() such that we're sending an empty buffer with last_buf set, which I
 // think nginx will reject.
 ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
+  ngx_int_t rc;
   Lock();
-  ngx_int_t rc = CopyBufferToNginx(link_ptr);
+  rc = CopyBufferToNginx(link_ptr);
   Unlock();
-
-  if (rc == NGX_DECLINED) {
-    *link_ptr = NULL;
-    return NGX_OK;
-  }
   return rc;
 }
 
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
-  Lock();
   const ResponseHeaders* pagespeed_headers = response_headers();
-  Unlock();
-  return ngx_psol::copy_response_headers_to_ngx(request_, *pagespeed_headers);
+
+  if (content_length_known()) {
+     headers_out->content_length = NULL;
+     headers_out->content_length_n = content_length();
+  }
+
+  return copy_response_headers_to_ngx(request_, *pagespeed_headers,
+                                      preserve_caching_headers_);
 }
 
 void NgxBaseFetch::RequestCollection() {
@@ -175,10 +134,15 @@ void NgxBaseFetch::RequestCollection() {
 }
 
 void NgxBaseFetch::HandleHeadersComplete() {
+  int status_code = response_headers()->status_code();
+  bool status_ok = (status_code != 0) && (status_code < 400);
+
+  if (status_ok || handle_error_) {
     // If this is a 404 response we need to count it in the stats.
     if (response_headers()->status_code() == HttpStatus::kNotFound) {
       server_context_->rewrite_stats()->resource_404_count()->Add(1);
     }
+  }
 
   RequestCollection();  // Headers available.
 }

src/ngx_base_fetch.h

@@ -57,15 +57,10 @@ class NgxBaseFetch : public AsyncFetch {
  public:
   NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
                NgxServerContext* server_context,
-               const RequestContextPtr& request_ctx);
+               const RequestContextPtr& request_ctx,
+               PreserveCachingHeaders preserve_caching_headers);
   virtual ~NgxBaseFetch();
 
-  // Copies the request headers out of request_->headers_in->headers.
-  void PopulateRequestHeaders();
-
-  // Copies the response headers out of request_->headers_out->headers.
-  void PopulateResponseHeaders();
-
   // Puts a chain in link_ptr if we have any output data buffered.  Returns
   // NGX_OK on success, NGX_ERROR on errors.  If there's no data to send, sends
   // data only if Done() has been called.  Indicates the end of output by
@@ -84,6 +79,7 @@ class NgxBaseFetch : public AsyncFetch {
 
   // Called by nginx when it's done with us.
   void Release();
+  void set_handle_error(bool x) { handle_error_ = x; }
 
  private:
   virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
@@ -91,18 +87,15 @@ class NgxBaseFetch : public AsyncFetch {
   virtual void HandleHeadersComplete();
   virtual void HandleDone(bool success);
 
-  // Helper method for PopulateRequestHeaders and PopulateResponseHeaders.
-  template<class HeadersT>
-  void CopyHeadersFromTable(ngx_list_t* headers_from, HeadersT* headers_to);
-
   // Indicate to nginx that we would like it to call
   // CollectAccumulatedWrites().
   void RequestCollection();
 
   // Lock must be acquired first.
   // Returns:
-  //   NGX_DECLINED: nothing to send, short circuit.  Buffer not allocated.
-  //   NGX_OK, NGX_ERROR: success, failure
+  //   NGX_ERROR: failure
+  //   NGX_AGAIN: still has buffer to send, need to checkout link_ptr
+  //   NGX_OK: done, HandleDone has been called
   // Allocates an nginx buffer, copies our buffer_ contents into it, clears
   // buffer_.
   ngx_int_t CopyBufferToNginx(ngx_chain_t** link_ptr);
@@ -124,6 +117,8 @@ class NgxBaseFetch : public AsyncFetch {
   // decremented once when Done() is called and once when Release() is called.
   int references_;
   pthread_mutex_t mutex_;
+  bool handle_error_;
+  PreserveCachingHeaders preserve_caching_headers_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
 };

src/ngx_caching_headers.cc

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_caching_headers.h"
+#include "ngx_list_iterator.h"
+
+#include "ngx_pagespeed.h"
+
+namespace net_instaweb {
+
+bool NgxCachingHeaders::Lookup(const StringPiece& key,
+                               StringPieceVector* values) {
+  ngx_table_elt_t* header;
+  NgxListIterator it(&(request_->headers_out.headers.part));
+  while ((header = it.Next()) != NULL) {
+    if (header->hash != 0 && key == str_to_string_piece(header->key)) {
+      // This will be called multiple times if there are multiple headers with
+      // this name.  Each time it will append to values.
+      SplitStringPieceToVector(str_to_string_piece(header->value), ",", values,
+                               true /* omit empty strings */);
+    }
+  }
+
+  if (values->size() == 0) {
+    // No header found with this name.
+    return false;
+  }
+
+  for (int i = 0, n = values->size(); i < n; ++i) {
+    TrimWhitespace(&((*values)[i]));
+  }
+
+  return true;
+}
+
+}  // namespace net_instaweb

src/ngx_caching_headers.h

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#ifndef NGX_CACHING_HEADERS_H_
+#define NGX_CACHING_HEADERS_H_
+
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "pagespeed/kernel/base/basictypes.h"
+#include "pagespeed/kernel/http/caching_headers.h"
+
+namespace net_instaweb {
+
+// Based off of ApacheCachingHeaders in net/instaweb/apache/header_util.cc
+// Needed so ps_header_filter can call GenerateDisabledCacheControl().
+class NgxCachingHeaders : public CachingHeaders {
+ public:
+  explicit NgxCachingHeaders(ngx_http_request_t* request)
+      : CachingHeaders(request->headers_out.status),
+        request_(request) {
+  }
+
+  virtual bool Lookup(const StringPiece& key, StringPieceVector* values);
+
+  virtual bool IsLikelyStaticResourceType() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+  virtual bool IsCacheableResourceStatusCode() const {
+    DCHECK(false);  // not called in our use-case.
+    return false;
+  }
+
+ private:
+  ngx_http_request_t* request_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxCachingHeaders);
+};
+
+}  // namespace net_instaweb
+
+#endif  // NGX_CACHING_HEADERS_H_

src/ngx_fetch.cc

@@ -24,6 +24,10 @@
 //  - The read handler parses the response. Add the response to the buffer at
 //    last.
 
+extern "C" {
+#include <nginx.h>
+}
+
 #include "ngx_fetch.h"
 #include "net/instaweb/util/public/basictypes.h"
 #include "base/logging.h"
@@ -66,7 +70,8 @@ namespace net_instaweb {
         fetch_start_ms_(0),
         fetch_end_ms_(0),
         done_(false),
-        content_length_(0) {
+        content_length_(-1),
+        content_length_known_(false) {
             ngx_memzero(&url_, sizeof(url_));
             log_ = log;
             pool_ = NULL;
@@ -138,19 +143,26 @@ namespace net_instaweb {
     // The host is either a domain name or an IP address.  First check
     // if it's a valid IP address and only if that fails fall back to
     // using the DNS resolver.
-    GoogleString s_ipaddress(reinterpret_cast<char*>(url_.host.data),
-                             url_.host.len);
+
+    // Maybe we have a Proxy.
+    ngx_url_t* tmp_url = &url_;
+    if (0 != fetcher_->proxy_.url.len) {
+      tmp_url = &fetcher_->proxy_;
+    }
+
+    GoogleString s_ipaddress(reinterpret_cast<char*>(tmp_url->host.data),
+                             tmp_url->host.len);
     ngx_memzero(&sin_, sizeof(sin_));
     sin_.sin_family = AF_INET;
-    sin_.sin_port = htons(url_.port);
+    sin_.sin_port = htons(tmp_url->port);
     sin_.sin_addr.s_addr = inet_addr(s_ipaddress.c_str());
 
     if (sin_.sin_addr.s_addr == INADDR_NONE) {
       // inet_addr returned INADDR_NONE, which means the hostname
       // isn't a valid IP address.  Check DNS.
       ngx_resolver_ctx_t temp;
-      temp.name.data = url_.host.data;
-      temp.name.len = url_.host.len;
+      temp.name.data = tmp_url->host.data;
+      temp.name.len = tmp_url->host.len;
       resolver_ctx_ = ngx_resolve_start(fetcher_->resolver_, &temp);
       if (resolver_ctx_ == NULL || resolver_ctx_ == NGX_NO_RESOLVER) {
         // TODO(oschaaf): this spams the log, but is useful in the fetcher's
@@ -162,9 +174,13 @@ namespace net_instaweb {
       }
 
       resolver_ctx_->data = this;
-      resolver_ctx_->name.data = url_.host.data;
-      resolver_ctx_->name.len = url_.host.len;
+      resolver_ctx_->name.data = tmp_url->host.data;
+      resolver_ctx_->name.len = tmp_url->host.len;
+
+#if (nginx_version < 1005008)
       resolver_ctx_->type = NGX_RESOLVE_A;
+#endif
+
       resolver_ctx_->handler = NgxFetchResolveDone;
       resolver_ctx_->timeout = fetcher_->resolver_timeout_;
 
@@ -255,39 +271,14 @@ namespace net_instaweb {
       return false;
     }
     str_url_.copy(reinterpret_cast<char*>(url_.url.data), str_url_.length(), 0);
-    size_t scheme_offset;
-    u_short port;
-    if (ngx_strncasecmp(url_.url.data, reinterpret_cast<u_char*>(
-                                     const_cast<char*>("http://")), 7) == 0) {
-      scheme_offset = 7;
-      port = 80;
-    } else if (ngx_strncasecmp(url_.url.data, reinterpret_cast<u_char*>(
-                                    const_cast<char*>("https://")), 8) == 0) {
-      scheme_offset = 8;
-      port = 443;
-    } else {
-      scheme_offset = 0;
-      port = 80;
-    }
 
-    url_.url.data += scheme_offset;
-    url_.url.len -= scheme_offset;
-    url_.default_port = port;
-    // TODO(oschaaf): no_resolve was set to 0, which is why url_.port
-    // would always be '0' after parsing it. See:
-    // http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
-    url_.no_resolve = 0;
-    url_.uri_part = 1;
-
-    if (ngx_parse_url(pool_, &url_) == NGX_OK) {
-      return true;
-    }
-    return false;
+    return NgxUrlAsyncFetcher::ParseUrl(&url_, pool_);
   }
 
   // Issue a request after the resolver is done
   void NgxFetch::NgxFetchResolveDone(ngx_resolver_ctx_t* resolver_ctx) {
     NgxFetch* fetch = static_cast<NgxFetch*>(resolver_ctx->data);
+    NgxUrlAsyncFetcher* fetcher = fetch->fetcher_;
     if (resolver_ctx->state != NGX_OK) {
       if (fetch->timeout_event() != NULL && fetch->timeout_event()->timer_set) {
         ngx_del_timer(fetch->timeout_event());
@@ -301,9 +292,25 @@ namespace net_instaweb {
       return;
     }
     ngx_memzero(&fetch->sin_, sizeof(fetch->sin_));
+
+#if (nginx_version < 1005008)
+    fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[0];
+#else
+
+    struct sockaddr_in *sin;
+    sin = reinterpret_cast<struct sockaddr_in *>(
+        resolver_ctx->addrs[0].sockaddr);
+    fetch->sin_.sin_family = sin->sin_family;
+    fetch->sin_.sin_addr.s_addr = sin->sin_addr.s_addr;
+#endif
+
     fetch->sin_.sin_family = AF_INET;
     fetch->sin_.sin_port = htons(fetch->url_.port);
-    fetch->sin_.sin_addr.s_addr = resolver_ctx->addrs[0];
+
+    // Maybe we have Proxy
+    if (0 != fetcher->proxy_.url.len) {
+      fetch->sin_.sin_port = htons(fetcher->proxy_.port);
+    }
 
     char* ip_address = inet_ntoa(fetch->sin_.sin_addr);
 
@@ -335,7 +342,13 @@ namespace net_instaweb {
     bool have_host = false;
     GoogleString port;
 
-    size = sizeof("GET ") - 1 + url_.uri.len + sizeof(" HTTP/1.0\r\n") - 1;
+    const char* method = request_headers->method_string();
+    size_t method_len = strlen(method);
+
+    size = (method_len +
+            1 /* for the space */ +
+            url_.uri.len +
+            sizeof(" HTTP/1.0\r\n") - 1);
 
     for (int i = 0; i < request_headers->NumAttributes(); i++) {
       // if no explicit host header is given in the request headers,
@@ -363,7 +376,8 @@ namespace net_instaweb {
       return NGX_ERROR;
     }
 
-    out_->last = ngx_cpymem(out_->last, "GET ", 4);
+    out_->last = ngx_cpymem(out_->last, method, method_len);
+    out_->last = ngx_cpymem(out_->last, " ", 1);
     out_->last = ngx_cpymem(out_->last, url_.uri.data, url_.uri.len);
     out_->last = ngx_cpymem(out_->last, " HTTP/1.0\r\n", 11);
 
@@ -413,12 +427,16 @@ namespace net_instaweb {
     pc.rcvbuf = -1;
 
     int rc = ngx_event_connect_peer(&pc);
+    if (rc == NGX_ERROR || rc == NGX_DECLINED || rc == NGX_BUSY) {
+      return rc;
+    }
+
     connection_ = pc.connection;
     connection_->write->handler = NgxFetchWrite;
     connection_->read->handler = NgxFetchRead;
     connection_->data = this;
 
-    // TODO(junmin): set connect timeout when rc == NGX_AGAIN
+    // Timer set in Init() is still in effect.
     return rc;
   }
 
@@ -434,10 +452,10 @@ namespace net_instaweb {
       if (n >= 0) {
         out->pos += n;
       } else if (n == NGX_AGAIN) {
-        // TODO(junmin): set write event timeout
         if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
           fetch->CallbackDone(false);
         }
+        // Timer set in Init() is still in effect.
         return;
       } else {
         c->error = 1;
@@ -467,9 +485,16 @@ namespace net_instaweb {
       }
 
       if (n == 0) {
-        // connection is closed prematurely by remote server,
-        // or the content-length was 0
-        fetch->CallbackDone(fetch->content_length_ == 0);
+        // If the content length was not known, we assume that we have read
+        // all if we at least parsed the headers.
+        // If we do know the content length, having a mismatch on the bytes read
+        // will be interpreted as an error.
+        if (fetch->content_length_known_) {
+          fetch->CallbackDone(fetch->content_length_ == fetch->bytes_received_);
+        } else {
+          fetch->CallbackDone(fetch->parser_.headers_complete());
+        }
+
         return;
       } else if (n > 0) {
         fetch->in_->pos = fetch->in_->start;
@@ -499,7 +524,7 @@ namespace net_instaweb {
       fetch->CallbackDone(false);
     }
 
-    // TODO(junmin): set read event timeout
+    // Timer set in Init() is still in effect.
   }
 
   // Parse the status line: "HTTP/1.1 200 OK\r\n"
@@ -536,13 +561,21 @@ namespace net_instaweb {
     if (n > size) {
       return false;
     } else if (fetch->parser_.headers_complete()) {
-      int64 content_length = -1;
-      fetch->async_fetch_->response_headers()->FindContentLength(
-          &content_length);
-      fetch->content_length_ = content_length;
-      if (fetch->fetcher_->track_original_content_length()) {
+      if (fetch->async_fetch_->response_headers()->FindContentLength(
+              &fetch->content_length_)) {
+        if (fetch->content_length_ < 0) {
+          fetch->message_handler_->Message(
+              kError, "Negative content-length in response header");
+          return false;
+        } else {
+          fetch->content_length_known_ = true;
+        }
+      }
+
+      if (fetch->fetcher_->track_original_content_length()
+         && fetch->content_length_known_) {
         fetch->async_fetch_->response_headers()->SetOriginalContentLength(
-            content_length);
+            fetch->content_length_);
       }
 
       fetch->in_->pos += n;
@@ -561,11 +594,12 @@ namespace net_instaweb {
       return true;
     }
 
-    fetch->bytes_received_add(static_cast<int64>(size));
+    fetch->bytes_received_add(size);
+
     if (fetch->async_fetch_->Write(StringPiece(data, size),
         fetch->message_handler())) {
-      fetch->content_length_ -= size;
-      if (fetch->content_length_ <= 0) {
+      if (fetch->content_length_known_ &&
+          fetch->bytes_received_ == fetch->content_length_) {
         fetch->done_ = true;
       }
       return true;

src/ngx_fetch.h

@@ -19,15 +19,13 @@
 // The fetch is started by the main thread. It will fetch the remote resource
 // from the specific url asynchronously.
 
-#ifndef NET_INSTAWEB_NGX_FETCHER_H_
-#define NET_INSTAWEB_NGX_FETCHER_H_
+#ifndef NET_INSTAWEB_NGX_FETCH_H_
+#define NET_INSTAWEB_NGX_FETCH_H_
 
 extern "C" {
 #include <ngx_config.h>
 #include <ngx_core.h>
 #include <ngx_http.h>
-
-  typedef bool (*response_handler_pt)(ngx_connection_t* c);
 }
 
 #include "ngx_url_async_fetcher.h"
@@ -35,11 +33,14 @@ extern "C" {
 #include "net/instaweb/util/public/basictypes.h"
 #include "net/instaweb/util/public/pool.h"
 #include "net/instaweb/util/public/string.h"
-#include "net/instaweb/http/public/url_pollable_async_fetcher.h"
+#include "net/instaweb/http/public/url_async_fetcher.h"
 #include "net/instaweb/http/public/response_headers.h"
 #include "net/instaweb/http/public/response_headers_parser.h"
 
 namespace net_instaweb {
+
+typedef bool (*response_handler_pt)(ngx_connection_t* c);
+
 class NgxUrlAsyncFetcher;
 class NgxFetch : public PoolElement<NgxFetch> {
  public:
@@ -50,15 +51,15 @@ namespace net_instaweb {
            ngx_log_t* log);
   ~NgxFetch();
 
-      // Start the fetch
+  // Start the fetch.
   bool Start(NgxUrlAsyncFetcher* fetcher);
-      // Show the completed url, for logging purpose
+  // Show the completed url, for logging purposes.
   const char* str_url();
-      // This fetch task is done. Call the done of async_fetch.
-      // It will copy the buffer to cache.
+  // This fetch task is done. Call Done() on the async_fetch. It will copy the
+  // buffer to cache.
   void CallbackDone(bool success);
 
-      // Show the bytes received
+  // Show the bytes received.
   size_t bytes_received();
   void bytes_received_add(int64 x);
   int64 fetch_start_ms();
@@ -66,24 +67,22 @@ namespace net_instaweb {
   int64 fetch_end_ms();
   void set_fetch_end_ms(int64 end_ms);
   MessageHandler* message_handler();
+
   int get_major_version() {
     return static_cast<int>(status_->http_version / 1000);
   }
-
   int get_minor_version() {
     return static_cast<int>(status_->http_version % 1000);
   }
-
   int get_status_code() {
     return static_cast<int>(status_->code);
   }
-
   ngx_event_t* timeout_event() {
     return timeout_event_;
-      };
+  }
   void set_timeout_event(ngx_event_t* x) {
     timeout_event_ = x;
-      };
+  }
 
  private:
   response_handler_pt response_handler;
@@ -99,23 +98,20 @@ namespace net_instaweb {
   }
   // Only the Static functions could be used in callbacks.
   static void NgxFetchResolveDone(ngx_resolver_ctx_t* ctx);
-
-      // Write the request
+  // Write the request.
   static void NgxFetchWrite(ngx_event_t* wev);
-
-      // Wait for the response
+  // Wait for the response.
   static void NgxFetchRead(ngx_event_t* rev);
-      // Read and parse the first status line
+  // Read and parse the first status line.
   static bool NgxFetchHandleStatusLine(ngx_connection_t* c);
-      // Read and parse the HTTP headers
+  // Read and parse the HTTP headers.
   static bool NgxFetchHandleHeader(ngx_connection_t* c);
-      // Read the response body
+  // Read the response body.
   static bool NgxFetchHandleBody(ngx_connection_t* c);
-
-      // Cancel the fetch when it's timeout
+  // Cancel the fetch when it's timeout.
   static void NgxFetchTimeout(ngx_event_t* tev);
 
-      // Add the pagespeed User-Agent
+  // Add the pagespeed User-Agent.
   void FixUserAgent();
   void FixHost();
 
@@ -125,12 +121,13 @@ namespace net_instaweb {
   AsyncFetch* async_fetch_;
   ResponseHeadersParser parser_;
   MessageHandler* message_handler_;
-      size_t bytes_received_;
+  int64 bytes_received_;
   int64 fetch_start_ms_;
   int64 fetch_end_ms_;
   int64 timeout_ms_;
   bool done_;
   int64 content_length_;
+  bool content_length_known_;
 
   struct sockaddr_in sin_;
   ngx_log_t* log_;
@@ -148,4 +145,4 @@ namespace net_instaweb {
 
 }  // namespace net_instaweb
 
-#endif  // NET_INSTAWEB_NGX_FETCHER_H_
+#endif  // NET_INSTAWEB_NGX_FETCH_H_

src/ngx_list_iterator.cc

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Author: jefftk@google.com (Jeff Kaufman)
+
+#include "ngx_list_iterator.h"
+
+namespace net_instaweb {
+
+NgxListIterator::NgxListIterator(ngx_list_part_t* part) :
+    part_(part),
+    index_within_part_(0) {}
+
+ngx_table_elt_t* NgxListIterator::Next() {
+  if (index_within_part_ >= part_->nelts) {
+    if (part_->next == NULL) {
+      return NULL;
+    }
+    part_ = part_->next;
+    index_within_part_ = 0;
+  }
+  ngx_table_elt_t* elts = static_cast<ngx_table_elt_t*>(part_->elts);
+  return &elts[index_within_part_++];  // Intentional post-increment.
+}
+
+}  // namespace net_instaweb

src/ngx_list_iterator.h

@@ -15,32 +15,36 @@
  */
 
 // Author: jefftk@google.com (Jeff Kaufman)
+//
+// Simplifies iteration over nginx lists.
+//
+//
 
-#include "ngx_thread_system.h"
+#ifndef NGX_LIST_ITERATOR_H_
+#define NGX_LIST_ITERATOR_H_
 
-#include "apr_thread_proc.h"
+extern "C" {
+#include <ngx_http.h>
+}
+
+#include "base/basictypes.h"
 
 namespace net_instaweb {
 
-class Timer;
+class NgxListIterator {
+ public:
+  explicit NgxListIterator(ngx_list_part_t* part);
 
-NgxThreadSystem::NgxThreadSystem() : may_start_threads_(false) {}
+  // Return the next element of the list if there is one, NULL otherwise.
+  ngx_table_elt_t* Next();
 
-NgxThreadSystem::~NgxThreadSystem() {}
+ private:
+  ngx_list_part_t* part_;
+  ngx_uint_t index_within_part_;
 
-void NgxThreadSystem::PermitThreadStarting() {
-  CHECK(!may_start_threads_);
-  may_start_threads_ = true;
-}
-
-void NgxThreadSystem::BeforeThreadRunHook() {
-  // We disable all signals here, since the nginx worker process is expecting to
-  // catch them and pagespeed doesn't use signals.
-  apr_setup_signal_thread();
-
-  // If this fails you can get a backtrace from gdb by setting a breakpoint on
-  // "pthread_create".
-  CHECK(may_start_threads_);
-}
+  DISALLOW_COPY_AND_ASSIGN(NgxListIterator);
+};
 
 }  // namespace net_instaweb
+
+#endif  // NGX_LIST_ITERATOR_H_

src/ngx_message_handler.cc

@@ -18,13 +18,13 @@
 
 #include <signal.h>
 
-#include "apr_time.h"
-
 #include "net/instaweb/util/public/abstract_mutex.h"
 #include "net/instaweb/util/public/debug.h"
 #include "net/instaweb/util/public/shared_circular_buffer.h"
 #include "net/instaweb/util/public/string_util.h"
 #include "net/instaweb/public/version.h"
+#include "pagespeed/kernel/base/posix_timer.h"
+#include "pagespeed/kernel/base/time_util.h"
 
 namespace {
 
@@ -118,10 +118,9 @@ void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
   // Prepend time and severity to message.
   // Format is [time] [severity] [pid] message.
   GoogleString message;
-  char time_buffer[APR_CTIME_LEN + 1];
-  const char* time = time_buffer;
-  apr_status_t status = apr_ctime(time_buffer, apr_time_now());
-  if (status != APR_SUCCESS) {
+  GoogleString time;
+  PosixTimer timer;
+  if (!ConvertTimeToString(timer.NowMs(), &time)) {
     time = "?";
   }
   StrAppend(&message, "[", time, "] ",
@@ -149,14 +148,4 @@ void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,
   }
 }
 
-// TODO(sligocki): It'd be nice not to do so much string copying.
-GoogleString NgxMessageHandler::Format(const char* msg, va_list args) {
-  GoogleString buffer;
-
-  // Ignore the name of this routine: it formats with vsnprintf.
-  // See base/stringprintf.cc.
-  StringAppendV(&buffer, msg, args);
-  return buffer;
-}
-
 }  // namespace net_instaweb

src/ngx_message_handler.h

@@ -54,6 +54,7 @@ class NgxMessageHandler : public GoogleMessageHandler {
   // Messages logged before that will be passed on to handler_;
   void set_buffer(SharedCircularBuffer* buff);
   void set_log(ngx_log_t* log) { log_ = log; }
+  ngx_log_t* log() { return log_; }
 
   void SetPidString(const int64 pid) {
     pid_string_ = StrCat("[", Integer64ToString(pid), "]");
@@ -69,7 +70,6 @@ class NgxMessageHandler : public GoogleMessageHandler {
 
  private:
   ngx_uint_t GetNgxLogLevel(MessageType type);
-  GoogleString Format(const char* msg, va_list args);
 
   scoped_ptr<AbstractMutex> mutex_;
   GoogleString pid_string_;

src/ngx_pagespeed.h

@@ -43,10 +43,9 @@ class GzipInflater;
 class NgxBaseFetch;
 class ProxyFetch;
 class RewriteDriver;
-
-}  // namespace net_instaweb
-
-namespace ngx_psol {
+class RequestHeaders;
+class ResponseHeaders;
+class InPlaceResourceRecorder;
 
 // Allocate chain links and buffers from the supplied pool, and copy over the
 // data from the string piece.  If the string piece is empty, return
@@ -75,24 +74,55 @@ StringPiece str_to_string_piece(ngx_str_t s);
 // Allocate memory out of the pool for the string piece, and copy the contents
 // over.  Returns NULL if we can't get memory.
 char* string_piece_to_pool_string(ngx_pool_t* pool, StringPiece sp);
-ngx_int_t copy_response_headers_to_ngx(
-    ngx_http_request_t* r,
-    const net_instaweb::ResponseHeaders& pagespeed_headers);
+
+enum PreserveCachingHeaders {
+  kPreserveAllCachingHeaders,  // Cache-Control, ETag, Last-Modified, etc
+  kPreserveOnlyCacheControl,   // Only Cache-Control.
+  kDontPreserveHeaders,
+};
 
 typedef struct {
-  net_instaweb::ProxyFetch* proxy_fetch;
-  net_instaweb::NgxBaseFetch* base_fetch;
-  net_instaweb::RewriteDriver* driver;
-  bool data_received;
-  int pipe_fd;
+  NgxBaseFetch* base_fetch;
+
   ngx_connection_t* pagespeed_connection;
   ngx_http_request_t* r;
-  bool is_resource_fetch;
-  bool sent_headers;
+
+  bool html_rewrite;
+  bool in_place;
+
   bool write_pending;
-  net_instaweb::GzipInflater* inflater_;
+  bool fetch_done;
+
+  PreserveCachingHeaders preserve_caching_headers;
+
+  // for html rewrite
+  ProxyFetch* proxy_fetch;
+  GzipInflater* inflater_;
+
+  // for in place resource
+  RewriteDriver* driver;
+  InPlaceResourceRecorder* recorder;
+  ResponseHeaders* ipro_response_headers;
+
+  // We need to remember the URL here as well since we may modify what NGX
+  // gets by stripping our special query params and honoring X-Forwarded-Proto.
+  GoogleString url_string;
 } ps_request_ctx_t;
 
-}  // namespace ngx_psol
+
+void copy_request_headers_from_ngx(const ngx_http_request_t* r,
+                                   RequestHeaders* headers);
+
+void copy_response_headers_from_ngx(const ngx_http_request_t* r,
+                                    ResponseHeaders* headers);
+
+ngx_int_t copy_response_headers_to_ngx(
+    ngx_http_request_t* r,
+    const ResponseHeaders& pagespeed_headers,
+    PreserveCachingHeaders preserve_caching_headers);
+
+StringPiece ps_determine_host(ngx_http_request_t* r);
+
+}  // namespace net_instaweb
 
 #endif  // NGX_PAGESPEED_H_

src/ngx_request_context.cc

@@ -1,82 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "ngx_request_context.h"
-
-extern "C" {
-  #include <ngx_http.h>
-}
-
-#include "ngx_pagespeed.h"
-
-#include "base/logging.h"
-#include "net/instaweb/http/public/meta_data.h"
-
-namespace net_instaweb {
-
-NgxRequestContext::NgxRequestContext(AbstractMutex* logging_mutex,
-                                     Timer* timer,
-                                     ngx_http_request_t* r)
-    : RequestContext(logging_mutex, timer),
-      local_port_(-1) {
-  // Note that at the time we create a RequestContext we have full
-  // access to the nginx internal request structure.  However,
-  // due to Cloning and (I believe) Detaching, we can initiate fetches after
-  // http_http_request_t* has been retired.  So deep-copy the bits we need
-  // at the time we create our RequestContext.
-
-  // Save our own IP as well, LoopbackRouteFetcher will need it.
-
-  // Based on ngx_http_variable_server_port.
-  bool port_set = false;
-#if (NGX_HAVE_INET6)
-  if (r->connection->local_sockaddr->sa_family == AF_INET6) {
-    local_port_ = ntohs(reinterpret_cast<struct sockaddr_in6*>(
-        r->connection->local_sockaddr)->sin6_port);
-    port_set = true;
-  }
-#endif
-  if (!port_set) {
-    local_port_ = ntohs(reinterpret_cast<struct sockaddr_in*>(
-        r->connection->local_sockaddr)->sin_port);
-  }
-
-  ngx_str_t  s;
-  u_char addr[NGX_SOCKADDR_STRLEN];
-  s.len = NGX_SOCKADDR_STRLEN;
-  s.data = addr;
-  ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &s, 0);
-  if (rc != NGX_OK) {
-    s.len = 0;
-  }
-  local_ip_ =  ngx_psol::str_to_string_piece(s).as_string();
-}
-
-NgxRequestContext::~NgxRequestContext() {
-}
-
-NgxRequestContext* NgxRequestContext::DynamicCast(RequestContext* rc) {
-  if (rc == NULL) {
-    return NULL;
-  }
-  NgxRequestContext* out = dynamic_cast<NgxRequestContext*>(rc);
-  DCHECK(out != NULL) << "Invalid request conversion. Do not rely on RTTI for "
-                      << "functional behavior. Ngx handling flows must use "
-                      << "NgxRequestContexts.";
-  return out;
-}
-
-}  // namespace net_instaweb

src/ngx_request_context.h

@@ -1,64 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: oschaaf@google.com (Otto van der Schaaf)
-//
-// I tried to keep this as close to ApacheRequestContext as possible.
-// Captures the NGINX request details in our request context, including
-// the port (used for loopback fetches).
-
-#ifndef NGX_REQUEST_CONTEXT_H_
-#define NGX_REQUEST_CONTEXT_H_
-
-#include "ngx_pagespeed.h"
-
-#include "net/instaweb/http/public/request_context.h"
-#include "net/instaweb/util/public/basictypes.h"
-#include "net/instaweb/util/public/string.h"
-#include "net/instaweb/util/public/string_util.h"
-
-
-namespace net_instaweb {
-
-class AbstractMutex;
-class Timer;
-
-class NgxRequestContext : public RequestContext {
- public:
-  NgxRequestContext(AbstractMutex* logging_mutex,
-                    Timer* timer,
-                    ngx_http_request_t* ps_request_context);
-
-  // Returns rc as an NgxRequestContext* if it is one and CHECK
-  // fails if it is not. Returns NULL if rc is NULL.
-  static NgxRequestContext* DynamicCast(RequestContext* rc);
-
-  int local_port() const { return local_port_; }
-  const GoogleString& local_ip() const { return local_ip_; }
-
- protected:
-  virtual ~NgxRequestContext();
-
- private:
-  int local_port_;
-  GoogleString local_ip_;
-
-  DISALLOW_COPY_AND_ASSIGN(NgxRequestContext);
-};
-
-}  // namespace net_instaweb
-
-#endif  // NGX_REQUEST_CONTEXT_H_

src/ngx_rewrite_driver_factory.cc

@@ -24,25 +24,25 @@
 #include "ngx_message_handler.h"
 #include "ngx_rewrite_options.h"
 #include "ngx_server_context.h"
-#include "ngx_thread_system.h"
 #include "ngx_url_async_fetcher.h"
-#include "pthread_shared_mem.h"
 
-#include "net/instaweb/apache/serf_url_async_fetcher.h"
 #include "net/instaweb/http/public/content_type.h"
-#include "net/instaweb/http/public/fake_url_async_fetcher.h"
+#include "net/instaweb/http/public/rate_controller.h"
+#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
 #include "net/instaweb/http/public/wget_url_fetcher.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
 #include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/in_place_resource_recorder.h"
+#include "net/instaweb/system/public/serf_url_async_fetcher.h"
 #include "net/instaweb/system/public/system_caches.h"
 #include "net/instaweb/system/public/system_rewrite_options.h"
 #include "net/instaweb/util/public/google_message_handler.h"
 #include "net/instaweb/util/public/null_shared_mem.h"
+#include "net/instaweb/util/public/posix_timer.h"
 #include "net/instaweb/util/public/property_cache.h"
 #include "net/instaweb/util/public/scheduler_thread.h"
-#include "net/instaweb/util/public/posix_timer.h"
 #include "net/instaweb/util/public/shared_circular_buffer.h"
 #include "net/instaweb/util/public/shared_mem_statistics.h"
 #include "net/instaweb/util/public/slow_worker.h"
@@ -50,6 +50,7 @@
 #include "net/instaweb/util/public/string.h"
 #include "net/instaweb/util/public/string_util.h"
 #include "net/instaweb/util/public/thread_system.h"
+#include "pagespeed/kernel/thread/pthread_shared_mem.h"
 
 namespace net_instaweb {
 
@@ -65,33 +66,26 @@ class Writer;
 const char NgxRewriteDriverFactory::kStaticAssetPrefix[] =
     "/ngx_pagespeed_static/";
 
-namespace {
-
-const char kShutdownCount[] = "child_shutdown_count";
-
-}  // namespace
+class SharedCircularBuffer;
 
 NgxRewriteDriverFactory::NgxRewriteDriverFactory(
-    NgxThreadSystem* ngx_thread_system)
-    : SystemRewriteDriverFactory(ngx_thread_system),
-      ngx_thread_system_(ngx_thread_system),
-      // TODO(oschaaf): mod_pagespeed ifdefs this:
-      shared_mem_runtime_(new ngx::PthreadSharedMem()),
+    const ProcessContext& process_context,
+    SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
+    : SystemRewriteDriverFactory(process_context, system_thread_system,
+        NULL /* default shared memory runtime */, hostname, port),
       main_conf_(NULL),
       threads_started_(false),
       use_per_vhost_statistics_(false),
-      is_root_process_(true),
       ngx_message_handler_(new NgxMessageHandler(thread_system()->NewMutex())),
       ngx_html_parse_message_handler_(
           new NgxMessageHandler(thread_system()->NewMutex())),
       install_crash_handler_(false),
-      message_buffer_size_(0),
-      shared_circular_buffer_(NULL),
-      statistics_frozen_(false),
-      ngx_url_async_fetcher_(NULL),
       log_(NULL),
       resolver_timeout_(NGX_CONF_UNSET_MSEC),
-      use_native_fetcher_(false) {
+      use_native_fetcher_(false),
+      ngx_shared_circular_buffer_(NULL),
+      hostname_(hostname.as_string()),
+      port_(port) {
   InitializeDefaultOptions();
   default_options()->set_beacon_url("/ngx_pagespeed_beacon");
   SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -100,61 +94,33 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
   system_options->set_avoid_renaming_introspective_javascript(true);
   set_message_handler(ngx_message_handler_);
   set_html_parse_message_handler(ngx_html_parse_message_handler_);
-
-  // see https://code.google.com/p/modpagespeed/issues/detail?id=672
-  int thread_limit = 1;
-  caches_.reset(
-      new SystemCaches(this, shared_mem_runtime_.get(), thread_limit));
 }
 
 NgxRewriteDriverFactory::~NgxRewriteDriverFactory() {
   ShutDown();
-
-  CHECK(uninitialized_server_contexts_.empty() || is_root_process_);
+  ngx_shared_circular_buffer_ = NULL;
   STLDeleteElements(&uninitialized_server_contexts_);
-  shared_mem_statistics_.reset(NULL);
 }
 
 Hasher* NgxRewriteDriverFactory::NewHasher() {
   return new MD5Hasher;
 }
 
-UrlFetcher* NgxRewriteDriverFactory::DefaultUrlFetcher() {
-  CHECK(false) << "Nothing should still be using DefaultUrlFetcher()";
-  return NULL;
-}
-
-UrlAsyncFetcher* NgxRewriteDriverFactory::DefaultAsyncUrlFetcher() {
-  const char* fetcher_proxy = "";
-  if (main_conf_ != NULL) {
-    fetcher_proxy = main_conf_->fetcher_proxy().c_str();
-  }
-
+UrlAsyncFetcher* NgxRewriteDriverFactory::AllocateFetcher(
+    SystemRewriteOptions* config) {
   if (use_native_fetcher_) {
-    net_instaweb::NgxUrlAsyncFetcher* fetcher =
-        new net_instaweb::NgxUrlAsyncFetcher(
-            fetcher_proxy,
+    NgxUrlAsyncFetcher* fetcher = new NgxUrlAsyncFetcher(
+        config->fetcher_proxy().c_str(),
         log_,
         resolver_timeout_,
-            25000,
+        config->blocking_fetch_timeout_ms(),
         resolver_,
         thread_system(),
         message_handler());
-    ngx_url_async_fetcher_ = fetcher;
+    ngx_url_async_fetchers_.push_back(fetcher);
     return fetcher;
   } else {
-    net_instaweb::SerfUrlAsyncFetcher* fetcher =
-        new net_instaweb::SerfUrlAsyncFetcher(
-            fetcher_proxy,
-            NULL,
-            thread_system(),
-            statistics(),
-            timer(),
-            2500,
-            message_handler());
-    // Make sure we don't block the nginx event loop
-    fetcher->set_force_threaded(true);
-    return fetcher;
+    return SystemRewriteDriverFactory::AllocateFetcher(config);
   }
 }
 
@@ -179,43 +145,25 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
   return NULL;
 }
 
-void NgxRewriteDriverFactory::SetupCaches(ServerContext* server_context) {
-  caches_->SetupCaches(server_context);
-
-  server_context->set_enable_property_cache(true);
-  PropertyCache* pcache = server_context->page_property_cache();
-  if (pcache->GetCohort(RewriteDriver::kBeaconCohort) == NULL) {
-    pcache->AddCohort(RewriteDriver::kBeaconCohort);
-  }
-  if (pcache->GetCohort(RewriteDriver::kDomCohort) == NULL) {
-    pcache->AddCohort(RewriteDriver::kDomCohort);
-  }
-}
-
 RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
-  NgxRewriteOptions* options = new NgxRewriteOptions();
+  NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
   options->SetRewriteLevel(RewriteOptions::kCoreFilters);
   return options;
 }
 
-
 void NgxRewriteDriverFactory::InitStaticAssetManager(
     StaticAssetManager* static_asset_manager) {
   static_asset_manager->set_library_url_prefix(kStaticAssetPrefix);
 }
 
-void NgxRewriteDriverFactory::PrintMemCacheStats(GoogleString* out) {
-  // TODO(morlovich): Port the client code to proper API, so it gets
-  // shm stats, too.
-  caches_->PrintCacheStats(SystemCaches::kIncludeMemcached, out);
-}
-
-bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetcher() {
-  if (ngx_url_async_fetcher_ == NULL) {
-    return true;
-  }
+bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
   log_ = ngx_cycle->log;
-  return ngx_url_async_fetcher_->Init();
+  for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {
+    if (!ngx_url_async_fetchers_[i]->Init()) {
+      return false;
+    }
+  }
+  return true;
 }
 
 bool NgxRewriteDriverFactory::CheckResolver() {
@@ -225,52 +173,39 @@ bool NgxRewriteDriverFactory::CheckResolver() {
   return true;
 }
 
-void NgxRewriteDriverFactory::StopCacheActivity() {
-  RewriteDriverFactory::StopCacheActivity();
-  caches_->StopCacheActivity();
-}
-
-NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext() {
-  NgxServerContext* server_context = new NgxServerContext(this);
+NgxServerContext* NgxRewriteDriverFactory::MakeNgxServerContext(
+    StringPiece hostname, int port) {
+  NgxServerContext* server_context = new NgxServerContext(this, hostname, port);
   uninitialized_server_contexts_.insert(server_context);
   return server_context;
 }
 
+ServerContext* NgxRewriteDriverFactory::NewDecodingServerContext() {
+  ServerContext* sc = new NgxServerContext(this, hostname_, port_);
+  InitStubDecodingServerContext(sc);
+  return sc;
+}
+
 ServerContext* NgxRewriteDriverFactory::NewServerContext() {
   LOG(DFATAL) << "MakeNgxServerContext should be used instead";
   return NULL;
 }
 
-void NgxRewriteDriverFactory::ShutDown() {
-  StopCacheActivity();
-  if (!is_root_process_) {
-    Variable* child_shutdown_count = statistics()->GetVariable(kShutdownCount);
-    child_shutdown_count->Add(1);
-  }
-
-  RewriteDriverFactory::ShutDown();
-  caches_->ShutDown(message_handler());
-
+void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
   ngx_message_handler_->set_buffer(NULL);
   ngx_html_parse_message_handler_->set_buffer(NULL);
-
-  if (is_root_process_) {
-    // Cleanup statistics.
-    // TODO(morlovich): This looks dangerous with async.
-    if (shared_mem_statistics_.get() != NULL) {
-      shared_mem_statistics_->GlobalCleanup(message_handler());
-    }
-    if (shared_circular_buffer_ != NULL) {
-      shared_circular_buffer_->GlobalCleanup(message_handler());
-    }
+  for (NgxMessageHandlerSet::iterator p =
+           server_context_message_handlers_.begin();
+       p != server_context_message_handlers_.end(); ++p) {
+    (*p)->set_buffer(NULL);
   }
+  server_context_message_handlers_.clear();
 }
 
 void NgxRewriteDriverFactory::StartThreads() {
   if (threads_started_) {
     return;
   }
-  ngx_thread_system_->PermitThreadStarting();
   // TODO(jefftk): use a native nginx timer instead of running our own thread.
   // See issue #111.
   SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
@@ -280,114 +215,44 @@ void NgxRewriteDriverFactory::StartThreads() {
   threads_started_ = true;
 }
 
-void NgxRewriteDriverFactory::ParentOrChildInit(ngx_log_t* log) {
+void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
+  net_instaweb::log_message_handler::Install(log);
   if (install_crash_handler_) {
     NgxMessageHandler::InstallCrashHandler(log);
   }
   ngx_message_handler_->set_log(log);
   ngx_html_parse_message_handler_->set_log(log);
-  SharedCircularBufferInit(is_root_process_);
 }
 
-// TODO(jmarantz): make this per-vhost.
-void NgxRewriteDriverFactory::SharedCircularBufferInit(bool is_root) {
-  // Set buffer size to 0 means turning it off
-  if (shared_mem_runtime() != NULL && (message_buffer_size_ != 0)) {
-    // TODO(jmarantz): it appears that filename_prefix() is not actually
-    // established at the time of this construction, calling into question
-    // whether we are naming our shared-memory segments correctly.
-    shared_circular_buffer_.reset(new SharedCircularBuffer(
-        shared_mem_runtime(),
-        message_buffer_size_,
-        filename_prefix().as_string(),
-        "foo.com" /*hostname_identifier()*/));
-    if (shared_circular_buffer_->InitSegment(is_root, message_handler())) {
-      ngx_message_handler_->set_buffer(shared_circular_buffer_.get());
-      ngx_html_parse_message_handler_->set_buffer(
-          shared_circular_buffer_.get());
-    }
-  }
+void NgxRewriteDriverFactory::SetCircularBuffer(
+    SharedCircularBuffer* buffer) {
+  ngx_shared_circular_buffer_ = buffer;
+  ngx_message_handler_->set_buffer(buffer);
+  ngx_html_parse_message_handler_->set_buffer(buffer);
 }
 
-void NgxRewriteDriverFactory::RootInit(ngx_log_t* log) {
-  net_instaweb::log_message_handler::Install(log);
-
-  ParentOrChildInit(log);
-
-  // Let SystemCaches know about the various paths we have in configuration
-  // first, as well as the memcached instances.
-  for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
-           e = uninitialized_server_contexts_.end(); p != e; ++p) {
-    NgxServerContext* server_context = *p;
-    caches_->RegisterConfig(server_context->config());
-  }
-
-  caches_->RootInit();
-}
-
-void NgxRewriteDriverFactory::ChildInit(ngx_log_t* log) {
-  is_root_process_ = false;
-
-  ParentOrChildInit(log);
-  if (shared_mem_statistics_.get() != NULL) {
-    shared_mem_statistics_->Init(false, message_handler());
-  }
-
-  caches_->ChildInit();
-  for (NgxServerContextSet::iterator p = uninitialized_server_contexts_.begin(),
-           e = uninitialized_server_contexts_.end(); p != e; ++p) {
-    NgxServerContext* server_context = *p;
-    server_context->ChildInit();
-  }
-
-  uninitialized_server_contexts_.clear();
-}
-
-// Initializes global statistics object if needed, using factory to
-// help with the settings if needed.
-// Note: does not call set_statistics() on the factory.
-Statistics* NgxRewriteDriverFactory::MakeGlobalSharedMemStatistics(
-    bool logging, int64 logging_interval_ms,
-    const GoogleString& logging_file_base) {
-  if (shared_mem_statistics_.get() == NULL) {
-    shared_mem_statistics_.reset(AllocateAndInitSharedMemStatistics(
-        "global", logging, logging_interval_ms, logging_file_base));
-  }
-  DCHECK(!statistics_frozen_);
-  statistics_frozen_ = true;
-  SetStatistics(shared_mem_statistics_.get());
-  return shared_mem_statistics_.get();
-}
-
-SharedMemStatistics* NgxRewriteDriverFactory::
-AllocateAndInitSharedMemStatistics(
-    const StringPiece& name, const bool logging,
-    const int64 logging_interval_ms,
-    const GoogleString& logging_file_base) {
-  // Note that we create the statistics object in the parent process, and
-  // it stays around in the kids but gets reinitialized for them
-  // inside ChildInit(), called from pagespeed_child_init.
-  SharedMemStatistics* stats = new SharedMemStatistics(
-      logging_interval_ms, StrCat(logging_file_base, name), logging,
-      StrCat(filename_prefix(), name), shared_mem_runtime(), message_handler(),
-      file_system(), timer());
-  InitStats(stats);
-  stats->Init(true, message_handler());
-  return stats;
+void NgxRewriteDriverFactory::SetServerContextMessageHandler(
+    ServerContext* server_context, ngx_log_t* log) {
+  NgxMessageHandler* handler = new NgxMessageHandler(
+      thread_system()->NewMutex());
+  handler->set_log(log);
+  // The ngx_shared_circular_buffer_ will be NULL if MessageBufferSize hasn't
+  // been raised from its default of 0.
+  handler->set_buffer(ngx_shared_circular_buffer_);
+  server_context_message_handlers_.insert(handler);
+  defer_cleanup(new Deleter<NgxMessageHandler>(handler));
+  server_context->set_message_handler(handler);
 }
 
 void NgxRewriteDriverFactory::InitStats(Statistics* statistics) {
   // Init standard PSOL stats.
+  SystemRewriteDriverFactory::InitStats(statistics);
   RewriteDriverFactory::InitStats(statistics);
+  RateController::InitStats(statistics);
 
   // Init Ngx-specific stats.
   NgxServerContext::InitStats(statistics);
-  SystemCaches::InitStats(statistics);
-  SerfUrlAsyncFetcher::InitStats(statistics);
-  PropertyCache::InitCohortStats(RewriteDriver::kBeaconCohort, statistics);
-  PropertyCache::InitCohortStats(RewriteDriver::kDomCohort, statistics);
-
-  statistics->AddVariable(kShutdownCount);
+  InPlaceResourceRecorder::InitStats(statistics);
 }
 
 }  // namespace net_instaweb

src/ngx_rewrite_driver_factory.h

@@ -28,7 +28,6 @@ extern "C" {
 
 #include <set>
 
-#include "apr_pools.h"
 #include "net/instaweb/system/public/system_rewrite_driver_factory.h"
 #include "net/instaweb/util/public/md5_hasher.h"
 #include "net/instaweb/util/public/scoped_ptr.h"
@@ -39,103 +38,64 @@ extern "C" {
 
 namespace net_instaweb {
 
-class AbstractSharedMem;
 class NgxMessageHandler;
 class NgxRewriteOptions;
 class NgxServerContext;
-class NgxThreadSystem;
 class NgxUrlAsyncFetcher;
 class SharedCircularBuffer;
 class SharedMemRefererStatistics;
-class SharedMemStatistics;
 class SlowWorker;
 class StaticAssetManager;
 class Statistics;
-class SystemCaches;
+class SystemThreadSystem;
 
 class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  public:
   static const char kStaticAssetPrefix[];
 
   // We take ownership of the thread system.
-  explicit NgxRewriteDriverFactory(NgxThreadSystem* ngx_thread_system);
+  explicit NgxRewriteDriverFactory(
+      const ProcessContext& process_context,
+      SystemThreadSystem* system_thread_system, StringPiece hostname, int port);
   virtual ~NgxRewriteDriverFactory();
   virtual Hasher* NewHasher();
-  virtual UrlFetcher* DefaultUrlFetcher();
-  virtual UrlAsyncFetcher* DefaultAsyncUrlFetcher();
+  virtual UrlAsyncFetcher* AllocateFetcher(SystemRewriteOptions* config);
   virtual MessageHandler* DefaultHtmlParseMessageHandler();
   virtual MessageHandler* DefaultMessageHandler();
   virtual FileSystem* DefaultFileSystem();
   virtual Timer* DefaultTimer();
   virtual NamedLockManager* DefaultLockManager();
-  virtual void SetupCaches(ServerContext* server_context);
   // Create a new RewriteOptions.  In this implementation it will be an
   // NgxRewriteOptions.
   virtual RewriteOptions* NewRewriteOptions();
   // Initializes the StaticAssetManager.
   virtual void InitStaticAssetManager(
       StaticAssetManager* static_asset_manager);
-  // Print out details of all the connections to memcached servers.
-  void PrintMemCacheStats(GoogleString* out);
-  bool InitNgxUrlAsyncFetcher();
+  virtual ServerContext* NewDecodingServerContext();
+  bool InitNgxUrlAsyncFetchers();
   // Check resolver configured or not.
   bool CheckResolver();
 
-  // Release all the resources. It also calls the base class ShutDown to
-  // release the base class resources.
   // Initializes all the statistics objects created transitively by
   // NgxRewriteDriverFactory, including nginx-specific and
   // platform-independent statistics.
   static void InitStats(Statistics* statistics);
-  virtual void ShutDown();
-  virtual void StopCacheActivity();
-  NgxServerContext* MakeNgxServerContext();
-  ServerContext* NewServerContext();
-  AbstractSharedMem* shared_mem_runtime() const {
-    return shared_mem_runtime_.get();
-  }
-
-  SystemCaches* caches() { return caches_.get(); }
+  NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
+  virtual ServerContext* NewServerContext();
 
   // Starts pagespeed threads if they've not been started already.  Must be
   // called after the caller has finished any forking it intends to do.
   void StartThreads();
-  // This helper method contains init procedures invoked by both RootInit()
-  // and ChildInit()
-  void ParentOrChildInit(ngx_log_t* log);
-  // For shared memory resources the general setup we follow is to have the
-  // first running process (aka the root) create the necessary segments and
-  // fill in their shared data structures, while processes created to actually
-  // handle requests attach to already existing shared data structures.
-  //
-  // During normal server startup[1], RootInit() is called from the nginx hooks
-  // in the root process for the first task, and then ChildInit() is called in
-  // any child process.
-  //
-  // Keep in mind, however, that when fork() is involved a process may
-  // effectively see both calls, in which case the 'ChildInit' call would
-  // come second and override the previous root status. Both calls are also
-  // invoked in the debug single-process mode.
-  //
-  // [1] Besides normal startup, nginx also uses a temporary process to
-  // syntax check the config file. That basically looks like a complete
-  // normal startup and shutdown to the code.
-  bool is_root_process() const { return is_root_process_; }
-  void RootInit(ngx_log_t* log);
-  void ChildInit(ngx_log_t* log);
-  void SharedCircularBufferInit(bool is_root);
-  // Build global shared-memory statistics.  This is invoked if at least
-  // one server context (global or VirtualHost) enables statistics.
-  Statistics* MakeGlobalSharedMemStatistics(bool logging,
-                                            int64 logging_interval_ms,
-                                            const GoogleString& logging_file);
 
-  // Creates and ::Initializes a shared memory statistics object.
-  SharedMemStatistics* AllocateAndInitSharedMemStatistics(
-      const StringPiece& name, const bool logging,
-      const int64 logging_interval_ms, const GoogleString& logging_file);
+  void SetServerContextMessageHandler(ServerContext* server_context,
+                                      ngx_log_t* log);
 
   NgxMessageHandler* ngx_message_handler() { return ngx_message_handler_; }
+
+  virtual void NonStaticInitStats(Statistics* statistics) {
+    InitStats(statistics);
+  }
+
   void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
 
   bool use_per_vhost_statistics() const {
@@ -150,12 +110,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_install_crash_handler(bool x) {
     install_crash_handler_ = x;
   }
-  bool message_buffer_size() const {
-    return message_buffer_size_;
-  }
-  void set_message_buffer_size(int x) {
-    message_buffer_size_ = x;
-  }
   void set_resolver(ngx_resolver_t* resolver) {
     resolver_ = resolver;
   }
@@ -169,6 +123,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   void set_use_native_fetcher(bool x) {
     use_native_fetcher_ = x;
   }
+  void set_rate_limit_background_fetches(bool x) {
+    rate_limit_background_fetches_ = x;
+  }
 
   // We use a beacon handler to collect data for critical images,
   // css, etc., so filters should be configured accordingly.
@@ -178,39 +135,43 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
     return true;
   }
 
+  void LoggingInit(ngx_log_t* log);
+
+  virtual void ShutDownMessageHandlers();
+
+  virtual void SetCircularBuffer(SharedCircularBuffer* buffer);
+
  private:
-  NgxThreadSystem* ngx_thread_system_;
   Timer* timer_;
-  scoped_ptr<AbstractSharedMem> shared_mem_runtime_;
 
   // main_conf will have only options set in the main block.  It may be NULL,
   // and we do not take ownership.
   NgxRewriteOptions* main_conf_;
-  typedef std::set<NgxServerContext*> NgxServerContextSet;
-  NgxServerContextSet uninitialized_server_contexts_;
-
-  // Manages all our caches & lock managers.
-  scoped_ptr<SystemCaches> caches_;
 
   bool threads_started_;
   // If true, we'll have a separate statistics object for each vhost
   // (along with a global aggregate), rather than just a single object
   // aggregating all of them.
   bool use_per_vhost_statistics_;
-  bool is_root_process_;
   NgxMessageHandler* ngx_message_handler_;
   NgxMessageHandler* ngx_html_parse_message_handler_;
   bool install_crash_handler_;
-  int message_buffer_size_;
-  scoped_ptr<SharedCircularBuffer> shared_circular_buffer_;
-  scoped_ptr<SharedMemStatistics> shared_mem_statistics_;
-  bool statistics_frozen_;
 
-  NgxUrlAsyncFetcher* ngx_url_async_fetcher_;
+  std::vector<NgxUrlAsyncFetcher*> ngx_url_async_fetchers_;
   ngx_log_t* log_;
   ngx_msec_t resolver_timeout_;
   ngx_resolver_t* resolver_;
   bool use_native_fetcher_;
+  bool rate_limit_background_fetches_;
+  typedef std::set<NgxMessageHandler*> NgxMessageHandlerSet;
+  NgxMessageHandlerSet server_context_message_handlers_;
+
+  // Owned by the superclass.
+  // TODO(jefftk): merge the nginx and apache ways of doing this.
+  SharedCircularBuffer* ngx_shared_circular_buffer_;
+
+  GoogleString hostname_;
+  int port_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };

src/ngx_rewrite_options.cc

@@ -35,9 +35,53 @@ extern "C" {
 
 namespace net_instaweb {
 
+namespace {
+
+// These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
+// indicates that they can not be set at the directory/location level. They set
+// options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
+// They are not alphabetized on purpose, but rather left in the same order as in
+// mod_instaweb.cc in case we end up needing to compare.
+// TODO(oschaaf): this duplication is a short term solution.
+const char* const server_only_options[] = {
+  "FetcherTimeoutMs",
+  "FetchProxy",
+  "ForceCaching",
+  "GeneratedFilePrefix",
+  "ImgMaxRewritesAtOnce",
+  "InheritVHostConfig",
+  "InstallCrashHandler",
+  "MessageBufferSize",
+  "NumRewriteThreads",
+  "NumExpensiveRewriteThreads",
+  "TrackOriginalContentLength",
+  "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
+  "BlockingRewriteRefererUrls",
+  "CreateSharedMemoryMetadataCache",
+  "LoadFromFile",
+  "LoadFromFileMatch",
+  "LoadFromFileRule",
+  "LoadFromFileRuleMatch",
+  "UseNativeFetcher"
+};
+
+// Options that can only be used in the main (http) option scope.
+const char* const main_only_options[] = {
+  "UseNativeFetcher"
+};
+
+}  // namespace
+
 RewriteOptions::Properties* NgxRewriteOptions::ngx_properties_ = NULL;
 
-NgxRewriteOptions::NgxRewriteOptions() {
+NgxRewriteOptions::NgxRewriteOptions(const StringPiece& description,
+                                     ThreadSystem* thread_system)
+    : SystemRewriteOptions(description, thread_system) {
+  Init();
+}
+
+NgxRewriteOptions::NgxRewriteOptions(ThreadSystem* thread_system)
+    : SystemRewriteOptions(thread_system) {
   Init();
 }
 
@@ -51,15 +95,11 @@ void NgxRewriteOptions::AddProperties() {
   // Nothing ngx-specific for now.
 
   MergeSubclassProperties(ngx_properties_);
-  NgxRewriteOptions config;
-  config.InitializeSignaturesAndDefaults();
-}
 
-void NgxRewriteOptions::InitializeSignaturesAndDefaults() {
-  // Calls to foo_.DoNotUseForSignatureComputation() would go here.
-
-  // Set default header value.
-  set_default_x_header_value(kModPagespeedVersion);
+  // Default properties are global but to set them the current API requires
+  // a RewriteOptions instance and we're in a static method.
+  NgxRewriteOptions dummy_config(NULL);
+  dummy_config.set_default_x_header_value(kModPagespeedVersion);
 }
 
 void NgxRewriteOptions::Initialize() {
@@ -80,6 +120,39 @@ bool NgxRewriteOptions::IsDirective(StringPiece config_directive,
   return StringCaseEqual(config_directive, compare_directive);
 }
 
+RewriteOptions::OptionScope NgxRewriteOptions::GetOptionScope(
+    StringPiece option_name) {
+  ngx_uint_t i;
+  ngx_uint_t size = sizeof(main_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(main_only_options[i], option_name)) {
+      return kProcessScopeStrict;
+    }
+  }
+
+  size = sizeof(server_only_options) / sizeof(char*);
+  for (i = 0; i < size; i++) {
+    if (StringCaseEqual(server_only_options[i], option_name)) {
+      return kServerScope;
+    }
+  }
+
+  // This could be made more efficient if RewriteOptions provided a map allowing
+  // access of options by their name. It's not too much of a worry at present
+  // since this is just during initialization.
+  for (OptionBaseVector::const_iterator it = all_options().begin();
+       it != all_options().end(); ++it) {
+    RewriteOptions::OptionBase* option = *it;
+    if (option->option_name() == option_name) {
+      // We treat kProcessScope as kProcessScopeStrict, failing to start if an
+      // option is out of place.
+      return option->scope() == kProcessScope ? kProcessScopeStrict
+                                              : option->scope();
+    }
+  }
+  return kDirectoryScope;
+}
+
 RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
     StringPiece directive, GoogleString* msg, MessageHandler* handler) {
   if (IsDirective(directive, "on")) {
@@ -94,42 +167,64 @@ RewriteOptions::OptionSettingResult NgxRewriteOptions::ParseAndSetOptions0(
   return RewriteOptions::kOptionOk;
 }
 
-
 RewriteOptions::OptionSettingResult
-    NgxRewriteOptions::ParseAndSetOptionFromEnum1(
-        OptionEnum directive, StringPiece arg,
+    NgxRewriteOptions::ParseAndSetOptionFromName1(
+        StringPiece name, StringPiece arg,
         GoogleString* msg, MessageHandler* handler) {
   // FileCachePath needs error checking.
-  if (directive == kFileCachePath) {
+  if (StringCaseEqual(name, kFileCachePath)) {
     if (!StringCaseStartsWith(arg, "/")) {
       *msg = "must start with a slash";
       return RewriteOptions::kOptionValueInvalid;
     }
   }
 
-  // TODO(jefftk): port these (no enums for them yet, even!)
-  //  DangerPermitFetchFromUnknownHosts, FetchWithGzip, ForceCaching
-
-  return SystemRewriteOptions::ParseAndSetOptionFromEnum1(
-      directive, arg, msg, handler);
+  return SystemRewriteOptions::ParseAndSetOptionFromName1(
+      name, arg, msg, handler);
 }
 
+template <class DriverFactoryT>
+RewriteOptions::OptionSettingResult ParseAndSetOptionHelper(
+    StringPiece option_value,
+    DriverFactoryT* driver_factory,
+    void (DriverFactoryT::*set_option_method)(bool)) {
+  bool parsed_value;
+  if (StringCaseEqual(option_value, "on") ||
+      StringCaseEqual(option_value, "true")) {
+    parsed_value = true;
+  } else if (StringCaseEqual(option_value, "off") ||
+             StringCaseEqual(option_value, "false")) {
+    parsed_value = false;
+  } else {
+    return RewriteOptions::kOptionValueInvalid;
+  }
+
+  (driver_factory->*set_option_method)(parsed_value);
+  return RewriteOptions::kOptionOk;
+}
+
+namespace {
+
+const char* ps_error_string_for_option(
+    ngx_pool_t* pool, StringPiece directive, StringPiece warning) {
+  GoogleString msg =
+      StrCat("\"", directive, "\" ", warning);
+  char* s = string_piece_to_pool_string(pool, msg);
+  if (s == NULL) {
+    return "failed to allocate memory";
+  }
+  return s;
+}
+
+}  // namespace
+
 // Very similar to apache/mod_instaweb::ParseDirective.
-const char*
-NgxRewriteOptions::ParseAndSetOptions(
+const char* NgxRewriteOptions::ParseAndSetOptions(
     StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-    NgxRewriteDriverFactory* driver_factory) {
+    NgxRewriteDriverFactory* driver_factory,
+    RewriteOptions::OptionScope scope) {
   CHECK_GE(n_args, 1);
 
-  int i;
-  fprintf(stderr, "Setting option from (");
-  for (i = 0 ; i < n_args ; i++) {
-    fprintf(stderr, "%s\"%s\"",
-            i == 0 ? "" : ", ",
-            args[i].as_string().c_str());
-  }
-  fprintf(stderr, ")\n");
-
   StringPiece directive = args[0];
 
   // Remove initial "ModPagespeed" if there is one.
@@ -138,6 +233,11 @@ NgxRewriteOptions::ParseAndSetOptions(
     directive.remove_prefix(mod_pagespeed.size());
   }
 
+  if (GetOptionScope(directive) > scope) {
+    return ps_error_string_for_option(
+        pool, directive, "cannot be set at this scope.");
+  }
+
   GoogleString msg;
   OptionSettingResult result;
   if (n_args == 1) {
@@ -147,29 +247,14 @@ NgxRewriteOptions::ParseAndSetOptions(
     // TODO(morlovich): Remove these special hacks, and handle these via
     // ParseAndSetOptionFromEnum1.
     if (IsDirective(directive, "UsePerVHostStatistics")) {
-        // TODO(oschaaf): mod_pagespeed has a nicer way to do this.
-        if (IsDirective(arg, "on")) {
-          driver_factory->set_use_per_vhost_statistics(true);
-          result = RewriteOptions::kOptionOk;
-        } else if (IsDirective(arg, "off")) {
-          driver_factory->set_use_per_vhost_statistics(false);
-          result = RewriteOptions::kOptionOk;
-        } else {
-          result = RewriteOptions::kOptionValueInvalid;
-        }
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_use_per_vhost_statistics);
     } else if (IsDirective(directive, "InstallCrashHandler")) {
-        // TODO(oschaaf): mod_pagespeed has a nicer way to do this.
-        if (IsDirective(arg, "on")) {
-          driver_factory->set_install_crash_handler(true);
-          result = RewriteOptions::kOptionOk;
-        } else if (IsDirective(arg, "off")) {
-          driver_factory->set_install_crash_handler(false);
-          result = RewriteOptions::kOptionOk;
-        } else {
-          result = RewriteOptions::kOptionValueInvalid;
-        }
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_install_crash_handler);
     } else if (IsDirective(directive, "MessageBufferSize")) {
-        // TODO(oschaaf): mod_pagespeed has a nicer way to do this.
       int message_buffer_size;
       bool ok = StringToInt(arg.as_string(), &message_buffer_size);
       if (ok && message_buffer_size >= 0) {
@@ -179,16 +264,25 @@ NgxRewriteOptions::ParseAndSetOptions(
         result = RewriteOptions::kOptionValueInvalid;
       }
     } else if (IsDirective(directive, "UseNativeFetcher")) {
-        // TODO(oschaaf): mod_pagespeed has a nicer way to do this.
-        if (IsDirective(arg, "on")) {
-          driver_factory->set_use_native_fetcher(true);
-          result = RewriteOptions::kOptionOk;
-        } else if (IsDirective(arg, "off")) {
-          driver_factory->set_use_native_fetcher(false);
-          result = RewriteOptions::kOptionOk;
-        } else {
-          result = RewriteOptions::kOptionValueInvalid;
-        }
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_use_native_fetcher);
+    } else if (IsDirective(directive, "RateLimitBackgroundFetches")) {
+      result = ParseAndSetOptionHelper<NgxRewriteDriverFactory>(
+          arg, driver_factory,
+          &NgxRewriteDriverFactory::set_rate_limit_background_fetches);
+    } else if (IsDirective(directive, "ForceCaching")) {
+      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
+          arg, driver_factory,
+          &SystemRewriteDriverFactory::set_force_caching);
+    } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) {
+      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
+          arg, driver_factory,
+          &SystemRewriteDriverFactory::list_outstanding_urls_on_error);
+    } else if (IsDirective(directive, "TrackOriginalContentLength")) {
+      result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
+          arg, driver_factory,
+          &SystemRewriteDriverFactory::set_track_original_content_length);
     } else {
       result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
     }
@@ -213,25 +307,22 @@ NgxRewriteOptions::ParseAndSetOptions(
     result = ParseAndSetOptionFromName3(
         directive, args[1], args[2], args[3], &msg, handler);
   } else {
-    return "unknown option";
+    return ps_error_string_for_option(
+        pool, directive, "not recognized or too many arguments");
   }
 
   switch (result) {
     case RewriteOptions::kOptionOk:
       return NGX_CONF_OK;
     case RewriteOptions::kOptionNameUnknown:
-      return "unknown option";
+      return ps_error_string_for_option(
+          pool, directive, "not recognized or too many arguments");
     case RewriteOptions::kOptionValueInvalid: {
-      GoogleString full_directive = "\"";
+      GoogleString full_directive;
       for (int i = 0 ; i < n_args ; i++) {
         StrAppend(&full_directive, i == 0 ? "" : " ", args[i]);
       }
-      StrAppend(&full_directive, "\": ", msg);
-      char* s = ngx_psol::string_piece_to_pool_string(pool, full_directive);
-      if (s == NULL) {
-        return "failed to allocate memory";
-      }
-      return s;
+      return ps_error_string_for_option(pool, full_directive, msg);
     }
   }
 
@@ -240,7 +331,8 @@ NgxRewriteOptions::ParseAndSetOptions(
 }
 
 NgxRewriteOptions* NgxRewriteOptions::Clone() const {
-  NgxRewriteOptions* options = new NgxRewriteOptions();
+  NgxRewriteOptions* options = new NgxRewriteOptions(
+      StrCat("cloned from ", description()), thread_system());
   options->Merge(*this);
   return options;
 }

src/ngx_rewrite_options.h

@@ -40,7 +40,9 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static void Initialize();
   static void Terminate();
 
-  NgxRewriteOptions();
+  NgxRewriteOptions(const StringPiece& description,
+                    ThreadSystem* thread_system);
+  explicit NgxRewriteOptions(ThreadSystem* thread_system);
   virtual ~NgxRewriteOptions() { }
 
   // args is an array of n_args StringPieces together representing a directive.
@@ -56,7 +58,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   // pool is a memory pool for allocating error strings.
   const char* ParseAndSetOptions(
       StringPiece* args, int n_args, ngx_pool_t* pool, MessageHandler* handler,
-      NgxRewriteDriverFactory* driver_factory);
+      NgxRewriteDriverFactory* driver_factory, OptionScope scope);
 
   // Make an identical copy of these options and return it.
   virtual NgxRewriteOptions* Clone() const;
@@ -85,9 +87,8 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   OptionSettingResult ParseAndSetOptions0(
       StringPiece directive, GoogleString* msg, MessageHandler* handler);
 
-  // These are called via RewriteOptions::ParseAndSetOptionFromName[123]
-  virtual OptionSettingResult ParseAndSetOptionFromEnum1(
-      OptionEnum name, StringPiece arg,
+  virtual OptionSettingResult ParseAndSetOptionFromName1(
+      StringPiece name, StringPiece arg,
       GoogleString* msg, MessageHandler* handler);
 
   // We may want to override 2- and 3-argument versions as well in the future,
@@ -102,21 +103,23 @@ class NgxRewriteOptions : public SystemRewriteOptions {
   static Properties* ngx_properties_;
   static void AddProperties();
   void Init();
-  void InitializeSignaturesAndDefaults();
 
   // Add an option to ngx_properties_
-  template<class RewriteOptionsSubclass, class OptionClass>
+  template<class OptionClass>
   static void add_ngx_option(typename OptionClass::ValueType default_value,
-                             OptionClass RewriteOptionsSubclass::*offset,
+                             OptionClass NgxRewriteOptions::*offset,
                              const char* id,
-                             OptionEnum option_enum) {
-    AddProperty(default_value, offset, id, option_enum, ngx_properties_);
+                             StringPiece option_name) {
+    AddProperty(default_value, offset, id, option_name, ngx_properties_);
   }
 
   // Helper for ParseAndSetOptions.  Returns whether the two directives equal,
   // ignoring case.
   bool IsDirective(StringPiece config_directive, StringPiece compare_directive);
 
+  // Returns a given option's scope.
+  RewriteOptions::OptionScope GetOptionScope(StringPiece option_name);
+
   // TODO(jefftk): support fetch proxy in server and location blocks.
 
   DISALLOW_COPY_AND_ASSIGN(NgxRewriteOptions);

src/ngx_server_context.cc

@@ -18,123 +18,63 @@
 
 #include "ngx_server_context.h"
 
-#include "ngx_request_context.h"
-#include "ngx_rewrite_options.h"
+extern "C" {
+  #include <ngx_http.h>
+}
+
+#include "ngx_pagespeed.h"
+#include "ngx_message_handler.h"
 #include "ngx_rewrite_driver_factory.h"
-#include "net/instaweb/apache/add_headers_fetcher.h"
-#include "net/instaweb/apache/loopback_route_fetcher.h"
+#include "ngx_rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
-#include "net/instaweb/rewriter/public/rewrite_stats.h"
-#include "net/instaweb/system/public/system_caches.h"
-#include "net/instaweb/util/public/shared_mem_statistics.h"
-#include "net/instaweb/util/public/split_statistics.h"
-#include "net/instaweb/util/public/statistics.h"
+#include "net/instaweb/system/public/add_headers_fetcher.h"
+#include "net/instaweb/system/public/loopback_route_fetcher.h"
+#include "net/instaweb/system/public/system_request_context.h"
 
 namespace net_instaweb {
 
-const char kCacheFlushCount[] = "cache_flush_count";
-const char kCacheFlushTimestampMs[] = "cache_flush_timestamp_ms";
-
-// Statistics histogram names.
-const char kHtmlRewriteTimeUsHistogram[] = "Html Time us Histogram";
-
-
-NgxServerContext::NgxServerContext(NgxRewriteDriverFactory* factory)
-    : SystemServerContext(factory),
-      ngx_factory_(factory),
-      initialized_(false) {
+NgxServerContext::NgxServerContext(
+    NgxRewriteDriverFactory* factory, StringPiece hostname, int port)
+    : SystemServerContext(factory, hostname, port) {
 }
 
-NgxServerContext::~NgxServerContext() {
-}
+NgxServerContext::~NgxServerContext() { }
 
 NgxRewriteOptions* NgxServerContext::config() {
   return NgxRewriteOptions::DynamicCast(global_options());
 }
 
-void NgxServerContext::ChildInit() {
-  DCHECK(!initialized_);
-  if (!initialized_) {
-    initialized_ = true;
-    set_lock_manager(ngx_factory_->caches()->GetLockManager(config()));
-
-    if (split_statistics_.get() != NULL) {
-      // Readjust the SHM stuff for the new process
-      local_statistics_->Init(false, message_handler());
-
-      // Create local stats for the ServerContext, and fill in its
-      // statistics() and rewrite_stats() using them; if we didn't do this here
-      // they would get set to the factory's by the InitServerContext call
-      // below.
-      set_statistics(split_statistics_.get());
-      local_rewrite_stats_.reset(new RewriteStats(
-          split_statistics_.get(), ngx_factory_->thread_system(),
-          ngx_factory_->timer()));
-      set_rewrite_stats(local_rewrite_stats_.get());
+SystemRequestContext* NgxServerContext::NewRequestContext(
+    ngx_http_request_t* r) {
+  // Based on ngx_http_variable_server_port.
+  bool port_set = false;
+  int local_port;
+#if (NGX_HAVE_INET6)
+  if (r->connection->local_sockaddr->sa_family == AF_INET6) {
+    local_port = ntohs(reinterpret_cast<struct sockaddr_in6*>(
+        r->connection->local_sockaddr)->sin6_port);
+    port_set = true;
+  }
+#endif
+  if (!port_set) {
+    local_port = ntohs(reinterpret_cast<struct sockaddr_in*>(
+        r->connection->local_sockaddr)->sin_port);
   }
 
-    ngx_factory_->InitServerContext(this);
-    // TODO(oschaaf): in mod_pagespeed, the ServerContext owns
-    // the fetchers, and sets up the UrlAsyncFetcherStats here
-  }
+  ngx_str_t local_ip;
+  u_char addr[NGX_SOCKADDR_STRLEN];
+  local_ip.len = NGX_SOCKADDR_STRLEN;
+  local_ip.data = addr;
+  ngx_int_t rc = ngx_connection_local_sockaddr(r->connection, &local_ip, 0);
+  if (rc != NGX_OK) {
+    local_ip.len = 0;
   }
 
-void NgxServerContext::CreateLocalStatistics(
-    Statistics* global_statistics) {
-  local_statistics_ =
-      ngx_factory_->AllocateAndInitSharedMemStatistics(
-          hostname_identifier(),
-          config()->statistics_logging_enabled(),
-          config()->statistics_logging_interval_ms(),
-          config()->statistics_logging_file());
-  split_statistics_.reset(new SplitStatistics(
-      ngx_factory_->thread_system(), local_statistics_, global_statistics));
-  // local_statistics_ was ::InitStat'd by AllocateAndInitSharedMemStatistics,
-  // but we need to take care of split_statistics_.
-  NgxRewriteDriverFactory::InitStats(split_statistics_.get());
-}
-
-void NgxServerContext::InitStats(Statistics* statistics) {
-  // TODO(oschaaf): we need to port the cache flush mechanism
-  statistics->AddVariable(kCacheFlushCount);
-  statistics->AddVariable(kCacheFlushTimestampMs);
-  Histogram* html_rewrite_time_us_histogram =
-      statistics->AddHistogram(kHtmlRewriteTimeUsHistogram);
-  // We set the boundary at 2 seconds which is about 2 orders of magnitude
-  // worse than anything we have reasonably seen, to make sure we don't
-  // cut off actual samples.
-  html_rewrite_time_us_histogram->SetMaxValue(2 * Timer::kSecondUs);
-  // TODO(oschaaf): Once the ServerContext owns the fetchers,
-  // initialise UrlAsyncFetcherStats here
-}
-
-void NgxServerContext::ApplySessionFetchers(
-    const RequestContextPtr& request, RewriteDriver* driver) {
-  const NgxRewriteOptions* conf = NgxRewriteOptions::DynamicCast(
-      driver->options());
-  CHECK(conf != NULL);
-  NgxRequestContext* ngx_request = NgxRequestContext::DynamicCast(
-      request.get());
-  if (ngx_request == NULL) {
-    return;  // decoding_driver has a null RequestContext.
-  }
-
-  // Note that these fetchers are applied in the opposite order of how they are
-  // added
-  // TODO(oschaaf): in mod_pagespeed, LoopbackRouteFetcher is not added when
-  // one of these is set:  disable_loopback_routing, slurping_enabled, or
-  // test_proxy.
-
-  // Note the port here is our port, not from the request, since
-  // LoopbackRouteFetcher may decide we should be talking to ourselves.
-  driver->SetSessionFetcher(new LoopbackRouteFetcher(
-      driver->options(), ngx_request->local_ip(),
-      ngx_request->local_port(), driver->async_fetcher()));
-
-  if (driver->options()->num_custom_fetch_headers() > 0) {
-    driver->SetSessionFetcher(new AddHeadersFetcher(driver->options(),
-                                                    driver->async_fetcher()));
-  }
+  return new SystemRequestContext(thread_system()->NewMutex(),
+                                  timer(),
+                                  ps_determine_host(r),
+                                  local_port,
+                                  str_to_string_piece(local_ip));
 }
 
 }  // namespace net_instaweb

src/ngx_server_context.h

@@ -21,19 +21,23 @@
 #ifndef NGX_SERVER_CONTEXT_H_
 #define NGX_SERVER_CONTEXT_H_
 
+#include "ngx_message_handler.h"
 #include "net/instaweb/system/public/system_server_context.h"
 
+extern "C" {
+#include <ngx_http.h>
+}
+
 namespace net_instaweb {
 
 class NgxRewriteDriverFactory;
 class NgxRewriteOptions;
-class RewriteStats;
-class SharedMemStatistics;
-class Statistics;
+class SystemRequestContext;
 
 class NgxServerContext : public SystemServerContext {
  public:
-  explicit NgxServerContext(NgxRewriteDriverFactory* factory);
+  NgxServerContext(
+      NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
   virtual ~NgxServerContext();
 
   // We expect to use ProxyFetch with HTML.
@@ -43,34 +47,16 @@ class NgxServerContext : public SystemServerContext {
   // nginx-specific behavior, call global_options() instead which doesn't
   // downcast.
   NgxRewriteOptions* config();
-  // Should be called after the child process is forked.
-  void ChildInit();
-  // Initialize this ServerContext to have its own statistics domain.
-  // Must be called after global_statistics has been created and had
-  // ::Initialize called on it.
-  void CreateLocalStatistics(Statistics* global_statistics);
-  static void InitStats(Statistics* statistics);
-  virtual void ApplySessionFetchers(const RequestContextPtr& req,
-                                    RewriteDriver* driver);
-  bool initialized() const { return initialized_; }
-  GoogleString hostname_identifier() { return hostname_identifier_; }
-  void set_hostname_identifier(GoogleString x) { hostname_identifier_ = x; }
+
   NgxRewriteDriverFactory* ngx_rewrite_driver_factory() { return ngx_factory_; }
+  SystemRequestContext* NewRequestContext(ngx_http_request_t* r);
+
+  NgxMessageHandler* ngx_message_handler() {
+    return dynamic_cast<NgxMessageHandler*>(message_handler());
+  }
 
  private:
   NgxRewriteDriverFactory* ngx_factory_;
-  // hostname_identifier_ is used to distinguish the name of shared memory
-  // segments associated with this ServerContext
-  GoogleString hostname_identifier_;
-  bool initialized_;
-
-  // Non-NULL if we have per-vhost stats.
-  scoped_ptr<Statistics> split_statistics_;
-
-  // May be NULL. Owned by *split_statistics_.
-  SharedMemStatistics* local_statistics_;
-  // These are non-NULL if we have per-vhost stats.
-  scoped_ptr<RewriteStats> local_rewrite_stats_;
 
   DISALLOW_COPY_AND_ASSIGN(NgxServerContext);
 };

src/ngx_thread_system.h

@@ -1,56 +0,0 @@
-/*
- * Copyright 2013 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: jefftk@google.com (Jeff Kaufman)
-//
-// See ApacheThreadSystem.
-//
-//
-
-#ifndef NGX_THREAD_SYSTEM_H_
-#define NGX_THREAD_SYSTEM_H_
-
-#include "base/logging.h"
-#include "net/instaweb/util/public/basictypes.h"
-#include "net/instaweb/util/public/pthread_thread_system.h"
-
-namespace net_instaweb {
-
-class Timer;
-
-class NgxThreadSystem : public PthreadThreadSystem {
- public:
-  NgxThreadSystem();
-  virtual ~NgxThreadSystem();
-
-  // In nginx we may only start threads after forking a worker process.  In
-  // order to enforce this, we call PermitThreadStarting() in the worker process
-  // right after forking, and CHECK-fail if something tries to start a thread
-  // before then.
-  void PermitThreadStarting();
-
- protected:
-  virtual void BeforeThreadRunHook();
-
- private:
-  bool may_start_threads_;
-
-  DISALLOW_COPY_AND_ASSIGN(NgxThreadSystem);
-};
-
-}  // namespace net_instaweb
-
-#endif  // NGX_THREAD_SYSTEM_H_

src/ngx_url_async_fetcher.cc

@@ -66,10 +66,10 @@ namespace net_instaweb {
       mutex_(NULL) {
     resolver_timeout_ = resolver_timeout;
     fetch_timeout_ = fetch_timeout;
-    ngx_memzero(&url_, sizeof(url_));
+    ngx_memzero(&proxy_, sizeof(proxy_));
     if (proxy != NULL && *proxy != '\0') {
-      url_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
-      url_.url.len = ngx_strlen(proxy);
+      proxy_.url.data = reinterpret_cast<u_char*>(const_cast<char*>(proxy));
+      proxy_.url.len = ngx_strlen(proxy);
     }
     mutex_ = thread_system_->NewMutex();
     log_ = log;
@@ -106,6 +106,36 @@ namespace net_instaweb {
     }
   }
 
+
+  bool NgxUrlAsyncFetcher::ParseUrl(ngx_url_t* url, ngx_pool_t* pool) {
+    size_t scheme_offset;
+    u_short port;
+    if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                     const_cast<char*>("http://")), 7) == 0) {
+      scheme_offset = 7;
+      port = 80;
+    } else if (ngx_strncasecmp(url->url.data, reinterpret_cast<u_char*>(
+                                    const_cast<char*>("https://")), 8) == 0) {
+      scheme_offset = 8;
+      port = 443;
+    } else {
+      scheme_offset = 0;
+      port = 80;
+    }
+
+    url->url.data += scheme_offset;
+    url->url.len -= scheme_offset;
+    url->default_port = port;
+    // See: http://lxr.evanmiller.org/http/source/core/ngx_inet.c#L875
+    url->no_resolve = 0;
+    url->uri_part = 1;
+
+    if (ngx_parse_url(pool, url) == NGX_OK) {
+      return true;
+    }
+    return false;
+  }
+
   // If there are still active requests, cancel them.
   void NgxUrlAsyncFetcher::CancelActiveFetches() {
     // TODO(oschaaf): this seems tricky, this may end up calling
@@ -167,15 +197,15 @@ namespace net_instaweb {
     command_connection_->read->handler = CommandHandler;
     ngx_add_event(command_connection_->read, NGX_READ_EVENT, 0);
 
-    if (url_.url.len == 0) {
+    if (proxy_.url.len == 0) {
       return true;
     }
 
     // TODO(oschaaf): shouldn't we do this earlier? Do we need to clean
     // up when parsing the url failed?
-    if (ngx_parse_url(pool_, &url_) != NGX_OK) {
+    if (!ParseUrl(&proxy_, pool_)) {
       ngx_log_error(NGX_LOG_ERR, log_, 0,
-          "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &url_.url);
+          "NgxUrlAsyncFetcher::Init parse proxy[%V] failed", &proxy_.url);
       return false;
     }
     return true;
@@ -191,7 +221,7 @@ namespace net_instaweb {
   void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                  MessageHandler* message_handler,
                                  AsyncFetch* async_fetch) {
-    async_fetch = EnableInflation(async_fetch, NULL);
+    async_fetch = EnableInflation(async_fetch);
     NgxFetch* fetch = new NgxFetch(url, async_fetch,
           message_handler, fetch_timeout_, log_);
     ScopedMutex lock(mutex_);

src/ngx_url_async_fetcher.h

@@ -33,11 +33,11 @@ extern "C" {
 }
 
 #include <vector>
+#include "net/instaweb/http/public/url_async_fetcher.h"
 #include "net/instaweb/util/public/basictypes.h"
 #include "net/instaweb/util/public/pool.h"
 #include "net/instaweb/util/public/string.h"
 #include "net/instaweb/util/public/thread_system.h"
-#include "net/instaweb/http/public/url_pollable_async_fetcher.h"
 
 
 namespace net_instaweb {
@@ -115,13 +115,14 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
 
  private:
   static void TimeoutHandler(ngx_event_t* tev);
+  static bool ParseUrl(ngx_url_t* url, ngx_pool_t* pool);
   friend class NgxFetch;
 
   NgxFetchPool active_fetches_;
   // Add the pending task to this list
   NgxFetchPool pending_fetches_;
   NgxFetchPool completed_fetches_;
-  ngx_url_t url_;
+  ngx_url_t proxy_;
 
   int fetchers_count_;
   bool shutdown_;

src/pthread_shared_mem.cc

@@ -1,257 +0,0 @@
-// Copyright 2011 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: morlovich@google.com (Maksim Orlovich)
-
-#include "pthread_shared_mem.h"
-
-#include <fcntl.h>
-#include <pthread.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <cerrno>
-#include <cstddef>
-#include <map>
-#include <utility>
-#include "net/instaweb/util/public/abstract_shared_mem.h"
-#include "net/instaweb/util/public/abstract_mutex.h"
-#include "net/instaweb/util/public/basictypes.h"
-#include "net/instaweb/util/public/message_handler.h"
-#include "net/instaweb/util/public/string.h"
-
-namespace net_instaweb {
-
-namespace ngx {
-
-namespace {
-
-// This implementation relies on readonly copies of old memory and shared R/W
-// mappings being kept across a fork. It simply stashes addresses of
-// shared mmap segments into a map where kid processes can pick them up.
-
-// close() a fd logging failure and dealing with EINTR.
-void CheckedClose(int fd, MessageHandler* message_handler) {
-  while (close(fd) != 0) {
-    if (errno != EINTR) {
-      message_handler->Message(kWarning, "Problem closing SHM segment fd:%d",
-                               errno);
-      return;
-    }
-  }
-}
-
-// Unlike PthreadMutex this doesn't own the lock, but rather refers to an
-// external one.
-class PthreadSharedMemMutex : public AbstractMutex {
- public:
-  explicit PthreadSharedMemMutex(pthread_mutex_t* external_mutex)
-      : external_mutex_(external_mutex) {}
-
-  virtual bool TryLock() {
-    return (pthread_mutex_trylock(external_mutex_) == 0);
-  }
-
-  virtual void Lock() {
-    pthread_mutex_lock(external_mutex_);
-  }
-
-  virtual void Unlock() {
-    pthread_mutex_unlock(external_mutex_);
-  }
-
- private:
-  pthread_mutex_t* external_mutex_;
-
-  DISALLOW_COPY_AND_ASSIGN(PthreadSharedMemMutex);
-};
-
-class PthreadSharedMemSegment : public AbstractSharedMemSegment {
- public:
-  // We will be representing memory mapped in the [base, base + size) range.
-  PthreadSharedMemSegment(char* base, size_t size, MessageHandler* handler)
-      : base_(base),
-        size_(size) {
-  }
-
-  virtual ~PthreadSharedMemSegment() {
-  }
-
-  virtual volatile char* Base() {
-    return base_;
-  }
-
-  virtual size_t SharedMutexSize() const {
-    return sizeof(pthread_mutex_t);
-  }
-
-  virtual bool InitializeSharedMutex(size_t offset, MessageHandler* handler) {
-    pthread_mutexattr_t attr;
-    if (pthread_mutexattr_init(&attr) != 0) {
-      handler->Message(kError, "pthread_mutexattr_init failed with errno:%d",
-                       errno);
-      return false;
-    }
-
-    if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) != 0) {
-      pthread_mutexattr_destroy(&attr);
-      handler->Message(
-          kError, "pthread_mutexattr_setpshared failed with errno:%d", errno);
-      return false;
-    }
-
-    if (pthread_mutex_init(MutexPtr(offset), &attr) != 0) {
-      pthread_mutexattr_destroy(&attr);
-      handler->Message(kError, "pthread_mutex_init failed with errno:%d",
-                       errno);
-      return false;
-    }
-
-    pthread_mutexattr_destroy(&attr);
-    return true;
-  }
-
-  virtual AbstractMutex* AttachToSharedMutex(size_t offset) {
-    return new PthreadSharedMemMutex(MutexPtr(offset));
-  }
-
- private:
-  pthread_mutex_t* MutexPtr(size_t offset) {
-    return reinterpret_cast<pthread_mutex_t*>(base_ + offset);
-  }
-
-  char* const base_;
-  const size_t size_;
-
-  DISALLOW_COPY_AND_ASSIGN(PthreadSharedMemSegment);
-};
-
-pthread_mutex_t segment_bases_lock = PTHREAD_MUTEX_INITIALIZER;
-
-}  // namespace
-
-size_t PthreadSharedMem::s_instance_count_ = 0;
-
-PthreadSharedMem::SegmentBaseMap* PthreadSharedMem::segment_bases_ = NULL;
-
-PthreadSharedMem::PthreadSharedMem() {
-  instance_number_ = ++s_instance_count_;
-}
-
-PthreadSharedMem::~PthreadSharedMem() {
-}
-
-size_t PthreadSharedMem::SharedMutexSize() const {
-  return sizeof(pthread_mutex_t);
-}
-
-AbstractSharedMemSegment* PthreadSharedMem::CreateSegment(
-    const GoogleString& name, size_t size, MessageHandler* handler) {
-  GoogleString prefixed_name = PrefixSegmentName(name);
-  // Create the memory
-  int fd = open("/dev/zero", O_RDWR);
-  if (fd == -1) {
-    handler->Message(kError, "Unable to create SHM segment %s, errno=%d.",
-                     prefixed_name.c_str(), errno);
-    return NULL;
-  }
-
-  // map it
-  char* base = reinterpret_cast<char*>(
-                   mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0));
-  CheckedClose(fd, handler);
-  if (base == MAP_FAILED) {
-    return NULL;
-  }
-
-  SegmentBaseMap* bases = AcquireSegmentBases();
-  (*bases)[prefixed_name] = base;
-  UnlockSegmentBases();
-  return new PthreadSharedMemSegment(base, size, handler);
-}
-
-AbstractSharedMemSegment* PthreadSharedMem::AttachToSegment(
-    const GoogleString& name, size_t size, MessageHandler* handler) {
-  GoogleString prefixed_name = PrefixSegmentName(name);
-  SegmentBaseMap* bases = AcquireSegmentBases();
-  SegmentBaseMap::const_iterator i = bases->find(prefixed_name);
-  if (i == bases->end()) {
-    handler->Message(kError, "Unable to find SHM segment %s to attach to.",
-                     prefixed_name.c_str());
-    UnlockSegmentBases();
-    return NULL;
-  }
-  char* base = i->second;
-  UnlockSegmentBases();
-  return new PthreadSharedMemSegment(base, size, handler);
-}
-
-void PthreadSharedMem::DestroySegment(const GoogleString& name,
-                                      MessageHandler* handler) {
-  GoogleString prefixed_name = PrefixSegmentName(name);
-
-  // Note that in the process state children will not see any mutations
-  // we make here, so it acts mostly for checking in that case.
-  SegmentBaseMap* bases = AcquireSegmentBases();
-  SegmentBaseMap::iterator i = bases->find(prefixed_name);
-  if (i != bases->end()) {
-    bases->erase(i);
-    if (bases->empty()) {
-      delete segment_bases_;
-      segment_bases_ = NULL;
-    }
-  } else {
-    handler->Message(kError, "Attempt to destroy unknown SHM segment %s.",
-                     prefixed_name.c_str());
-  }
-  UnlockSegmentBases();
-}
-
-PthreadSharedMem::SegmentBaseMap* PthreadSharedMem::AcquireSegmentBases() {
-  PthreadSharedMemMutex lock(&segment_bases_lock);
-  lock.Lock();
-
-  if (segment_bases_ == NULL) {
-    segment_bases_ = new SegmentBaseMap();
-  }
-
-  return segment_bases_;
-}
-
-void PthreadSharedMem::UnlockSegmentBases() {
-  PthreadSharedMemMutex lock(&segment_bases_lock);
-  lock.Unlock();
-}
-
-GoogleString PthreadSharedMem::PrefixSegmentName(const GoogleString& name) {
-  GoogleString res;
-  StrAppend(&res, "[", IntegerToString(instance_number_), "]", name);
-  return res;
-}
-
-
-void PthreadSharedMem::Terminate() {
-  // Clean up the local memory associated with the maps to shared memory
-  // storage.
-  PthreadSharedMemMutex lock(&segment_bases_lock);
-  lock.Lock();
-  if (segment_bases_ != NULL) {
-    delete segment_bases_;
-    segment_bases_ = NULL;
-  }
-  lock.Unlock();
-}
-
-}  // namespace ngx
-
-}  // namespace net_instaweb

src/pthread_shared_mem.h

@@ -1,88 +0,0 @@
-// Copyright 2011 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: morlovich@google.com (Maksim Orlovich)
-
-#ifndef NET_INSTAWEB_UTIL_PUBLIC_PTHREAD_SHARED_MEM_H_
-#define NET_INSTAWEB_UTIL_PUBLIC_PTHREAD_SHARED_MEM_H_
-
-#include <cstddef>
-#include <map>
-
-#include "net/instaweb/util/public/abstract_shared_mem.h"
-#include "net/instaweb/util/public/basictypes.h"
-#include "net/instaweb/util/public/string.h"
-
-namespace net_instaweb {
-class MessageHandler;
-
-namespace ngx {
-
-// POSIX shared memory support, using mmap/pthread_mutexattr_setpshared
-// Supports both processes and threads, but processes that want to access it
-// must be results of just fork (without exec), and all the CreateSegment
-// calls must occur before the fork.
-//
-// This implementation is also not capable of deallocating segments except
-// at exit, so it should not be used when the set of segments may be dynamic.
-class PthreadSharedMem : public AbstractSharedMem {
- public:
-  PthreadSharedMem();
-  virtual ~PthreadSharedMem();
-
-  virtual size_t SharedMutexSize() const;
-
-  virtual AbstractSharedMemSegment* CreateSegment(
-      const GoogleString& name, size_t size, MessageHandler* handler);
-
-  virtual AbstractSharedMemSegment* AttachToSegment(
-      const GoogleString& name, size_t size, MessageHandler* handler);
-
-  virtual void DestroySegment(const GoogleString& name,
-                              MessageHandler* handler);
-
-  // Frees all lazy-initialized memory used to track shared-memory segments.
-  static void Terminate();
-
- private:
-  typedef std::map<GoogleString, char*> SegmentBaseMap;
-
-  // Accessor for below. Note that the segment_bases_lock will be held at exit.
-  static SegmentBaseMap* AcquireSegmentBases();
-
-  static void UnlockSegmentBases();
-
-  // Prefixes the passed in segment name with the current instance number.
-  GoogleString PrefixSegmentName(const GoogleString& name);
-
-  // The root process stores segment locations here. Child processes will
-  // inherit a readonly copy of this map after the fork. Note that this is
-  // initialized in a thread-unsafe manner, given the above assumptions.
-  static SegmentBaseMap* segment_bases_;
-  // Holds the number of times a PthreadSharedMem has been created.
-  static size_t s_instance_count_;
-  // Used to prefix segment names, so that when two runtimes are active at the
-  // same moment they will not have overlapping segment names. This occurs in
-  // ngx_pagespeed during a configuration reload, where first a new factory is
-  // created, before destroying the old one.
-  size_t instance_number_;
-
-  DISALLOW_COPY_AND_ASSIGN(PthreadSharedMem);
-};
-
-}  // namespace ngx
-
-}  // namespace net_instaweb
-
-#endif  // NET_INSTAWEB_UTIL_PUBLIC_PTHREAD_SHARED_MEM_H_

test/pagespeed_test.conf.template

@@ -5,9 +5,9 @@
 worker_processes  1;
 
 daemon @@DAEMON@@;
-master_process @@MASTER_PROCESS@@;
+master_process on;
 
-error_log "@@TEST_TMP@@/error.log" debug;
+error_log "@@ERROR_LOG@@" debug;
 pid "@@TEST_TMP@@/nginx.pid";
 
 events {
@@ -16,12 +16,34 @@ events {
 
 
 http {
-  access_log "@@TEST_TMP@@/access.log";
+  server_names_hash_bucket_size 128;
+
+  log_format cache '$time_local '
+                   '$upstream_cache_status '
+                   '$http_host $request ($status) '
+                   '"$http_user_agent"';
+  access_log "@@ACCESS_LOG@@" cache;
+
+  proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
+  proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
   root "@@SERVER_ROOT@@";
-  pagespeed UsePerVHostStatistics on;
 
+  # Block 5a: Decide on Cache-Control header value to use for outgoing
+  # response.
+  # Map new_cache_control_header_val to "no-cache, max-age=0" if the
+  # content is html and use the original Cache-Control header value
+  # in all other cases.
+  map $upstream_http_content_type $new_cache_control_header_val {
+    default $upstream_http_cache_control;
+    "~*text/html" "no-cache, max-age=0";
+  }
+
+  pagespeed UsePerVHostStatistics on;
+  pagespeed InPlaceResourceOptimization on;
   pagespeed CreateSharedMemoryMetadataCache "@@SHM_CACHE@@" 8192;
+  pagespeed PreserveUrlRelativity on;
+  pagespeed BlockingRewriteKey psatest;
 
   # CriticalImagesBeaconEnabled is now on by default, but we disable in testing.
   # With this option enabled, the inline image system test will currently fail.
@@ -30,6 +52,15 @@ http {
   # critical images to be inlined, so we just disable the option here.
   pagespeed CriticalImagesBeaconEnabled false;
 
+  # By default, resources will not be used for inlining without explicit
+  # authorization. Supported values are off or a comma-separated list of strings
+  # from {Script,Stylesheet}.
+  pagespeed InlineResourcesWithoutExplicitAuthorization off;
+
+  pagespeed Statistics on;
+  pagespeed StatisticsLogging on;
+  pagespeed LogDir "@@TEST_TMP@@/logdir";
+
   server {
     # Sets up a logical home-page server on
     # max-cacheable-content-length.example.com.  This server is only used to
@@ -39,7 +70,6 @@ http {
     server_name max-cacheable-content-length.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed BlockingRewriteKey psatest;
 
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_javascript;
@@ -49,6 +79,162 @@ http {
   pagespeed UseNativeFetcher "@@NATIVE_FETCHER@@";
   @@RESOLVER@@
 
+  server {
+    # Block 1:  Basic port, server_name definitions.
+    # This server represents the external caching layer server which
+    # receives user requests and proxies them to the upstream server
+    # running on the PRIMARY_PORT when the response is not available in
+    # the cache. It also services purge requests from the upstream server.
+    listen @@SECONDARY_PORT@@;
+    server_name proxy_cache.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    # Disable PageSpeed on this server.
+    pagespeed off;
+
+    # Block 2: Define prefix for proxy_cache_key based on the UserAgent.
+
+    # Define placeholder PS-CapabilityList header values for large and small
+    # screens with no UA dependent optimizations. Note that these placeholder
+    # values should not contain any of ll, ii, dj, jw or ws, since these
+    # codes will end up representing optimizations to be supported for the
+    # request.
+    set $default_ps_capability_list_for_large_screens "LargeScreen.SkipUADependentOptimizations";
+    set $default_ps_capability_list_for_small_screens "TinyScreen.SkipUADependentOptimizations";
+
+    # As a fallback, the PS-CapabilityList header that is sent to the upstream
+    # PageSpeed server should be for a large screen device with no browser
+    # specific optimizations.
+    set $ps_capability_list $default_ps_capability_list_for_large_screens;
+
+    # Cache-fragment 1: Desktop User-Agents that support lazyload_images (ll),
+    # inline_images (ii) and defer_javascript (dj).
+    # Note: Wget is added for testing purposes only.
+    if ($http_user_agent ~* "Chrome/|Firefox/|MSIE |Safari|Wget") {
+      set $ps_capability_list "ll,ii,dj:";
+    }
+    # Cache-fragment 2: Desktop User-Agents that support lazyload_images (ll),
+    # inline_images (ii), defer_javascript (dj), webp (jw) and lossless_webp
+    # (ws).
+    if ($http_user_agent ~*
+        "Chrome/[2][3-9]+\.|Chrome/[[3-9][0-9]+\.|Chrome/[0-9]{3,}\.") {
+      set $ps_capability_list "ll,ii,dj,jw,ws:";
+    }
+    # Cache-fragment 3: This fragment contains (a) Desktop User-Agents that
+    # match fragments 1 or 2 but should not because they represent older
+    # versions of certain browsers or bots and (b) Tablet User-Agents that
+    # correspond to large screens. These will only get optimizations that work
+    # on all browsers and use image compression qualities applicable to large
+    # screens. Note that even Tablets that are capable of supporting inline or
+    # webp images, e.g. Android 4.1.2, will not get these advanced
+    # optimizations.
+    if ($http_user_agent ~* "Firefox/[1-2]\.|MSIE [5-8]\.|bot|Yahoo!|Ruby|RPT-HTTPClient|(Google \(\+https\:\/\/developers\.google\.com\/\+\/web\/snippet\/\))|Android|iPad|TouchPad|Silk-Accelerated|Kindle Fire") {
+      set $ps_capability_list $default_ps_capability_list_for_large_screens;
+    }
+    # Cache-fragment 4: Mobiles and small screen Tablets will use image compression
+    # qualities applicable to small screens, but all other optimizations will be
+    # those that work on all browsers.
+    if ($http_user_agent ~* "Mozilla.*Android.*Mobile*|iPhone|BlackBerry|Opera Mobi|Opera Mini|SymbianOS|UP.Browser|J-PHONE|Profile/MIDP|portalmmm|DoCoMo|Obigo|Galaxy Nexus|GT-I9300|GT-N7100|HTC One|Nexus [4|7|S]|Xoom|XT907") {
+      set $ps_capability_list $default_ps_capability_list_for_small_screens;
+    }
+
+    # Block 3a: Bypass the cache for .pagespeed. resource. PageSpeed has its own
+    # cache for these, and these could bloat up the caching layer.
+    if ($uri ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+") {
+      set $bypass_cache "1";
+    }
+
+    set_random $rand 0 100;
+    set $should_beacon_header_val "";
+    if ($rand ~* "^[0-4]$") {
+      set $should_beacon_header_val "random_rebeaconing_key";
+      set $bypass_cache 1;
+    }
+
+    # Block 3b: Only cache responses to clients that support gzip.  Most clients
+    # do, and the cache holds much more if it stores gzipped responses.
+    if ($http_accept_encoding !~* gzip) {
+      set $bypass_cache "1";
+    }
+
+    # Block 4: Location block for purge requests.
+    location ~ /purge(/.*) {
+      allow 127.0.0.1;
+      deny all;
+      proxy_cache_purge htmlcache $ps_capability_list$1$is_args$args;
+    }
+
+    # Block 6: Location block with proxy_cache directives.
+    location /mod_pagespeed_test/cachable_rewritten_html/ {
+      # 1: Upstream PageSpeed server is running at localhost:8050.
+      proxy_pass http://localhost:@@PRIMARY_PORT@@;
+      # 2: Use htmlcache as the zone for caching.
+      proxy_cache htmlcache;
+      # 3: Bypass requests that correspond to .pagespeed. resources
+      # or clients that do not support gzip etc.
+      proxy_cache_bypass $bypass_cache;
+      # 4: Use the redefined proxy_cache_key and make sure the /purge/
+      # block uses the same key.
+      proxy_cache_key $ps_capability_list$uri$is_args$args;
+      # 5: Forward Host header to upstream server.
+      proxy_set_header Host $host;
+      # 6: Set the PS-CapabilityList header for PageSpeed server to respect.
+      proxy_set_header PS-CapabilityList $ps_capability_list;
+      add_header PS-CapabilityList $ps_capability_list;
+      # 7: Add a header for identifying cache hits/misses/expires. This is
+      # for debugging purposes only and can be commented out in production.
+      add_header X-Cache $upstream_cache_status;
+
+      # Block 5b: Override Cache-Control headers as needed.
+      # Hide the upstream cache control header.
+      proxy_hide_header Cache-Control;
+      # Add the inferred Cache-Control header.
+      add_header Cache-Control $new_cache_control_header_val;
+
+      proxy_set_header PS-ShouldBeacon $should_beacon_header_val;
+      proxy_hide_header PS-ShouldBeacon;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-server.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    set $inline_javascript "No";
+
+    if ($http_x_custom_header_inline_js) {
+      # TODO(jefftk): Turn on NGX_HTTP_SIF_CONF and figure out how to get
+      # pagespeed directives inside of a server location block to be respected,
+      # then uncomment the following line and duplicate the if-in-location test
+      # for if-in-server.
+      #pagespeed EnableFilters inline_javascript;
+      set $inline_javascript "Yes";
+    }
+
+    add_header "X-Inline-Javascript" $inline_javascript;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name if-in-location.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+
+    location / {
+      set $inline_javascript "No";
+      pagespeed RewriteLevel PassThrough;
+
+      if ($http_x_custom_header_inline_js) {
+        pagespeed EnableFilters inline_javascript;
+        set $inline_javascript "Yes";
+      }
+
+      add_header "X-Inline-Javascript" $inline_javascript;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name mpd.example.com;
@@ -63,6 +249,7 @@ http {
     listen @@SECONDARY_PORT@@;
     server_name experiment.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization off;
 
     pagespeed RunExperiment on;
     pagespeed AnalyticsID "123-45-6734";
@@ -70,6 +257,7 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
   server {
     listen @@SECONDARY_PORT@@;
@@ -81,6 +269,7 @@ http {
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
+    pagespeed ExperimentSpec "id=3;default;percent=0";
   }
 
   server {
@@ -122,6 +311,87 @@ http {
     pagespeed CriticalImagesBeaconEnabled true;
   }
 
+  server {
+    # Setup a vhost with the critical image beacon enabled to make sure that
+    # downstream caches and rebeaconing interact correctly.
+    listen @@SECONDARY_PORT@@;
+    server_name downstreamcacherebeacon.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed CriticalImagesBeaconEnabled true;
+    # Enable the downstream caching feature and specify a rebeaconing key.
+    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+    pagespeed DownstreamCacheRebeaconingKey random_rebeaconing_key;
+    location ~ .*[.]html {
+      add_header Cache-Control "private, max-age=3000";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name renderedimagebeacon.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters resize_rendered_image_dimensions;
+    pagespeed CriticalImagesBeaconEnabled true;
+  }
+
+
+  # Build a configuration hierarchy where at the root we have turned on
+  # OptimizeForBandwidth, and in various subdirectories we override settings
+  # to make them more aggressive.
+  #
+  # In Apache we can do this all with Directory blocks, but to get the same
+  # inheretence in Nginx we need to have location blocks inside a server block.
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name optimizeforbandwidth.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel OptimizeForBandwidth;
+    pagespeed DisableFilters add_instrumentation;
+
+    location /mod_pagespeed_test/optimize_for_bandwidth/inline_css {
+      pagespeed EnableFilters inline_css;
+    }
+    location /mod_pagespeed_test/optimize_for_bandwidth/css_urls {
+      pagespeed CssPreserveURLs off;
+    }
+    location /mod_pagespeed_test/optimize_for_bandwidth/image_urls {
+      pagespeed ImagePreserveURLs off;
+    }
+    location /mod_pagespeed_test/optimize_for_bandwidth/core_filters {
+      pagespeed RewriteLevel CoreFilters;
+    }
+  }
+
+  server {
+    # For testing with a custom origin header.  In this VirtualHost,
+    # /mod_pagespeed_test is included in our DocumentRoot and thus does
+    # not need to be in any resource URL paths.  This helps us verify that
+    # we are looping back to the corect VirtualHost -- if we hit the wrong
+    # one it will not work.  Also we don't have a VirtualHost for
+    # sharedcdn.example.com, so the default Host header used for
+    # origin-mapping won't work either.  Instead, we want origin-fetches
+    # to go back to this VirtualHost so we rely on the new third optional
+    # argument to MapOriginDomain.
+    listen @@SECONDARY_PORT@@;
+    server_name customhostheader.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@_test";
+    root "@@SERVER_ROOT@@/mod_pagespeed_test";
+
+    pagespeed on;
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+    # Don't use localhost, as ngx_pagespeed's native fetcher cannot resolve it
+    pagespeed MapOriginDomain 127.0.0.1:@@SECONDARY_PORT@@/customhostheader
+              sharedcdn.example.com/test customhostheader.example.com;
+    pagespeed JpegRecompressionQuality 50;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+
   server {
     # Sets up a virtual host where we can specify forbidden filters without
     # affecting any other hosts.
@@ -129,7 +399,6 @@ http {
     server_name forbidden.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed BlockingRewriteKey psatest;
 
     # Start with all core filters enabled ...
     pagespeed RewriteLevel CoreFilters;
@@ -140,6 +409,31 @@ http {
     pagespeed DisableFilters inline_css;
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name unauthorizedresources.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed InlineResourcesWithoutExplicitAuthorization Script,Stylesheet;
+    pagespeed CssInlineMaxBytes 1000000;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name client-domain-rewrite.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    # Don't actually try to rewrite any resources; the ones in
+    # rewrite_domains.html don't actually exist.
+    pagespeed RewriteLevel PassThrough;
+
+    pagespeed MapRewriteDomain http://client-domain-rewrite.example.com
+              http://src.example.com;
+    pagespeed ClientDomainRewrite true;
+    pagespeed EnableFilters rewrite_domains;
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name url-attribute.example.com;
@@ -159,6 +453,9 @@ http {
     pagespeed UrlValuedAttribute custom a Image;
     pagespeed UrlValuedAttribute custom b otherResource;
     pagespeed UrlValuedAttribute custom c hyperlink;
+    pagespeed UrlValuedAttribute img alt-src Image;
+    pagespeed UrlValuedAttribute video alt-a Image;
+    pagespeed UrlValuedAttribute video alt-b Image;
   }
 
   server {
@@ -236,7 +533,7 @@ http {
   # Note that we test with two distinct caches.
   server {
     listen @@SECONDARY_PORT@@;
-    server_name embed-config-html.example.com;
+    server_name embed-config-html.example.org;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
     root "@@SERVER_ROOT@@/mod_pagespeed_test";
@@ -246,6 +543,9 @@ http {
     pagespeed DisableFilters inline_css,extend_cache,inline_javascript;
     pagespeed Domain embed-config-resources.example.com;
 
+    # Share a cache keyspace with embed-config-resources.example.com.
+    pagespeed CacheFragment "embed-config";
+
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
   }
@@ -261,12 +561,26 @@ http {
 
     # Note that we do not set the jpeg quality here, but take
     # it from image URL query parameters that we synthesize in
-    # from embed-config-html.example.com.
+    # from embed-config-html.example.org.
+
+    # Share a cache keyspace with embed-config-html.example.org.
+    pagespeed CacheFragment "embed-config";
 
     pagespeed LoadFromFile "http://embed-config-resources.example.com/"
       "@@SERVER_ROOT@@/mod_pagespeed_example/";
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-for-browser.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name respectvary.example.com;
@@ -318,6 +632,7 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical origin for CDNs to fetch content from, on
@@ -330,6 +645,7 @@ http {
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
+    pagespeed CacheFragment "example";
   }
   server {
     # Sets up a logical cdn, which is where we tell browsers to fetch resources
@@ -385,11 +701,164 @@ http {
     server_name blocking.example.com;
     pagespeed FileCachePath "@@SECONDARY_CACHE@@";
 
-    pagespeed BlockingRewriteKey psatest;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_images;
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-html.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+    error_log "@@TEST_TMP@@/keepalive-html.example.com.error.log" warn;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-resource.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-resource.example.com.error.log" warn;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-beacon-get.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-beacon-get.example.com.error.log" warn;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-beacon-post.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-beacon-post.example.com.error.log" warn;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name keepalive-static.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    error_log "@@TEST_TMP@@/keepalive-static.example.com.error.log" warn;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name response-header-filters.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed on;
+    add_header PageSpeedFilters add_instrumentation;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name response-header-disable.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+    pagespeed EnableFilters add_instrumentation;
+    pagespeed on;
+    add_header PageSpeed off;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
+
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+
+    location /mod_pagespeed_test/ipro/nocache/test_image_dont_reuse.png {
+      add_header Cache-Control no-cache;
+    }
+
+    location /mod_pagespeed_test/ipro/no-cache-control-header {
+      add_header Cache-Control "";
+      pagespeed ImplicitCacheTtlMs 333000;
+    }
+  }
+
+  # Test hosts to cover all possible cache configurations.  L1 will be filecache
+  # or memcache depending on the setting of MEMCACHED_TEST.  These four hosts
+  # are for the four settings for the L2 cache.
+
+  # 1. L2_d=LRU, L2_m=LRU
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name lrud-lrum.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_lrum";
+
+    pagespeed LRUCacheKbPerProcess 1024;
+    pagespeed LRUCacheByteLimit 2000;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+
+  # 2. L2_d=LRU, L2_m=SHM
+  pagespeed CreateSharedMemoryMetadataCache
+            "@@SECONDARY_CACHE@@_lrud_shmm" 8192;
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name lrud-shmm.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_lrud_shmm";
+
+    pagespeed LRUCacheKbPerProcess 1024;
+    pagespeed LRUCacheByteLimit 2000;
+    pagespeed EnableFilters rewrite_images;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+
+  # 3. L2_d=none, L2_m=SHM
+  pagespeed CreateSharedMemoryMetadataCache
+            "@@SECONDARY_CACHE@@_noned_shmm" 8192;
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name noned-shmm.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_shmm";
+
+    pagespeed EnableFilters rewrite_images;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+
+  # 4. L2_d=none, L2_m=none
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name noned-nonem.example.com;
+    pagespeed FileCachePath "@@SECONDARY_CACHE@@_noned_nonem";
+
+    pagespeed EnableFilters rewrite_images;
+    pagespeed CriticalImagesBeaconEnabled false;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name date.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
+  }
+
+  # Proxy + IPRO a gzip'd file for testing Issue 896.
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name ipro-proxy.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@_ipro_proxy";
+
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed EnableFilters rewrite_domains;
+
+    location / {
+      proxy_pass
+       "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/mod_deflate/";
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name compressed-css.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed InPlaceResourceOptimization on;
+  }
+
   server {
     listen       @@PRIMARY_PORT@@;
     server_name  localhost;
@@ -399,11 +868,40 @@ http {
       add_header "" "";
     }
 
+    # Backend for ipro-proxy.example.com
+    location /mod_pagespeed_test/ipro/mod_deflate/ {
+      pagespeed off;
+    }
+
+    location /mod_pagespeed_test/cachable_rewritten_html/ {
+      # This location has the html files that will be configured to be stored
+      # in the proxy_cache layer.
+      pagespeed DownstreamCachePurgeMethod "GET";
+      pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
+      pagespeed DownstreamCacheRebeaconingKey "random_rebeaconing_key";
+      # We use a very small deadline here to force the rewriting to not complete
+      # in the very first attempt.
+      pagespeed RewriteDeadlinePerFlushMs 1;
+      pagespeed RewriteLevel PassThrough;
+      pagespeed EnableFilters collapse_whitespace,extend_cache,recompress_images;
+      pagespeed CriticalImagesBeaconEnabled true;
+      add_header Cache-Control "public, max-age=100";
+    }
+
+    location /mod_pagespeed_test/disable_no_transform/index.html {
+      pagespeed DisableRewriteOnNoTransform off;
+    }
+
+    location /mod_pagespeed_test/disable_no_transform/disable_no_transform.css {
+      add_header 'Cache-Control' 'no-transform';
+    }
+
     # uncomment the following two lines if you're testing memcached
     #pagespeed MemcachedServers "localhost:11211";
     #pagespeed MemcachedThreads 1;
 
     pagespeed on;
+    pagespeed MessageBufferSize 200000;
 
     #pagespeed CacheFlushPollIntervalSec 1;
 
@@ -415,14 +913,6 @@ http {
     pagespeed Library 43 1o978_K0_LNE5_ystNklf
     http://www.modpagespeed.com/rewrite_javascript.js;
 
-    # If X-PSA-Blocking-Rewrite request header is present and its value matches
-    # the value of BlockingRewriteKey below, the response will be fully
-    # rewritten before being flushed to the client.
-    pagespeed BlockingRewriteKey psatest;
-
-    # Disable parsing if the size of the HTML exceeds 50kB.
-    pagespeed MaxHtmlParseBytes 50000;
-
     add_header X-Extra-Header 1;
 
     # Establish a proxy mapping where the current server proxies an image
@@ -444,6 +934,12 @@ http {
     # As opposed to mod_pagespeed, our default for aris is 'on'
     pagespeed AvoidRenamingIntrospectiveJavascript off;
 
+    location /mod_pagespeed_test/nostore {
+      add_header "Cache-Control" "max-age=12345";
+      add_header "Cache-Control" "public, no-store";
+      add_header "Cache-Control" "max-age=14";
+    }
+
     location /mod_pagespeed_test/forbid_all_disabled/disabled {
       # Prevent the enabling of these filters for files in this directory
       # -and- all subdirectories.
@@ -455,6 +951,10 @@ http {
       pagespeed DisableFilters collapse_whitespace;
     }
 
+    location /mod_pagespeed_test/max_html_parse_size {
+      pagespeed MaxHtmlParseBytes 5000;
+    }
+
     location ~ \.php$ {
       fastcgi_param SCRIPT_FILENAME $request_filename;
       fastcgi_param QUERY_STRING $query_string;
@@ -512,6 +1012,12 @@ http {
       add_header Cache-Control "private, max-age=3000";
     }
 
+    location /mod_pagespeed_test/retain_cache_control_with_downstream_caching/ {
+      pagespeed ModifyCachingHeaders on;
+      pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:8020/";
+      add_header Cache-Control "private, max-age=3000";
+    }
+
     location /mod_pagespeed_test/avoid_renaming_introspective_javascript__on/ {
       pagespeed AvoidRenamingIntrospectiveJavascript on;
     }
@@ -520,6 +1026,75 @@ http {
       pagespeed DisableFilters convert_jpeg_to_progressive;
     }
 
+    location /mod_pagespeed_test/ipro/test_image_dont_reuse.png {
+      expires 5m;
+    }
+
+    location /mod_pagespeed_test/ipro/instant/wait/ {
+      pagespeed InPlaceWaitForOptimized on;
+
+      # TODO(jefftk): When running with valgrind we need to raise the rewrite
+      # deadline or else we get "Deadline exceeded for rewrite of resource".
+      # I had thought InPlaceWaitForOptimized was supposed to disable the
+      # rewrite deadline, but it seems not to.
+      #
+      # InPlaceWaitForOptimized is definitely doing something, though: when I
+      # remove the InPlaceRewriteDeadlineMs line below and run without valgrind,
+      # purple.css is optimized via ipro on the first request but if I set
+      # InPlaceWaitForOptimized to off then it is not.
+      pagespeed InPlaceRewriteDeadlineMs 1000;
+    }
+
+    location /mod_pagespeed_test/ipro/instant/deadline/ {
+      pagespeed InPlaceRewriteDeadlineMs -1;
+    }
+
+    # Test to make sure that user-authenticated resources do not get cached and
+    # optimized.
+    location /mod_pagespeed_test/auth/ {
+       auth_basic "Restricted";
+       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+    }
+
+    location /mod_pagespeed_test/ipro/cookie/ {
+      # Add Vary:Cookie.  This should prevent us from optimizing the
+      # vary_cookie.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookies.
+      add_header Vary Cookie;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/ipro/cookie2/ {
+      # Add Vary:Cookie2.  This should prevent us from optimizing the
+      # vary_cookie2.css even though PagespeedRespectVary is off.
+      # test/nginx_system_test.sh does the fetches test with and w/o cookie2.
+      add_header Vary Cookie2;
+      pagespeed RespectVary off;
+      pagespeed InPlaceWaitForOptimized on;
+    }
+
+    location /mod_pagespeed_test/vary/ {
+      pagespeed RespectVary on;
+    }
+
+    location ~ /mod_pagespeed_test/vary/\.(js|css)$ {
+        add_header Vary User-Agent;
+    }
+
+    location /mod_pagespeed_test/vary/no_respect/ {
+      pagespeed DisableFilters add_instrumentation,inline_css;
+      pagespeed RespectVary off;
+    }
+
+    location /mod_pagespeed_test/experimental_js_minifier/ {
+      pagespeed UseExperimentalJsMinifier on;
+    }
+
+    pagespeed LoadFromFile
+        "http://localhost:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
+        "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
+
     pagespeed EnableFilters remove_comments;
 
     # Test LoadFromFile mapping by mapping one dir to another.
@@ -558,6 +1133,7 @@ http {
   # set up gzip
   gzip  on;
   gzip_vary on;
+  gzip_proxied any;
   # Turn on gzip for all content types that should benefit from it.
   gzip_types application/ecmascript;
   gzip_types application/javascript;

test/valgrind.sup

@@ -0,0 +1,165 @@
+# Copyright 2013 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Author: oschaaf@we-amp.com (Otto van der Schaaf)
+
+
+# The first few suppressions can be found in other modules
+# and easily found when searched for, and seem false positives.
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_signal_worker_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_pass_open_channel
+   fun:ngx_start_cache_manager_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_pass_open_channel
+   fun:ngx_start_cache_manager_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Leak
+   fun:malloc
+   fun:ngx_alloc
+   fun:ngx_event_process_init
+   fun:ngx_worker_process_init
+   fun:ngx_worker_process_cycle
+   fun:ngx_spawn_process
+   fun:ngx_start_worker_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+{
+   <nginx false positive>
+   Memcheck:Param
+   socketcall.sendmsg(msg.msg_iov[i])
+   fun:__sendmsg_nocancel
+   fun:ngx_write_channel
+   fun:ngx_pass_open_channel
+   fun:ngx_start_worker_processes
+   fun:ngx_master_process_cycle
+   fun:main
+}
+
+# similar to http://trac.nginx.org/nginx/ticket/369
+{
+   <nginx false positive>
+   Memcheck:Param
+   pwrite64(buf)
+   obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
+   fun:ngx_write_file
+   fun:ngx_write_chain_to_file
+   fun:ngx_write_chain_to_temp_file
+   fun:ngx_event_pipe_write_chain_to_temp_file
+   fun:ngx_event_pipe
+   fun:ngx_http_upstream_process_upstream
+   fun:ngx_http_upstream_process_header
+   fun:ngx_http_upstream_handler
+   fun:ngx_epoll_process_events
+   fun:ngx_process_events_and_timers
+   fun:ngx_worker_process_cycle
+}
+# Mentioned in https://github.com/pagespeed/ngx_pagespeed/issues/103
+# Assuming a false postives as the issue is closed.
+{
+   <nginx false positive>
+   Memcheck:Param
+   write(buf)
+   obj:/lib/x86_64-linux-gnu/libpthread-2.15.so
+   fun:ngx_log_error_core
+   fun:ngx_http_parse_complex_uri
+   fun:ngx_http_process_request_uri
+   fun:ngx_http_process_request_line
+   fun:ngx_http_wait_request_handler
+   fun:ngx_epoll_process_events
+   fun:ngx_process_events_and_timers
+   fun:ngx_worker_process_cycle
+   fun:ngx_spawn_process
+   fun:ngx_start_worker_processes
+   fun:ngx_master_process_cycle
+}
+
+# Extra suppresions for testing in release mode:
+
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Cond
+   fun:_ZN3re24Prog8OptimizeEv
+   ...
+}
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Value8
+   fun:_ZN3re24Prog8OptimizeEv
+   ...
+}
+
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Cond
+   fun:_ZN3re2L4AddQEPNS_9SparseSetEi
+   ...
+}
+
+{
+   <re2 uninitialised value in optimized code>
+   Memcheck:Value8
+   fun:_ZN3re2L4AddQEPNS_9SparseSetEi
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Value8
+   fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
+   ...
+}
+
+{
+   <re2 uninitialized value in optimized code>
+   Memcheck:Cond
+   fun:_ZN3re23DFA10AddToQueueEPNS0_5WorkqEij
+   ...
+}