1.9.32.13 -> 1.9.32.14

avoid warning
1.9.32.11 -> 1.9.32.13
2016-03-28 09:10:35 -04:00 · 2016-03-28 09:10:05 -04:00 · 2016-02-02 13:50:27 -05:00 · 2015-12-10 10:05:43 -05:00 · 2015-10-08 17:26:44 -04:00 · 2015-07-30 07:27:19 -04:00
25 changed files with 2636 additions and 1524 deletions
@@ -17,8 +17,6 @@
 #   PSOL_BINARY: absolute path to pagespeed_automatic.a

 mod_pagespeed_dir="${MOD_PAGESPEED_DIR:-unset}"
-position_aux="${POSITION_AUX:-unset}"
-
 if [ "$mod_pagespeed_dir" = "unset" ] ; then
  mod_pagespeed_dir="$ngx_addon_dir/psol/include"
  build_from_source=false
@@ -29,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
    echo "   You need to separately download the pagespeed library:"
    echo ""
    echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.1.tar.gz"
-    echo "     $ tar -xzvf 1.9.32.1.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.14.tar.gz"
+    echo "     $ tar -xzvf 1.9.32.14.tar.gz # expands to psol/"
    echo ""
    echo "   Or see the installation instructions:"
    echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -105,7 +103,7 @@ psol_binary="${PSOL_BINARY:-unset}"
 if [ "$psol_binary" = "unset" ] ; then
  if $build_from_source ; then
    psol_binary="\
-        $mod_pagespeed_dir/pagespeed/automatic/pagespeed_automatic.a"
+        $mod_pagespeed_dir/net/instaweb/automatic/pagespeed_automatic.a"
  else
    psol_library_dir="$ngx_addon_dir/psol/lib/$buildtype/$os_name/$arch_name"
    psol_binary="$psol_library_dir/pagespeed_automatic.a"
@@ -171,7 +169,6 @@ if [ $ngx_found = yes ]; then
    $ps_src/log_message_handler.h \
    $ps_src/ngx_base_fetch.h \
    $ps_src/ngx_caching_headers.h \
-    $ps_src/ngx_event_connection.h \
    $ps_src/ngx_fetch.h \
    $ps_src/ngx_gzip_setter.h \
    $ps_src/ngx_list_iterator.h \
@@ -186,7 +183,6 @@ if [ $ngx_found = yes ]; then
    $ps_src/log_message_handler.cc \
    $ps_src/ngx_base_fetch.cc \
    $ps_src/ngx_caching_headers.cc \
-    $ps_src/ngx_event_connection.cc \
    $ps_src/ngx_fetch.cc \
    $ps_src/ngx_gzip_setter.cc \
    $ps_src/ngx_list_iterator.cc \
@@ -197,14 +193,9 @@ if [ $ngx_found = yes ]; then
    $ps_src/ngx_server_context.cc \
    $ps_src/ngx_url_async_fetcher.cc"

-  if [ "$position_aux" = "true" ] ; then
-     HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
-  else
-    # Make pagespeed run immediately before gzip.
-    HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-      sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
-  fi
-
+  # Make pagespeed run immediately before gzip.
+  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
+    sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
  # Make the etag header filter run immediately after gzip.
  HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
    sed "s/$HTTP_GZIP_FILTER_MODULE/ngx_pagespeed_etag_filter $HTTP_GZIP_FILTER_MODULE/")
@@ -214,7 +205,7 @@ if [ $ngx_found = yes ]; then
 else
  cat << END
 $0: error: module ngx_pagespeed requires the pagespeed optimization library.
-Look in obj/autoconf.err for more details.
+Look in objs/autoconf.err for more details.
 END
  exit 1
 fi
@@ -29,7 +29,7 @@
 #include "base/debug/stack_trace.h"
 #include "base/logging.h"
 #include "net/instaweb/public/version.h"
-#include "pagespeed/kernel/base/string_util.h"
+#include "net/instaweb/util/public/string_util.h"

 // Make sure we don't attempt to use LOG macros here, since doing so
 // would cause us to go into an infinite log loop.
@@ -15,154 +15,37 @@
 */

 // Author: jefftk@google.com (Jeff Kaufman)
-#include <unistd.h> //for usleep

 #include "ngx_base_fetch.h"
-#include "ngx_event_connection.h"
 #include "ngx_list_iterator.h"

 #include "ngx_pagespeed.h"

+#include "net/instaweb/http/public/response_headers.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/http/response_headers.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/message_handler.h"

 namespace net_instaweb {

-const char kHeadersComplete = 'H';
-const char kFlush = 'F';
-const char kDone = 'D';
-
-NgxEventConnection* NgxBaseFetch::event_connection = NULL;
-int NgxBaseFetch::active_base_fetches = 0;
-
-NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r,
+NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
                           NgxServerContext* server_context,
                           const RequestContextPtr& request_ctx,
-                           PreserveCachingHeaders preserve_caching_headers,
-                           NgxBaseFetchType base_fetch_type)
+                           PreserveCachingHeaders preserve_caching_headers)
    : AsyncFetch(request_ctx),
      request_(r),
      server_context_(server_context),
      done_called_(false),
      last_buf_sent_(false),
+      pipe_fd_(pipe_fd),
      references_(2),
-      base_fetch_type_(base_fetch_type),
-      preserve_caching_headers_(preserve_caching_headers),
-      detached_(false),
-      suppress_(false) {
+      ipro_lookup_(false),
+      preserve_caching_headers_(preserve_caching_headers) {
  if (pthread_mutex_init(&mutex_, NULL)) CHECK(0);
-  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, 1);
 }

 NgxBaseFetch::~NgxBaseFetch() {
  pthread_mutex_destroy(&mutex_);
-  __sync_add_and_fetch(&NgxBaseFetch::active_base_fetches, -1);
-}
-
-bool NgxBaseFetch::Initialize(ngx_cycle_t* cycle) {
-  CHECK(event_connection == NULL) << "event connection already set";
-  event_connection = new NgxEventConnection(ReadCallback);
-  return event_connection->Init(cycle);
-}
-
-void NgxBaseFetch::Terminate() {
-  if (event_connection != NULL) {
-    GoogleMessageHandler handler;
-    PosixTimer timer;
-    int64 timeout_us = Timer::kSecondUs * 30;
-    int64 end_us = timer.NowUs() + timeout_us;
-    static unsigned int sleep_microseconds = 100;
-
-    handler.Message(
-        kInfo,"NgxBaseFetch::Terminate rounding up %d active base fetches.",
-        NgxBaseFetch::active_base_fetches);
-
-    // Try to continue processing and get the active base fetch count to 0
-    // untill the timeout expires.
-    // TODO(oschaaf): This needs more work.
-    while (NgxBaseFetch::active_base_fetches > 0 && end_us > timer.NowUs()) {
-      event_connection->Drain();
-      usleep(sleep_microseconds);
-    }
-
-    if (NgxBaseFetch::active_base_fetches != 0) {
-      handler.Message(
-          kWarning,"NgxBaseFetch::Terminate timed out with %d active base fetches.",
-          NgxBaseFetch::active_base_fetches);
-    }
-
-    // Close down the named pipe.
-    event_connection->Shutdown();
-    delete event_connection;
-    event_connection = NULL;
-  }
-}
-
-const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
-  switch(type) {
-    case kPageSpeedResource:
-      return "ps resource";
-    case kHtmlTransform:
-      return "html transform";
-    case kAdminPage:
-      return "admin page";
-    case kIproLookup:
-      return "ipro lookup";
-  }
-  CHECK(false);
-  return "can't get here";
-}
-
-void NgxBaseFetch::ReadCallback(const ps_event_data& data) {
-  NgxBaseFetch* base_fetch = reinterpret_cast<NgxBaseFetch*>(data.sender);
-  ngx_http_request_t* r = base_fetch->request();
-  bool detached = base_fetch->detached();
-#if (NGX_DEBUG)  // `type` is unused if NGX_DEBUG isn't set, needed for -Werror.
-  const char* type = BaseFetchTypeToCStr(base_fetch->base_fetch_type_);
-#endif
-  int refcount = base_fetch->DecrementRefCount();
-
-#if (NGX_DEBUG)
-  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
-     "pagespeed [%p] event: %c. bf:%p (%s) - refcnt:%d - det: %c", r,
-     data.type, base_fetch, type, refcount, detached ? 'Y': 'N');
-#endif
-
-  // If we ended up destructing the base fetch, or the request context is
-  // detached, skip this event.
-  if (refcount == 0 || detached) {
-    return;
-  }
-  ps_request_ctx_t* ctx = ps_get_request_context(r);
-
-  CHECK(data.sender == ctx->base_fetch);
-  CHECK(r->count > 0) << "r->count: " << r->count;
-
-  int rc;
-  // If we are unlucky enough to have our connection finalized mid-ipro-lookup,
-  // we must enter a different flow. Also see ps_in_place_check_header_filter().
-  if ((ctx->base_fetch->base_fetch_type_ != kIproLookup)
-      && r->connection->error) {
-    ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
-      "pagespeed [%p] request already finalized", r);
-    rc = NGX_ERROR;
-  } else {
-    rc = ps_base_fetch::ps_base_fetch_handler(r);
-  }
-
-#if (NGX_DEBUG)
-  ngx_log_error(NGX_LOG_DEBUG, ngx_cycle->log, 0,
-                "pagespeed [%p] ps_base_fetch_handler() returned %d for %c",
-                r, rc, data.type);
-#endif
-
-  ngx_connection_t* c = r->connection;
-  ngx_http_finalize_request(r, rc);
-  // See http://forum.nginx.org/read.php?2,253006,253061
-  ngx_http_run_posted_requests(c);
 }

 void NgxBaseFetch::Lock() {
@@ -232,19 +115,21 @@ ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
                                      preserve_caching_headers_);
 }

-void NgxBaseFetch::RequestCollection(char type) {
-  if (suppress_) {
-    return;
-  }
+void NgxBaseFetch::RequestCollection() {
+  int rc;
+  char c = 'A';  // What byte we write is arbitrary.
+  while (true) {
+    rc = write(pipe_fd_, &c, 1);
+    if (rc == 1) {
+      break;
+    } else if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
+      // TODO(jefftk): is this rare enough that spinning isn't a problem?  Could
+      // we get into a case where the pipe fills up and we spin forever?

-  // We must optimistically increment the refcount, and decrement it
-  // when we conclude we failed. If we only increment on a successfull write,
-  // there's a small chance that between writing and adding to the refcount
-  // both pagespeed and nginx will release their refcount -- destructing
-  // this NgxBaseFetch instance.
-  IncrementRefCount();
-  if (!event_connection->WriteEvent(type, this)) {
-    DecrementRefCount();
+    } else {
+      perror("NgxBaseFetch::RequestCollection");
+      break;
+    }
  }
 }

@@ -252,54 +137,48 @@ void NgxBaseFetch::HandleHeadersComplete() {
  int status_code = response_headers()->status_code();
  bool status_ok = (status_code != 0) && (status_code < 400);

-  if ((base_fetch_type_ != kIproLookup) || status_ok) {
+  if (!ipro_lookup_ || status_ok) {
    // If this is a 404 response we need to count it in the stats.
    if (response_headers()->status_code() == HttpStatus::kNotFound) {
      server_context_->rewrite_stats()->resource_404_count()->Add(1);
    }
  }

-  RequestCollection(kHeadersComplete);  // Headers available.
-
-  // For the IPRO lookup, supress notification of the nginx side here.
-  // If we send both the headerscomplete event and the one from done, nasty
-  // stuff will happen if we loose the race with with the nginx side destructing
-  // this base fetch instance.
-  if (base_fetch_type_ == kIproLookup && !status_ok) {
-    suppress_ = true;
+  // For the IPRO lookup, suppress notification of the nginx side here.
+  // If we send both this event and the one from done, nasty stuff will happen
+  // if we loose the race with with the nginx side destructing this base fetch
+  // instance (and thereby clearing the byte and its pending extraneous event.
+  if (!ipro_lookup_) {
+    RequestCollection();  // Headers available.
  }
 }

 bool NgxBaseFetch::HandleFlush(MessageHandler* handler) {
-  RequestCollection(kFlush);  // A new part of the response body is available
+  RequestCollection();  // A new part of the response body is available.
  return true;
 }

-int NgxBaseFetch::DecrementRefCount() {
-  return DecrefAndDeleteIfUnreferenced();
+void NgxBaseFetch::Release() {
+  DecrefAndDeleteIfUnreferenced();
 }

-int NgxBaseFetch::IncrementRefCount() {
-  return __sync_add_and_fetch(&references_, 1);
-}
-
-int NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
+void NgxBaseFetch::DecrefAndDeleteIfUnreferenced() {
  // Creates a full memory barrier.
-  int r = __sync_add_and_fetch(&references_, -1);
-  if (r == 0) {
+  if (__sync_add_and_fetch(&references_, -1) == 0) {
    delete this;
  }
-  return r;
 }

 void NgxBaseFetch::HandleDone(bool success) {
  // TODO(jefftk): it's possible that instead of locking here we can just modify
  // CopyBufferToNginx to only read done_called_ once.
-  CHECK(!done_called_) << "Done already called!";
  Lock();
  done_called_ = true;
  Unlock();
-  RequestCollection(kDone);
+
+  close(pipe_fd_);  // Indicates to nginx that we're done with the rewrite.
+  pipe_fd_ = -1;
+
  DecrefAndDeleteIfUnreferenced();
 }

@@ -17,32 +17,22 @@
 // Author: jefftk@google.com (Jeff Kaufman)
 //
 // Collects output from pagespeed and buffers it until nginx asks for it.
-// Notifies nginx via NgxEventConnection to call ReadCallback() when
-// the headers are computed, when a flush should be performed, and when done.
+// Notifies nginx via pipe to call CollectAccumulatedWrites() on flush.
 //
 //  - nginx creates a base fetch and passes it to a new proxy fetch.
 //  - The proxy fetch manages rewriting and thread complexity, and through
 //    several chained steps passes rewritten html to HandleWrite().
 //  - Written data is buffered.
-//  - When HandleHeadersComplete(), HandleFlush(), or HandleDone() is called by
-//    PSOL, events are written to NgxEventConnection which will end up being
-//    handled by ReadCallback() on nginx's thread.
-//    When applicable, request processing will be continued via a call to
-//    ps_base_fetch_handler().
-//  - ps_base_fetch_handler() will pull the header and body bytes from PSOL
-//    via CollectAccumulatedWrites() and write those to the module's output.
+//  - When Flush() is called the base fetch writes a byte to a pipe nginx is
+//    watching so nginx knows to call CollectAccumulatedWrites() to pick up the
+//    rewritten html.
+//  - When Done() is called the base fetch closes the pipe, which tells nginx to
+//    make a final call to CollectAccumulatedWrites().
 //
-// This class is referred to in three places: the proxy fetch, nginx's request,
-// and pending events written to the associated NgxEventConnection. It must stay
-// alive until the proxy fetch and nginx request are finished, and no more
-// events are pending.
-//  - The proxy fetch will call Done() to indicate this.
-//  - nginx will call Detach() when the associated request is handled
-//    completely (e.g. the request context is about to be destroyed).
-//  - ReadCallback() will call DecrementRefCount() on instances associated to
-//    events it handles.
-//
-// When the last reference is dropped, this class will delete itself.
+// This class is referred two in two places: the proxy fetch and nginx's
+// request.  It must stay alive until both are finished.  The proxy fetch will
+// call Done() to indicate this; nginx will call Release().  Once both Done()
+// and Release() have been called this class will delete itself.

 #ifndef NGX_BASE_FETCH_H_
 #define NGX_BASE_FETCH_H_
@@ -55,42 +45,22 @@ extern "C" {

 #include "ngx_pagespeed.h"

-#include "ngx_event_connection.h"
 #include "ngx_server_context.h"

 #include "net/instaweb/http/public/async_fetch.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/http/headers.h"
+#include "net/instaweb/http/public/headers.h"
+#include "net/instaweb/util/public/string.h"

 namespace net_instaweb {

-enum NgxBaseFetchType {
-  kIproLookup,
-  kHtmlTransform,
-  kPageSpeedResource,
-  kAdminPage
-};
-
 class NgxBaseFetch : public AsyncFetch {
 public:
-  NgxBaseFetch(ngx_http_request_t* r, NgxServerContext* server_context,
+  NgxBaseFetch(ngx_http_request_t* r, int pipe_fd,
+               NgxServerContext* server_context,
               const RequestContextPtr& request_ctx,
-               PreserveCachingHeaders preserve_caching_headers,
-               NgxBaseFetchType base_fetch_type);
+               PreserveCachingHeaders preserve_caching_headers);
  virtual ~NgxBaseFetch();

-  // Statically initializes event_connection, require for PSOL and nginx to
-  // communicate.
-  static bool Initialize(ngx_cycle_t* cycle);
-
-  // Attempts to finish up request processing queued up in the named pipe and
-  // PSOL for a fixed amount of time. If time is up, a fast and rough shutdown
-  // is attempted.
-  // Statically terminates and NULLS event_connection.
-  static void Terminate();
-
-  static void ReadCallback(const ps_event_data& data);
-
  // Puts a chain in link_ptr if we have any output data buffered.  Returns
  // NGX_OK on success, NGX_ERROR on errors.  If there's no data to send, sends
  // data only if Done() has been called.  Indicates the end of output by
@@ -107,22 +77,9 @@ class NgxBaseFetch : public AsyncFetch {
  // time for resource fetches.  Not called at all for proxy fetches.
  ngx_int_t CollectHeaders(ngx_http_headers_out_t* headers_out);

-  // Called by nginx to decrement the refcount.
-  int DecrementRefCount();
-
-  // Called by pagespeed to increment the refcount.
-  int IncrementRefCount();
-
-  // Detach() is called when the nginx side releases this base fetch. It
-  // sets detached_ to true and decrements the refcount. We need to know
-  // this to be able to handle events which nginx request context has been
-  // released while the event was in-flight.
-  void Detach() { detached_ = true; DecrementRefCount(); }
-
-  bool detached() { return detached_; }
-
-  ngx_http_request_t* request() { return request_; }
-  NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
+  // Called by nginx when it's done with us.
+  void Release();
+  void set_ipro_lookup(bool x) { ipro_lookup_ = x; }

 private:
  virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
@@ -132,7 +89,7 @@ class NgxBaseFetch : public AsyncFetch {

  // Indicate to nginx that we would like it to call
  // CollectAccumulatedWrites().
-  void RequestCollection(char type);
+  void RequestCollection();

  // Lock must be acquired first.
  // Returns:
@@ -148,29 +105,20 @@ class NgxBaseFetch : public AsyncFetch {

  // Called by Done() and Release().  Decrements our reference count, and if
  // it's zero we delete ourself.
-  int DecrefAndDeleteIfUnreferenced();
-
-  static NgxEventConnection* event_connection;
-  
-  // Live count of NgxBaseFetch instances that are currently in use.
-  static int active_base_fetches;
+  void DecrefAndDeleteIfUnreferenced();

  ngx_http_request_t* request_;
  GoogleString buffer_;
  NgxServerContext* server_context_;
  bool done_called_;
  bool last_buf_sent_;
+  int pipe_fd_;
  // How many active references there are to this fetch. Starts at two,
-  // decremented once when Done() is called and once when Detach() is called.
-  // Incremented for each event written by pagespeed for this NgxBaseFetch, and
-  // decremented on the nginx side for each event read for it.
+  // decremented once when Done() is called and once when Release() is called.
  int references_;
  pthread_mutex_t mutex_;
-  NgxBaseFetchType base_fetch_type_;
+  bool ipro_lookup_;
  PreserveCachingHeaders preserve_caching_headers_;
-  // Set to true just before the nginx side releases its reference
-  bool detached_;
-  bool suppress_;

  DISALLOW_COPY_AND_ASSIGN(NgxBaseFetch);
 };
@@ -1,172 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: oschaaf@we-amp.com (Otto van der Schaaf)
-
-extern "C" {
-
-#include <ngx_channel.h>
-
-}
-
-#include "ngx_event_connection.h"
-
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/message_handler.h"
-
-namespace net_instaweb {
-
-  NgxEventConnection::NgxEventConnection(callbackPtr callback)
-    : event_handler_(callback) {
-}
-
-bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
-  int file_descriptors[2];
-
-  if (pipe(file_descriptors) != 0) {
-    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "pagespeed: pipe() failed");
-    return false;
-  }
-  if (ngx_nonblocking(file_descriptors[0]) == -1) {
-    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
-                  ngx_nonblocking_n "pagespeed:  pipe[0] failed");
-  } else if (ngx_nonblocking(file_descriptors[1]) == -1) {
-    ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno,
-                  ngx_nonblocking_n "pagespeed:  pipe[1] failed");
-  } else if (!CreateNgxConnection(cycle, file_descriptors[0])) {
-    ngx_log_error(NGX_LOG_EMERG, cycle->log, 0,
-                  "pagespeed: failed to create connection.");
-  } else {
-    pipe_read_fd_ = file_descriptors[0];
-    pipe_write_fd_ = file_descriptors[1];
-    return true;
-  }
-  close(file_descriptors[0]);
-  close(file_descriptors[1]);
-  return false;
-}
-
-bool NgxEventConnection::CreateNgxConnection(ngx_cycle_t* cycle,
-                                             ngx_fd_t pipe_fd) {
-  // pipe_fd (the read side of the pipe will end up as c->fd on the
-  // underlying ngx_connection_t that gets created here)
-  ngx_int_t rc = ngx_add_channel_event(cycle, pipe_fd, NGX_READ_EVENT,
-      &NgxEventConnection::ReadEventHandler);
-  return rc  == NGX_OK;
-}
-
-void NgxEventConnection::ReadEventHandler(ngx_event_t* ev) {
-  ngx_connection_t* c = static_cast<ngx_connection_t*>(ev->data);
-  ngx_int_t result = ngx_handle_read_event(ev, 0);
-  if (result != NGX_OK) {
-    CHECK(false) << "pagespeed: ngx_handle_read_event error: " << result;
-  }
-
-  if (ev->timedout) {
-    ev->timedout = 0;
-    return;
-  }
-
-  if (!NgxEventConnection::ReadAndNotify(c->fd)) {
-    // This was copied from ngx_channel_handler(): for epoll, we need to call
-    // ngx_del_conn(). Sadly, no documentation as to why.
-    if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
-      ngx_del_conn(c, 0);
-    }
-    ngx_close_connection(c);
-    ngx_del_event(ev, NGX_READ_EVENT, 0);
-  }
-}
-
-// Deserialize ps_event_data's from the pipe as they become available.
-// Subsequently do some bookkeeping, cleanup, and error checking to keep
-// the mess out of ps_base_fetch_handler.
-bool NgxEventConnection::ReadAndNotify(ngx_fd_t fd) {
-  while (true) {
-    // We read only one ps_event_data at a time for now:
-    // We can end up recursing all the way and end up calling ourselves here.
-    // If that happens in the middle of looping over multiple ps_event_data's we
-    // have obtained with read(), the results from the next read() will make us
-    // process events out of order. Which can give headaches.
-    // Alternatively, we could maintain a queue to make sure we process in
-    // sequence
-    ps_event_data data;
-    ngx_int_t size = read(fd, static_cast<void*>(&data), sizeof(data));
-
-    if (size == -1) {
-      if (errno == EINTR) {
-        continue;
-      // TODO(oschaaf): should we worry about spinning here?
-      } else if (ngx_errno == EAGAIN || ngx_errno == EWOULDBLOCK) {
-        return true;
-      }
-    }
-
-    if (size <= 0) {
-      return false;
-    }
-
-    data.connection->event_handler_(data);
-    return true;
-  }
-}
-
-bool NgxEventConnection::WriteEvent(void* sender) {
-  return WriteEvent('X' /* Anything char is fine */, sender);
-}
-
-bool NgxEventConnection::WriteEvent(char type, void* sender) {
-  ssize_t size = 0;
-  ps_event_data data;
-
-  ngx_memzero(&data, sizeof(data));
-  data.type = type;
-  data.sender = sender;
-  data.connection = this;
-
-  while (true) {
-    size = write(pipe_write_fd_,
-                 static_cast<void*>(&data), sizeof(data));
-    if (size == sizeof(data)) {
-      return true;
-    } else if (size == -1) {
-      // TODO(oschaaf): should we worry about spinning here?
-      if (ngx_errno == EINTR || ngx_errno == EAGAIN
-          || ngx_errno == EWOULDBLOCK) {
-        continue;
-      } else {
-        return false;
-      }
-    } else {
-      CHECK(false) << "pagespeed: unexpected return value from write(): "
-                   << size;
-    }
-  }
-  CHECK(false) << "Should not get here";
-  return false;
-}
-
-// Reads and processes what is available in the pipe.
-void NgxEventConnection::Drain() {
-  NgxEventConnection::ReadAndNotify(pipe_read_fd_);
-}
-
-void NgxEventConnection::Shutdown() {
-  close(pipe_write_fd_);
-  close(pipe_read_fd_);
-}
-
-}  // namespace net_instaweb
@@ -1,84 +0,0 @@
-/*
- * Copyright 2014 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Author: oschaaf@we-amp.com (Otto van der Schaaf)
-//
-// NgxEventConnection implements a means to send events from other threads to
-// nginx's event loop, and is implemented by a named pipe under the hood.
-// A single instance is used by NgxBaseFetch, and one instance is created per
-// NgxUrlAsyncFetcher when native fetching is on.
-
-#ifndef NGX_EVENT_CONNECTION_H_
-#define NGX_EVENT_CONNECTION_H_
-
-extern "C" {
-#include <ngx_http.h>
-}
-
-#include <pthread.h>
-
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/http/headers.h"
-
-namespace net_instaweb {
-
-class NgxEventConnection;
-
-// Represents a single event that can be written to or read from the pipe.
-// Technically, sender is the only data we need to send. type and connection are
-// included to provide a means to trace the events along with some more
-// info.
-typedef struct {
-  char type;
-  void* sender;
-  NgxEventConnection* connection;
-} ps_event_data;
-
-// Handler signature for receiving events
-typedef void (*callbackPtr)(const ps_event_data&);
-
-// Abstracts a connection to nginx through which events can be written.
-class NgxEventConnection {
- public:
-  explicit NgxEventConnection(callbackPtr handler);
-
-  // Creates the file descriptors and ngx_connection_t required for event
-  // messaging between pagespeed and nginx.
-  bool Init(ngx_cycle_t* cycle);
-  // Shuts down the underlying file descriptors and connection created in Init()
-  void Shutdown();
-  // Constructs a ps_event_data and writes it to the underlying named pipe.
-  bool WriteEvent(char type, void* sender);
-  // Convenience overload for clients that have a single event type.
-  bool WriteEvent(void* sender);
-  // Reads and processes what is available in the named pipe's buffer.
-  void Drain();
- private:
-  static bool CreateNgxConnection(ngx_cycle_t* cycle, ngx_fd_t pipe_fd);
-  static void ReadEventHandler(ngx_event_t* e);
-  static bool ReadAndNotify(ngx_fd_t fd);
-
-  callbackPtr event_handler_;
-  // We own these file descriptors
-  ngx_fd_t pipe_write_fd_;
-  ngx_fd_t pipe_read_fd_;
-
-  DISALLOW_COPY_AND_ASSIGN(NgxEventConnection);
-};
-
-}  // namespace net_instaweb
-
-#endif  // NGX_EVENT_CONNECTION_H_
@@ -37,7 +37,7 @@ extern "C" {
 }

 #include "ngx_fetch.h"
-
+#include "net/instaweb/util/public/basictypes.h"
 #include "base/logging.h"

 #include <algorithm>
@@ -45,28 +45,71 @@ extern "C" {
 #include <typeinfo>
 #include <vector>

+#include "net/instaweb/util/public/scoped_ptr.h"
 #include "net/instaweb/http/public/async_fetch.h"
 #include "net/instaweb/http/public/inflating_fetch.h"
+#include "net/instaweb/http/public/request_headers.h"
+#include "net/instaweb/http/public/response_headers.h"
+#include "net/instaweb/http/public/response_headers_parser.h"
 #include "net/instaweb/public/version.h"
 #include "net/instaweb/public/global_constants.h"
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/condvar.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/pool.h"
-#include "pagespeed/kernel/base/pool_element.h"
-#include "pagespeed/kernel/base/scoped_ptr.h"
-#include "pagespeed/kernel/base/statistics.h"
-#include "pagespeed/kernel/base/string_writer.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/base/thread_system.h"
-#include "pagespeed/kernel/base/timer.h"
-#include "pagespeed/kernel/base/writer.h"
-#include "pagespeed/kernel/http/request_headers.h"
-#include "pagespeed/kernel/http/response_headers.h"
-#include "pagespeed/kernel/http/response_headers_parser.h"
+#include "net/instaweb/util/public/condvar.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/pool.h"
+#include "net/instaweb/util/public/pool_element.h"
+#include "net/instaweb/util/public/pthread_mutex.h"
+#include "net/instaweb/util/public/statistics.h"
+#include "net/instaweb/util/public/string_writer.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
+#include "net/instaweb/util/public/timer.h"
+#include "net/instaweb/util/public/writer.h"

 namespace net_instaweb {

+class NgxConnection : public PoolElement<NgxConnection> {
+ public:
+  NgxConnection(MessageHandler* handler, int max_keepalive_requests);
+  ~NgxConnection();
+  void SetSock(u_char *sockaddr, socklen_t socklen) {
+    socklen_ = socklen;
+    ngx_memcpy(&sockaddr_, sockaddr, socklen);
+  }
+  // Close ensures that NgxConnection deletes itself at the appropriate time,
+  // which can be after receiving a non-keepalive response, or when the remote
+  // server closes the connection when the NgxConnection is pooled and idle.
+  void Close();
+
+  // Once keepalive is disabled, it can't be toggled back on.
+  void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
+  bool keepalive() { return keepalive_; }
+
+  typedef Pool<NgxConnection> NgxConnectionPool;
+
+  static NgxConnection* Connect(ngx_peer_connection_t* pc,
+                                MessageHandler* handler,
+                                int max_keepalive_requests);
+  static void IdleWriteHandler(ngx_event_t* ev);
+  static void IdleReadHandler(ngx_event_t* ev);
+
+  static NgxConnectionPool connection_pool;
+  static PthreadMutex connection_pool_mutex;
+
+  // c_ is owned by NgxConnection and freed in ::Close()
+  ngx_connection_t* c_;
+  static const int64 keepalive_timeout_ms;
+  static const GoogleString ka_header;
+
+ private:
+  int max_keepalive_requests_;
+  bool keepalive_;
+  socklen_t socklen_;
+  u_char sockaddr_[NGX_SOCKADDRLEN];
+  MessageHandler* handler_;
+
+  DISALLOW_COPY_AND_ASSIGN(NgxConnection);
+};
+
 NgxConnection::NgxConnectionPool NgxConnection::connection_pool;
 PthreadMutex NgxConnection::connection_pool_mutex;
 // Default keepalive 60s.
@@ -87,18 +130,7 @@ NgxConnection::NgxConnection(MessageHandler* handler,
 }

 NgxConnection::~NgxConnection() {
-  CHECK(c_ == NULL) << "NgxConnection: Underlying connection should be NULL";
-}
-
-void NgxConnection::Terminate() {
-  for (NgxConnectionPool::iterator p = connection_pool.begin();
-       p != connection_pool.end(); ++p) {
-    NgxConnection* nc = *p;
-    ngx_close_connection(nc->c_);
-    nc->c_ = NULL;
-    delete nc;
-  }
-  connection_pool.Clear();
+  CHECK(c_ == NULL) << "NgxFetch: Underlying connection should be NULL";
 }

 NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
@@ -131,7 +163,7 @@ NgxConnection* NgxConnection::Connect(ngx_peer_connection_t* pc,
        connection_pool.Remove(nc);

        ngx_log_error(NGX_LOG_DEBUG, pc->log, 0,
-                      "NgxFetch: re-using connection %p (pool size: %l)",
+                      "NgxFetch: re-using connection %p (pool size: %l)\n",
                      nc, connection_pool.size());
        return nc;
      }
@@ -162,7 +194,7 @@ void NgxConnection::Close() {
        // out or has been closed remotely.
        connection_pool.Remove(this);
        ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
-                      "NgxFetch: removed connection %p (pool size: %l)",
+                      "NgxFetch: removed connection %p (pool size: %l)\n",
                      this, connection_pool.size());
        removed_from_pool = true;
        break;
@@ -209,7 +241,7 @@ void NgxConnection::Close() {
    connection_pool.Add(this);
    ngx_log_error(NGX_LOG_DEBUG, c_->log, 0,
                  "NgxFetch: Added connection %p (pool size: %l - "
-                  " max_keepalive_requests_ %d)",
+                  " max_keepalive_requests_ %d)\n",
                  this, connection_pool.size(), max_keepalive_requests_);
  }
 }
@@ -297,7 +329,7 @@ bool NgxFetch::Start(NgxUrlAsyncFetcher* fetcher) {
  fetcher_ = fetcher;
  bool ok = Init();
  if (ok) {
-    ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized",
+    ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: initialized\n",
                  this);
  }  // else Init() will have emitted a reason
  return ok;
@@ -382,7 +414,7 @@ bool NgxFetch::Init() {
      return false;
    } else {
      ngx_log_error(NGX_LOG_DEBUG, log_, 0,
-                    "NgxFetch %p: start resolve for: %s",
+                    "NgxFetch %p: start resolve for: %s\n",
                    this, s_ipaddress.c_str());
    }

@@ -415,10 +447,10 @@ const char* NgxFetch::str_url() {
  return str_url_.c_str();
 }

-// This function should be called only once. The only argument is sucess or
+// This function should be called only once. The only argument is success or
 // not.
 void NgxFetch::CallbackDone(bool success) {
-  ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s",
+  ngx_log_error(NGX_LOG_DEBUG, log_, 0, "NgxFetch %p: CallbackDone: %s\n",
                this, success ? "OK":"FAIL");

  if (async_fetch_ == NULL) {
@@ -455,7 +487,7 @@ void NgxFetch::CallbackDone(bool success) {
        }
      }
      ngx_log_error(NGX_LOG_DEBUG, log_, 0,
-                    "NgxFetch %p: connection %p attempt keep-alive: %s",
+                    "NgxFetch %p: connection %p attempt keep-alive: %s\n",
                    this, connection_, keepalive ? "Yes":"No");
    }

@@ -464,6 +496,9 @@ void NgxFetch::CallbackDone(bool success) {
    connection_ = NULL;
  }

+  // TODO(oschaaf): see https://github.com/pagespeed/ngx_pagespeed/pull/755
+  async_fetch_->Done(success);
+
  if (fetcher_ != NULL) {
    if (fetcher_->track_original_content_length()
        && async_fetch_->response_headers()->Has(
@@ -473,7 +508,7 @@ void NgxFetch::CallbackDone(bool success) {
    }
    fetcher_->FetchComplete(this);
  }
-  async_fetch_->Done(success);
+
  async_fetch_ = NULL;
 }

@@ -704,7 +739,7 @@ int NgxFetch::Connect() {
  connection_ = NgxConnection::Connect(&pc, message_handler(),
                                       fetcher_->max_keepalive_requests_);
  ngx_log_error(NGX_LOG_DEBUG, fetcher_->log_, 0,
-                "NgxFetch %p Connect() connection %p for [%s]",
+                "NgxFetch %p Connect() connection %p for [%s]\n",
                this, connection_, str_url());

  if (connection_ == NULL) {
@@ -720,13 +755,13 @@ int NgxFetch::Connect() {
 }

 // When the fetch sends the request completely, it will hook the read event,
-// and prepare to parse the response. Timer set in Init() is still in effect.
+// and prepare to parse the response.
 void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
  ngx_connection_t* c = static_cast<ngx_connection_t*>(wev->data);
  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
  ngx_buf_t* out = fetch->out_;
-  bool ok = true;
-  while (wev->ready && out->pos < out->last) {
+
+  while (out->pos < out->last) {
    int n = c->send(c, out->pos, out->last - out->pos);
    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
                  "NgxFetch %p: ConnectionWriteHandler "
@@ -735,36 +770,31 @@ void NgxFetch::ConnectionWriteHandler(ngx_event_t* wev) {
    if (n >= 0) {
      out->pos += n;
    } else if (n == NGX_AGAIN) {
-      break;
+      if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
+        fetch->CallbackDone(false);
+      }
+      // Timer set in Init() is still in effect.
+      return;
    } else {
-      ok = false;
-      break;
+      c->error = 1;
+      fetch->CallbackDone(false);
+      return;
    }
  }

-  if (ok) {
-    if (out->pos == out->last) {
-      ok = ngx_handle_read_event(c->read, 0) == NGX_OK;
-    } else {
-      ok = ngx_handle_write_event(c->write, 0) == NGX_OK;
-    }
-  }
-
-  if (!ok) {
-    fetch->message_handler()->Message(
-        kWarning, "NgxFetch %p: failed to hook next event", fetch);
+  if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
    c->error = 1;
    fetch->CallbackDone(false);
  }
+
+  return;
 }

-// Timer set in Init() is still in effect.
 void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {
  ngx_connection_t* c = static_cast<ngx_connection_t*>(rev->data);
  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
-  bool ok = true;

-  while(rev->ready) {
+  for (;;) {
    int n = c->recv(
        c, fetch->in_->start, fetch->in_->end - fetch->in_->start);

@@ -774,39 +804,56 @@ void NgxFetch::ConnectionReadHandler(ngx_event_t* rev) {

    if (n == NGX_AGAIN) {
      break;
-    } else if (n == 0) {
+    }
+
+    if (n == 0) {
      // If the content length was not known, we assume that we have read
      // all if we at least parsed the headers.
      // If we do know the content length, having a mismatch on the bytes read
      // will be interpreted as an error.
-      ok = (fetch->content_length_known_ && fetch->content_length_ == fetch->bytes_received_)
-          || fetch->parser_.headers_complete();
-      fetch->done_ = true;
-      break;
+      if (fetch->content_length_known_) {
+        fetch->CallbackDone(fetch->content_length_ == fetch->bytes_received_);
+      } else {
+        fetch->CallbackDone(fetch->parser_.headers_complete());
+      }
+
+      return;
    } else if (n > 0) {
      fetch->in_->pos = fetch->in_->start;
      fetch->in_->last = fetch->in_->start + n;
-      ok = fetch->response_handler(c);
-      if (fetch->done_ || !ok) {
-        break;
+      if (!fetch->response_handler(c)) {
+        fetch->CallbackDone(false);
+        return;
      }
+
+      if (fetch->done_) {
+        fetch->CallbackDone(true);
+        return;
+      }
+    }
+
+    if (!rev->ready) {
+      break;
    }
  }

-  if (!ok) {
-    fetch->CallbackDone(false);
-  } else if (fetch->done_) {
+  if (fetch->done_) {
    fetch->CallbackDone(true);
-  } else if (ngx_handle_read_event(rev, 0) != NGX_OK) {
+    return;
+  }
+
+  if (ngx_handle_read_event(rev, 0) != NGX_OK) {
    fetch->CallbackDone(false);
  }
+
+  // Timer set in Init() is still in effect.
 }

 // Parse the status line: "HTTP/1.1 200 OK\r\n"
 bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
  NgxFetch* fetch = static_cast<NgxFetch*>(c->data);
  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                "NgxFetch %p: Handle status line", fetch);
+                "NgxFetch %p: Handle status line\n", fetch);

  // This function only works after Nginx-1.1.4. Before nginx-1.1.4,
  // ngx_http_parse_status_line didn't save http_version.
@@ -819,20 +866,14 @@ bool NgxFetch::HandleStatusLine(ngx_connection_t* c) {
  } else if (n == NGX_AGAIN) {  // not completed
    return true;
  }
-
  ResponseHeaders* response_headers =
      fetch->async_fetch_->response_headers();
  response_headers->SetStatusAndReason(
      static_cast<HttpStatus::Code>(fetch->get_status_code()));
  response_headers->set_major_version(fetch->get_major_version());
  response_headers->set_minor_version(fetch->get_minor_version());
-
-  fetch->in_->pos += n;
  fetch->set_response_handler(NgxFetch::HandleHeader);
-  if ((fetch->in_->last - fetch->in_->pos) > 0) {
-    return fetch->response_handler(c);
-  }
-  return true;
+  return fetch->response_handler(c);
 }

 // Parse the HTTP headers
@@ -844,16 +885,12 @@ bool NgxFetch::HandleHeader(ngx_connection_t* c) {
                                       fetch->message_handler_);

  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                "NgxFetch %p: Handle headers", fetch);
+                "NgxFetch %p: Handle headers\n", fetch);

  if (n > size) {
    return false;
  } else if (fetch->parser_.headers_complete()) {
-    // TODO(oschaaf): We should also check if the request method was HEAD
-    // - but I don't think PSOL uses that at this point.
-    if (fetch->get_status_code() == 304 || fetch->get_status_code() == 204) {
-      fetch->done_ = true;
-    } else if (fetch->async_fetch_->response_headers()->FindContentLength(
+    if (fetch->async_fetch_->response_headers()->FindContentLength(
            &fetch->content_length_)) {
      if (fetch->content_length_ < 0) {
        fetch->message_handler_->Message(
@@ -874,11 +911,9 @@ bool NgxFetch::HandleHeader(ngx_connection_t* c) {
    }

    fetch->in_->pos += n;
-    if (!fetch->done_) {
-      fetch->set_response_handler(NgxFetch::HandleBody);
-      if ((fetch->in_->last - fetch->in_->pos) > 0) {
-        return fetch->response_handler(c);
-      }
+    fetch->set_response_handler(NgxFetch::HandleBody);
+    if ((fetch->in_->last - fetch->in_->pos) > 0) {
+      return fetch->response_handler(c);
    }
  } else {
    fetch->in_->pos += n;
@@ -895,7 +930,7 @@ bool NgxFetch::HandleBody(ngx_connection_t* c) {
  fetch->bytes_received_add(size);

  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                "NgxFetch %p: Handle body (%d bytes)", fetch, size);
+                "NgxFetch %p: Handle body (%d bytes)\n", fetch, size);

  if ( fetch->async_fetch_->Write(StringPiece(data, size),
                                  fetch->message_handler()) ) {
@@ -905,7 +940,7 @@ bool NgxFetch::HandleBody(ngx_connection_t* c) {
    fetch->in_->pos += size;
  } else {
    ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                  "NgxFetch %p: async fetch write failure", fetch);
+                  "NgxFetch %p: async fetch write failure\n", fetch);
    return false;
  }
  return true;
@@ -914,7 +949,7 @@ bool NgxFetch::HandleBody(ngx_connection_t* c) {
 void NgxFetch::TimeoutHandler(ngx_event_t* tev) {
  NgxFetch* fetch = static_cast<NgxFetch*>(tev->data);
  ngx_log_error(NGX_LOG_DEBUG, fetch->log_, 0,
-                "NgxFetch %p: TimeoutHandler called", fetch);
+                "NgxFetch %p: TimeoutHandler called\n", fetch);
  fetch->CallbackDone(false);
 }

@@ -39,14 +39,12 @@ extern "C" {

 #include "ngx_url_async_fetcher.h"
 #include <vector>
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/pool.h"
+#include "net/instaweb/util/public/string.h"
 #include "net/instaweb/http/public/url_async_fetcher.h"
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/pool.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/http/response_headers.h"
-#include "pagespeed/kernel/http/response_headers_parser.h"
-#include "pagespeed/kernel/thread/pthread_mutex.h"
-
+#include "net/instaweb/http/public/response_headers.h"
+#include "net/instaweb/http/public/response_headers_parser.h"

 namespace net_instaweb {

@@ -55,51 +53,6 @@ typedef bool (*response_handler_pt)(ngx_connection_t* c);
 class NgxUrlAsyncFetcher;
 class NgxConnection;

-class NgxConnection : public PoolElement<NgxConnection> {
- public:
-  NgxConnection(MessageHandler* handler, int max_keepalive_requests);
-  ~NgxConnection();
-  void SetSock(u_char *sockaddr, socklen_t socklen) {
-    socklen_ = socklen;
-    ngx_memcpy(&sockaddr_, sockaddr, socklen);
-  }
-  // Close ensures that NgxConnection deletes itself at the appropriate time,
-  // which can be after receiving a non-keepalive response, or when the remote
-  // server closes the connection when the NgxConnection is pooled and idle.
-  void Close();
-
-  // Once keepalive is disabled, it can't be toggled back on.
-  void set_keepalive(bool k) { keepalive_ = keepalive_ && k; }
-  bool keepalive() { return keepalive_; }
-
-  typedef Pool<NgxConnection> NgxConnectionPool;
-
-  static NgxConnection* Connect(ngx_peer_connection_t* pc,
-                                MessageHandler* handler,
-                                int max_keepalive_requests);
-  static void IdleWriteHandler(ngx_event_t* ev);
-  static void IdleReadHandler(ngx_event_t* ev);
-  // Terminate will cleanup any idle connections upon shutdown.
-  static void Terminate();
-
-  static NgxConnectionPool connection_pool;
-  static PthreadMutex connection_pool_mutex;
-
-  // c_ is owned by NgxConnection and freed in ::Close()
-  ngx_connection_t* c_;
-  static const int64 keepalive_timeout_ms;
-  static const GoogleString ka_header;
-
- private:
-  int max_keepalive_requests_;
-  bool keepalive_;
-  socklen_t socklen_;
-  u_char sockaddr_[NGX_SOCKADDRLEN];
-  MessageHandler* handler_;
-
-  DISALLOW_COPY_AND_ASSIGN(NgxConnection);
-};
-
 class NgxFetch : public PoolElement<NgxFetch> {
 public:
  NgxFetch(const GoogleString& url,
@@ -70,7 +70,7 @@ extern "C" {
  }
 }

-NgxGZipSetter::NgxGZipSetter() : enabled_(0) { }
+NgxGZipSetter::NgxGZipSetter() : enabled_(false), initialized_(false) { }
 NgxGZipSetter::~NgxGZipSetter() { }

 // Helper functions to determine signature.
@@ -95,7 +95,7 @@ bool IsNgxBitmaskCommand(ngx_command_t* command) {
          HasLocalConfig(command));
 }

-// Initialize the NgxGzipSetter.
+// Initialize the NgxGZipSetter.
 // Find the gzip, gzip_vary, gzip_http_version and gzip_types commands in the
 // gzip module. Enable if the signature of the zip command matches with what we
 // trust. Also sets up redirects for the configurations. These redirect handle
@@ -105,6 +105,16 @@ void NgxGZipSetter::Init(ngx_conf_t* cf) {
 #if (NGX_HTTP_GZIP)
  bool gzip_signature_mismatch = false;
  bool other_signature_mismatch = false;
+  // If we initialized already we don't have to scan again.
+  if (initialized_) {
+    // Config might have changed, so re-enable if we have gzip.
+    if (gzip_command_.command_ != NULL) {
+      enabled_ = true;
+    } else {
+      enabled_ = false;
+    }
+    return;
+  } 
  for (int m = 0; ngx_modules[m] != NULL; m++) {
    if (ngx_modules[m]->commands != NULL) {
      for (int c = 0; ngx_modules[m]->commands[c].name.len; c++) {
@@ -122,7 +132,7 @@ void NgxGZipSetter::Init(ngx_conf_t* cf) {
            current_command->set = ngx_gzip_redirect_conf_set_flag_slot;
            gzip_command_.command_ = current_command;
            gzip_command_.module_ = ngx_modules[m];
-            enabled_ = 1;
+            enabled_ = true;
          } else {
            ngx_conf_log_error(
                NGX_LOG_WARN, cf, 0,
@@ -189,6 +199,7 @@ void NgxGZipSetter::Init(ngx_conf_t* cf) {
      }
    }
  }
+  initialized_ = true;
  if (gzip_signature_mismatch) {
    return;  // Already logged error.
  } else if (!enabled_) {
@@ -381,7 +392,7 @@ void NgxGZipSetter::AddGZipHTTPTypes(ngx_conf_t* cf) {
 }

 void NgxGZipSetter::RollBackAndDisable(ngx_conf_t* cf) {
-  ngx_conf_log_error(NGX_LOG_INFO, cf, 0,
+  ngx_conf_log_error(NGX_LOG_DEBUG, cf, 0,
                     "pagespeed: rollback gzip, explicit configuration");
  for (std::vector<ngx_flag_t*>::iterator i = ngx_flags_set_.begin();
       i != ngx_flags_set_.end(); ++i) {
@@ -60,7 +60,7 @@ extern "C" {

 #include "ngx_pagespeed.h"

-#include "pagespeed/kernel/base/basictypes.h"
+#include "net/instaweb/util/public/basictypes.h"

 namespace net_instaweb {

@@ -91,6 +91,7 @@ class NgxGZipSetter {
  ngx_command_ctx gzip_vary_command_;
  ngx_command_ctx gzip_http_version_command_;
  bool enabled_;
+  bool initialized_;

 public:
  NgxGZipSetter();
@@ -18,13 +18,13 @@

 #include <signal.h>

+#include "net/instaweb/util/public/abstract_mutex.h"
+#include "net/instaweb/util/public/debug.h"
+#include "net/instaweb/util/public/shared_circular_buffer.h"
+#include "net/instaweb/util/public/string_util.h"
 #include "net/instaweb/public/version.h"
-#include "pagespeed/kernel/base/abstract_mutex.h"
-#include "pagespeed/kernel/base/debug.h"
 #include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/base/string_util.h"
 #include "pagespeed/kernel/base/time_util.h"
-#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"

 namespace {

@@ -88,28 +88,31 @@ ngx_uint_t NgxMessageHandler::GetNgxLogLevel(MessageType type) {
  return NGX_LOG_ALERT;
 }

-void NgxMessageHandler::MessageSImpl(MessageType type,
-                                     const GoogleString& message) {
+void NgxMessageHandler::MessageVImpl(MessageType type, const char* msg,
+                                     va_list args) {
+  ngx_uint_t log_level = GetNgxLogLevel(type);
+  GoogleString formatted_message = Format(msg, args);
  if (log_ != NULL) {
-    ngx_uint_t log_level = GetNgxLogLevel(type);
    ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s",
-                  kModuleName, kModPagespeedVersion, message.c_str());
+                  kModuleName, kModPagespeedVersion, formatted_message.c_str());
  } else {
-    GoogleMessageHandler::MessageSImpl(type, message);
+    GoogleMessageHandler::MessageVImpl(type, msg, args);
  }
  // Prepare a log message for the SharedCircularBuffer only.
-  AddMessageToBuffer(type, message);
+  AddMessageToBuffer(type, formatted_message);
 }

-void NgxMessageHandler::FileMessageSImpl(
-    MessageType type, const char* file, int line, const GoogleString& message) {
+void NgxMessageHandler::FileMessageVImpl(MessageType type, const char* file,
+                                         int line, const char* msg,
+                                         va_list args) {
+  ngx_uint_t log_level = GetNgxLogLevel(type);
+  GoogleString formatted_message = Format(msg, args);
  if (log_ != NULL) {
-    ngx_uint_t log_level = GetNgxLogLevel(type);
    ngx_log_error(log_level, log_, 0/*ngx_err_t*/, "[%s %s] %s:%d:%s",
                  kModuleName, kModPagespeedVersion, file, line,
-                  message.c_str());
+                  formatted_message.c_str());
  } else {
-    GoogleMessageHandler::FileMessageSImpl(type, file, line, message);
+    GoogleMessageHandler::FileMessageVImpl(type, file, line, msg, args);
  }
 }

@@ -28,11 +28,11 @@ extern "C" {

 #include <cstdarg>

-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/system/system_message_handler.h"
+#include "net/instaweb/system/public/system_message_handler.h"
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"

 namespace net_instaweb {

@@ -53,10 +53,10 @@ class NgxMessageHandler : public SystemMessageHandler {
  ngx_log_t* log() { return log_; }

 protected:
-  virtual void MessageSImpl(MessageType type, const GoogleString& message);
+  virtual void MessageVImpl(MessageType type, const char* msg, va_list args);

-  virtual void FileMessageSImpl(MessageType type, const char* file,
-                                int line, const GoogleString& message);
+  virtual void FileMessageVImpl(MessageType type, const char* filename,
+                                int line, const char* msg, va_list args);

 private:
  ngx_uint_t GetNgxLogLevel(MessageType type);
@@ -34,8 +34,8 @@ extern "C" {
 }

 #include "base/logging.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/http/response_headers.h"
+#include "net/instaweb/http/public/response_headers.h"
+#include "net/instaweb/util/public/string_util.h"

 namespace net_instaweb {

@@ -84,11 +84,15 @@ enum PreserveCachingHeaders {
 typedef struct {
  NgxBaseFetch* base_fetch;

+  ngx_connection_t* pagespeed_connection;
  ngx_http_request_t* r;

  bool html_rewrite;
  bool in_place;

+  bool write_pending;
+  bool fetch_done;
+
  PreserveCachingHeaders preserve_caching_headers;

  // for html rewrite
@@ -105,7 +109,6 @@ typedef struct {
  GoogleString url_string;
 } ps_request_ctx_t;

-ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

 void copy_request_headers_from_ngx(const ngx_http_request_t* r,
                                   RequestHeaders* headers);
@@ -120,12 +123,6 @@ ngx_int_t copy_response_headers_to_ngx(

 StringPiece ps_determine_host(ngx_http_request_t* r);

-namespace ps_base_fetch {
-
-ngx_int_t ps_base_fetch_handler(ngx_http_request_t* r);
-
-}  // namespace ps_base_fetch
-
 }  // namespace net_instaweb

 #endif  // NGX_PAGESPEED_H_
@@ -26,30 +26,30 @@
 #include "ngx_server_context.h"
 #include "ngx_url_async_fetcher.h"

+#include "net/instaweb/http/public/content_type.h"
 #include "net/instaweb/http/public/rate_controller.h"
 #include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
 #include "net/instaweb/http/public/wget_url_fetcher.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
 #include "net/instaweb/rewriter/public/server_context.h"
+#include "net/instaweb/system/public/in_place_resource_recorder.h"
+#include "net/instaweb/system/public/serf_url_async_fetcher.h"
+#include "net/instaweb/system/public/system_caches.h"
+#include "net/instaweb/system/public/system_rewrite_options.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/null_shared_mem.h"
+#include "net/instaweb/util/public/posix_timer.h"
 #include "net/instaweb/util/public/property_cache.h"
-#include "pagespeed/kernel/base/google_message_handler.h"
-#include "pagespeed/kernel/base/null_shared_mem.h"
-#include "pagespeed/kernel/base/posix_timer.h"
-#include "pagespeed/kernel/base/stdio_file_system.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/base/thread_system.h"
-#include "pagespeed/kernel/http/content_type.h"
-#include "pagespeed/kernel/sharedmem/shared_circular_buffer.h"
-#include "pagespeed/kernel/sharedmem/shared_mem_statistics.h"
+#include "net/instaweb/util/public/scheduler_thread.h"
+#include "net/instaweb/util/public/shared_circular_buffer.h"
+#include "net/instaweb/util/public/shared_mem_statistics.h"
+#include "net/instaweb/util/public/slow_worker.h"
+#include "net/instaweb/util/public/stdio_file_system.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
 #include "pagespeed/kernel/thread/pthread_shared_mem.h"
-#include "pagespeed/kernel/thread/scheduler_thread.h"
-#include "pagespeed/kernel/thread/slow_worker.h"
-#include "pagespeed/system/in_place_resource_recorder.h"
-#include "pagespeed/system/serf_url_async_fetcher.h"
-#include "pagespeed/system/system_caches.h"
-#include "pagespeed/system/system_rewrite_options.h"

 namespace net_instaweb {

@@ -84,8 +84,7 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
      hostname_(hostname.as_string()),
      port_(port),
      process_script_variables_(false),
-      process_script_variables_set_(false),
-      shut_down_(false) {
+      process_script_variables_set_(false) {
  InitializeDefaultOptions();
  default_options()->set_beacon_url("/ngx_pagespeed_beacon");
  SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
@@ -152,6 +151,16 @@ RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
  return options;
 }

+bool NgxRewriteDriverFactory::InitNgxUrlAsyncFetchers() {
+  log_ = ngx_cycle->log;
+  for (size_t i = 0; i < ngx_url_async_fetchers_.size(); ++i) {
+    if (!ngx_url_async_fetchers_[i]->Init()) {
+      return false;
+    }
+  }
+  return true;
+}
+
 bool NgxRewriteDriverFactory::CheckResolver() {
  if (use_native_fetcher_ && resolver_ == NULL) {
    return false;
@@ -177,13 +186,6 @@ ServerContext* NgxRewriteDriverFactory::NewServerContext() {
  return NULL;
 }

-void NgxRewriteDriverFactory::ShutDown() {
-  if (!shut_down_) {
-    shut_down_ = true;
-    SystemRewriteDriverFactory::ShutDown();
-  }
-}
-
 void NgxRewriteDriverFactory::ShutDownMessageHandlers() {
  ngx_message_handler_->set_buffer(NULL);
  ngx_html_parse_message_handler_->set_buffer(NULL);
@@ -209,7 +211,6 @@ void NgxRewriteDriverFactory::StartThreads() {
 }

 void NgxRewriteDriverFactory::LoggingInit(ngx_log_t* log) {
-  log_ = log;
  net_instaweb::log_message_handler::Install(log);
  if (install_crash_handler()) {
    NgxMessageHandler::InstallCrashHandler(log);
@@ -32,9 +32,13 @@ extern "C" {

 #include <set>

-#include "pagespeed/kernel/base/md5_hasher.h"
-#include "pagespeed/kernel/base/scoped_ptr.h"
-#include "pagespeed/system/system_rewrite_driver_factory.h"
+#include "net/instaweb/system/public/system_rewrite_driver_factory.h"
+#include "net/instaweb/util/public/md5_hasher.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
+
+// TODO(oschaaf): We should reparent ApacheRewriteDriverFactory and
+// NgxRewriteDriverFactory to a new class OriginRewriteDriverFactory and factor
+// out as much as possible.

 namespace net_instaweb {

@@ -66,6 +70,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  // NgxRewriteOptions.
  virtual RewriteOptions* NewRewriteOptions();
  virtual ServerContext* NewDecodingServerContext();
+  bool InitNgxUrlAsyncFetchers();
  // Check resolver configured or not.
  bool CheckResolver();

@@ -75,7 +80,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  static void InitStats(Statistics* statistics);
  NgxServerContext* MakeNgxServerContext(StringPiece hostname, int port);
  virtual ServerContext* NewServerContext();
-  virtual void ShutDown();

  // Starts pagespeed threads if they've not been started already.  Must be
  // called after the caller has finished any forking it intends to do.
@@ -159,7 +163,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  int port_;
  bool process_script_variables_;
  bool process_script_variables_set_;
-  bool shut_down_;

  DISALLOW_COPY_AND_ASSIGN(NgxRewriteDriverFactory);
 };
@@ -30,9 +30,9 @@ extern "C" {
 #include "net/instaweb/public/version.h"
 #include "net/instaweb/rewriter/public/file_load_policy.h"
 #include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/timer.h"
-#include "pagespeed/system/system_caches.h"
+#include "net/instaweb/system/public/system_caches.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/timer.h"

 namespace net_instaweb {

@@ -276,15 +276,10 @@ const char* NgxRewriteOptions::ParseAndSetOptions(

  ScriptLine* script_line;
  script_line = NULL;
+  // Only allow script variable support for LoadFromFile for now.
  // Note that LoadFromFile should not be scriptable on wildcard hosts,
  // as browsers might be able to manipulate its natural use-case: $http_host.
-  if (!StringCaseStartsWith(directive, "LoadFromFile") &&
-      !StringCaseEqual(directive, "EnableFilters") &&
-      !StringCaseEqual(directive, "DisableFilters") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeMethod") &&
-      !StringCaseEqual(directive,
-                       "DownstreamCacheRewrittenPercentageThreshold")) {
+  if (!StringCaseStartsWith(directive, "LoadFromFile")) {
    compile_scripts = false;
  }

@@ -515,6 +510,10 @@ NgxRewriteOptions* NgxRewriteOptions::Clone() const {
  return options;
 }

+void NgxRewriteOptions::Merge(const RewriteOptions& src) {
+  SystemRewriteOptions::Merge(src);
+}
+
 const NgxRewriteOptions* NgxRewriteOptions::DynamicCast(
    const RewriteOptions* instance) {
  return dynamic_cast<const NgxRewriteOptions*>(instance);
@@ -29,11 +29,11 @@ extern "C" {

 #include <vector>

+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/ref_counted_ptr.h"
+#include "net/instaweb/util/public/stl_util.h"          // for STLDeleteElements
 #include "net/instaweb/rewriter/public/rewrite_options.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/ref_counted_ptr.h"
-#include "pagespeed/kernel/base/stl_util.h"          // for STLDeleteElements
-#include "pagespeed/system/system_rewrite_options.h"
+#include "net/instaweb/system/public/system_rewrite_options.h"

 #define NGX_PAGESPEED_MAX_ARGS 10

@@ -140,6 +140,7 @@ class NgxRewriteOptions : public SystemRewriteOptions {

  // Make an identical copy of these options and return it.
  virtual NgxRewriteOptions* Clone() const;
+  virtual void Merge(const RewriteOptions& src);

  // Returns a suitably down cast version of 'instance' if it is an instance
  // of this class, NULL if not.
@@ -27,9 +27,9 @@ extern "C" {
 #include "ngx_rewrite_driver_factory.h"
 #include "ngx_rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_driver.h"
-#include "pagespeed/system/add_headers_fetcher.h"
-#include "pagespeed/system/loopback_route_fetcher.h"
-#include "pagespeed/system/system_request_context.h"
+#include "net/instaweb/system/public/add_headers_fetcher.h"
+#include "net/instaweb/system/public/loopback_route_fetcher.h"
+#include "net/instaweb/system/public/system_request_context.h"

 namespace net_instaweb {

@@ -22,7 +22,7 @@
 #define NGX_SERVER_CONTEXT_H_

 #include "ngx_message_handler.h"
-#include "pagespeed/system/system_server_context.h"
+#include "net/instaweb/system/public/system_server_context.h"

 extern "C" {
 #include <ngx_http.h>
@@ -31,22 +31,22 @@ extern "C" {
 #include <map>
 #include <set>

+#include "net/instaweb/util/public/basictypes.h"
 #include "net/instaweb/http/public/async_fetch.h"
 #include "net/instaweb/http/public/inflating_fetch.h"
+#include "net/instaweb/http/public/request_headers.h"
+#include "net/instaweb/http/public/response_headers.h"
+#include "net/instaweb/http/public/response_headers_parser.h"
 #include "net/instaweb/public/version.h"
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/condvar.h"
-#include "pagespeed/kernel/base/message_handler.h"
-#include "pagespeed/kernel/base/pool.h"
-#include "pagespeed/kernel/base/pool_element.h"
-#include "pagespeed/kernel/base/statistics.h"
-#include "pagespeed/kernel/base/string_util.h"
-#include "pagespeed/kernel/base/thread_system.h"
-#include "pagespeed/kernel/base/timer.h"
-#include "pagespeed/kernel/base/writer.h"
-#include "pagespeed/kernel/http/request_headers.h"
-#include "pagespeed/kernel/http/response_headers.h"
-#include "pagespeed/kernel/http/response_headers_parser.h"
+#include "net/instaweb/util/public/condvar.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/pool.h"
+#include "net/instaweb/util/public/pool_element.h"
+#include "net/instaweb/util/public/statistics.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
+#include "net/instaweb/util/public/timer.h"
+#include "net/instaweb/util/public/writer.h"

 namespace net_instaweb {

@@ -65,8 +65,7 @@ namespace net_instaweb {
      thread_system_(thread_system),
      message_handler_(handler),
      mutex_(NULL),
-      max_keepalive_requests_(max_keepalive_requests),
-      event_connection_(NULL) {
+      max_keepalive_requests_(max_keepalive_requests) {
    resolver_timeout_ = resolver_timeout;
    fetch_timeout_ = fetch_timeout;
    ngx_memzero(&proxy_, sizeof(proxy_));
@@ -77,17 +76,12 @@ namespace net_instaweb {
    mutex_ = thread_system_->NewMutex();
    log_ = log;
    pool_ = NULL;
+    command_connection_ = NULL;
+    pipe_fd_ = -1;
    resolver_ = resolver;
-    // If init fails, set shutdown_ so no fetches will be attempted.
-    if (!Init(const_cast<ngx_cycle_t*>(ngx_cycle))) {
-      shutdown_ = true;
-      message_handler_->Message(
-          kError, "NgxUrlAsyncFetcher failed to init, fetching disabled.");
-    }
  }

  NgxUrlAsyncFetcher::~NgxUrlAsyncFetcher() {
-    DCHECK(shutdown_)  << "Shut down before destructing NgxUrlAsyncFetcher.";
    message_handler_->Message(
        kInfo,
        "Destruct NgxUrlAsyncFetcher with [%d] active fetchers",
@@ -95,12 +89,19 @@ namespace net_instaweb {

    CancelActiveFetches();
    active_fetches_.DeleteAll();
-    NgxConnection::Terminate();

    if (pool_ != NULL) {
      ngx_destroy_pool(pool_);
      pool_ = NULL;
    }
+    if (command_connection_ != NULL) {
+      ngx_close_connection(command_connection_);
+      command_connection_ = NULL;
+    }
+    if (pipe_fd_ != -1) {
+      close(pipe_fd_);
+      pipe_fd_ = -1;
+    }
    if (mutex_ != NULL) {
      delete mutex_;
      mutex_ = NULL;
@@ -151,13 +152,9 @@ namespace net_instaweb {

  // Create the pool for fetcher, create the pipe, add the read event for main
  // thread. It should be called in the worker process.
-  bool NgxUrlAsyncFetcher::Init(ngx_cycle_t* cycle) {
-    log_ = cycle->log;
-    CHECK(event_connection_ == NULL) << "event connection already set";
-    event_connection_ = new NgxEventConnection(ReadCallback);
-    if (!event_connection_->Init(cycle)) {
-      return false;
-    }
+  bool NgxUrlAsyncFetcher::Init() {
+    log_ = ngx_cycle->log;
+
    if (pool_ == NULL) {
      pool_ = ngx_create_pool(4096, log_);
      if (pool_ == NULL) {
@@ -167,6 +164,41 @@ namespace net_instaweb {
      }
    }

+    int pipe_fds[2];
+    int rc = pipe(pipe_fds);
+    if (rc != 0) {
+      ngx_log_error(NGX_LOG_ERR, log_, 0, "pipe() failed");
+      return false;
+    }
+    if (ngx_nonblocking(pipe_fds[0]) == -1) {
+      ngx_log_error(NGX_LOG_ERR, log_, 0, "nonblocking pipe[0] failed");
+      return false;
+    }
+    if (ngx_nonblocking(pipe_fds[1]) == -1) {
+      ngx_log_error(NGX_LOG_ERR, log_, 0, "nonblocking pipe[1] failed");
+      return false;
+    }
+
+    pipe_fd_ = pipe_fds[1];
+    command_connection_ = ngx_get_connection(pipe_fds[0], log_);
+    if (command_connection_ == NULL) {
+      close(pipe_fds[1]);
+      close(pipe_fds[0]);
+      pipe_fd_ = -1;
+      return false;
+    }
+
+    command_connection_->recv = ngx_recv;
+    command_connection_->send = ngx_send;
+    command_connection_->recv_chain = ngx_recv_chain;
+    command_connection_->send_chain = ngx_send_chain;
+    command_connection_->log = log_;
+    command_connection_->read->log = log_;
+    command_connection_->write->log = log_;
+    command_connection_->data = this;
+    command_connection_->read->handler = CommandHandler;
+    ngx_add_event(command_connection_->read, NGX_READ_EVENT, 0);
+
    if (proxy_.url.len == 0) {
      return true;
    }
@@ -182,29 +214,8 @@ namespace net_instaweb {
  }

  void NgxUrlAsyncFetcher::ShutDown() {
-    shutdown_ = true;
-    if (!pending_fetches_.empty()) {
-      for (Pool<NgxFetch>::iterator p = pending_fetches_.begin(),
-           e = pending_fetches_.end(); p != e; p++) {
-        NgxFetch* fetch = *p;
-        fetch->CallbackDone(false);
-      }
-      pending_fetches_.DeleteAll();
-    }
-
-    if (!active_fetches_.empty()) {
-      for (Pool<NgxFetch>::iterator p = active_fetches_.begin(),
-           e = active_fetches_.end(); p != e; p++) {
-        NgxFetch* fetch = *p;
-        fetch->CallbackDone(false);
-      }
-      active_fetches_.Clear();
-    }
-    if (event_connection_ != NULL) {
-      event_connection_->Shutdown();
-      delete event_connection_;
-      event_connection_ = NULL;
-    }
+      shutdown_ = true;
+      SendCmd('S');
  }

  // It's called in the rewrite thread. All the fetches are started at
@@ -212,46 +223,92 @@ namespace net_instaweb {
  void NgxUrlAsyncFetcher::Fetch(const GoogleString& url,
                                 MessageHandler* message_handler,
                                 AsyncFetch* async_fetch) {
-    // Don't accept new fetches when shut down. This flow is also entered when
-    // we did not initialize properly in ::Init().
-    if (shutdown_) {
-      async_fetch->Done(false);
-      return;
-    }
    async_fetch = EnableInflation(async_fetch);
    NgxFetch* fetch = new NgxFetch(url, async_fetch,
          message_handler, log_);
    ScopedMutex lock(mutex_);
    pending_fetches_.Add(fetch);
+    SendCmd('F');
+  }

-    // TODO(oschaaf): thread safety on written vs shutdown.
-    // It is possible that shutdown() is called after writing an event? In that
-    // case, this could (rarely) fail when it shouldn't.
-    bool written = event_connection_->WriteEvent(this);
-    CHECK(written || shutdown_) << "NgxUrlAsyncFetcher: event write failure";
+  // send command to nginx main thread
+  // 'F' : start a fetch
+  // 'S' : shutdown the fetcher
+  bool NgxUrlAsyncFetcher::SendCmd(const char command) {
+    int rc;
+    while (true) {
+      rc = write(pipe_fd_, &command, 1);
+      if (rc == 1) {
+        return true;
+      } else if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
+        // TODO(junmin): It's rare. But it need be fixed.
+      } else {
+        return false;
+      }
+    }
+    return true;
  }

  // This is the read event which is called in the main thread.
  // It will do the real work. Add the work event and start the fetch.
-  void NgxUrlAsyncFetcher::ReadCallback(const ps_event_data& data) {
-    std::vector<NgxFetch*> to_start;
-    NgxUrlAsyncFetcher* fetcher = reinterpret_cast<NgxUrlAsyncFetcher*>(
-      data.sender);
+  void NgxUrlAsyncFetcher::CommandHandler(ngx_event_t* cmdev) {
+    char command;
+    int rc;
+    ngx_connection_t* c = static_cast<ngx_connection_t*>(cmdev->data);
+    NgxUrlAsyncFetcher* fetcher = static_cast<NgxUrlAsyncFetcher*>(c->data);
+    do {
+      rc = read(c->fd, &command, 1);
+    } while (rc == -1 && errno == EINTR);

-    fetcher->mutex_->Lock();
-    fetcher->completed_fetches_.DeleteAll();
+    CHECK(rc == -1 || rc == 0 || rc == 1);

-    for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
-             e = fetcher->pending_fetches_.end(); p != e; p++) {
-      NgxFetch* fetch = *p;
-      to_start.push_back(fetch);
+    if (rc == -1 || rc == 0) {
+      // EAGAIN
+      return;
    }

-    fetcher->pending_fetches_.Clear();
-    fetcher->mutex_->Unlock();
+    std::vector<NgxFetch*> to_start;

-    for (size_t i = 0; i < to_start.size(); i++) {
-      fetcher->StartFetch(to_start[i]);
+    switch (command) {
+      // All the new fetches are appended in the pending_fetches.
+      // Start all these fetches.
+      case 'F':
+        fetcher->mutex_->Lock();
+        fetcher->completed_fetches_.DeleteAll();
+        for (Pool<NgxFetch>::iterator p = fetcher->pending_fetches_.begin(),
+                 e = fetcher->pending_fetches_.end(); p != e; p++) {
+          NgxFetch* fetch = *p;
+          to_start.push_back(fetch);
+        }
+
+        fetcher->pending_fetches_.Clear();
+        fetcher->mutex_->Unlock();
+
+        for (size_t i = 0; i < to_start.size(); i++) {
+          fetcher->StartFetch(to_start[i]);
+        }
+        CHECK(ngx_handle_read_event(cmdev, 0) == NGX_OK);
+        break;
+
+      // Shutdown all the fetches.
+      case 'S':
+        if (!fetcher->pending_fetches_.empty()) {
+          fetcher->pending_fetches_.DeleteAll();
+        }
+
+        if (!fetcher->active_fetches_.empty()) {
+          for (Pool<NgxFetch>::iterator p = fetcher->active_fetches_.begin(),
+               e = fetcher->active_fetches_.end(); p != e; p++) {
+            NgxFetch* fetch = *p;
+            fetch->CallbackDone(false);
+          }
+          fetcher->active_fetches_.Clear();
+        }
+        CHECK(ngx_del_event(cmdev, NGX_READ_EVENT, 0) == NGX_OK);
+        break;
+
+      default:
+        break;
    }

    return;
@@ -33,14 +33,11 @@ extern "C" {
 }

 #include <vector>
-
-#include "ngx_event_connection.h"
-
 #include "net/instaweb/http/public/url_async_fetcher.h"
-#include "pagespeed/kernel/base/basictypes.h"
-#include "pagespeed/kernel/base/pool.h"
-#include "pagespeed/kernel/base/string.h"
-#include "pagespeed/kernel/base/thread_system.h"
+#include "net/instaweb/util/public/basictypes.h"
+#include "net/instaweb/util/public/pool.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/thread_system.h"


 namespace net_instaweb {
@@ -62,21 +59,22 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {
  ~NgxUrlAsyncFetcher();

  // It should be called in the module init_process callback function. Do some
-  // intializations which can't be done in the master process
-  bool Init(ngx_cycle_t* cycle);
+  // initializations which can't be done in the master process
+  bool Init();

  // shutdown all the fetches.
  virtual void ShutDown();

-  // the read handler in the main thread
-  static void ReadCallback(const ps_event_data& data);
-
  virtual bool SupportsHttps() const { return false; }

  virtual void Fetch(const GoogleString& url,
                     MessageHandler* message_handler,
                     AsyncFetch* callback);

+  // send the command from the current thread to main thread
+  bool SendCmd(const char command);
+  // the read handler in the main thread
+  static void CommandHandler(ngx_event_t* cmdev);
  bool StartFetch(NgxFetch* fetch);

  // Remove the completed fetch from the active fetch set, and put it into a
@@ -139,13 +137,13 @@ class NgxUrlAsyncFetcher : public UrlAsyncFetcher {

  ngx_pool_t* pool_;
  ngx_log_t* log_;
+  ngx_connection_t* command_connection_;  // the command pipe
+  int pipe_fd_;  // the write pipe end
  ngx_resolver_t* resolver_;
  int max_keepalive_requests_;
  ngx_msec_t resolver_timeout_;
  ngx_msec_t fetch_timeout_;

-  NgxEventConnection* event_connection_;
-
  DISALLOW_COPY_AND_ASSIGN(NgxUrlAsyncFetcher);
 };

@@ -266,14 +266,14 @@ http {
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
-    server_name secondary.example.com;
+    server_name mpd.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";

-    pagespeed MapProxyDomain secondary.example.com/gstatic_images
+    pagespeed MapProxyDomain mpd.example.com/gstatic_images
                             http://www.gstatic.com/psa/static;
  }

-  # These three vhosts are for testing the experiment framework (Furious).
+  # These two vhosts are for testing the experiment framework (Furious).
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
@@ -302,16 +302,6 @@ http {
    pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
    pagespeed ExperimentSpec "id=3;default;percent=0";
  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name experiment.devicematch.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed RunExperiment on;
-    pagespeed ExperimentSpec
-      "id=1;percent=100;matches_device_type=mobile;enable=recompress_images";
-  }

  server {
    listen @@SECONDARY_PORT@@;
@@ -382,34 +372,6 @@ http {
    pagespeed DownstreamCachePurgeLocationPrefix "http://localhost:@@SECONDARY_PORT@@/purge";
  }

-  # Set up a reverse proxy (rproxy.) and origin (origin.) as vhosts for
-  # showing that we can configure PageSpeed via response headers.
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name rproxy.rmcomments.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed RewriteLevel PassThrough;
-    pagespeed DisableFilters add_instrumentation,remove_comments;
-
-    # Note that we don't enable remove_comments here; that setting comes from
-    # the response headers from origin.rmcomments.example.com
-    location / {
-      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/;
-      proxy_set_header "Host" "origin.rmcomments.example.com";
-    }
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name origin.rmcomments.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed off;
-    location / {
-      add_header PageSpeedFilters remove_comments;
-    }
-  }
-
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
@@ -425,6 +387,9 @@ http {
  # Build a configuration hierarchy where at the root we have turned on
  # OptimizeForBandwidth, and in various subdirectories we override settings
  # to make them more aggressive.
+  #
+  # In Apache we can do this all with Directory blocks, but to get the same
+  # inheretence in Nginx we need to have location blocks inside a server block.
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
@@ -583,6 +548,15 @@ http {
    pagespeed EnableFilters rewrite_domains;
  }

+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name retaincomment.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RetainComment " google_ad_section*";
+  }
+
  server {
    # Test host for shared memory cache.
    listen @@SECONDARY_PORT@@;
@@ -825,6 +799,18 @@ http {
    pagespeed FileCachePath "@@FILE_CACHE@@";
  }

+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name notransform.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters rewrite_images;
+
+    add_header Cache-Control no-transform;
+  }
+
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
@@ -883,10 +869,7 @@ http {
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
    pagespeed RewriteLevel PassThrough;
    pagespeed on;
-    location / {
-      proxy_pass http://localhost:@@SECONDARY_PORT@@;
-      proxy_set_header Host "enable-filter-header-origin.example.com";
-    }
+    add_header PageSpeedFilters add_instrumentation;
  }

  server {
@@ -896,24 +879,7 @@ http {
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
    pagespeed EnableFilters add_instrumentation;
    pagespeed on;
-    location / {
-      proxy_pass http://localhost:@@SECONDARY_PORT@@;
-      proxy_set_header Host "pagespeed-off-header-origin.example.com";
-    }
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name headers.example.com;
-    pagespeed InPlaceRewriteDeadlineMs -1;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed LoadFromFile "http://headers.example.com/"
-              "@@SERVER_ROOT@@/";
-    location /mod_pagespeed_test/ {
-      more_set_headers "Server: override";
-      more_set_headers -s '404' 'Cache-Control: override';
-    }
+    add_header PageSpeed off;
  }

  server {
@@ -996,7 +962,7 @@ http {
    listen [::]:@@SECONDARY_PORT@@;
    server_name date.example.com;
    pagespeed FileCachePath "@@FILE_CACHE@@";
-    add_header "Date" "Fri, 16 Oct 2009 23:05:07 GMT";
+    add_header "Date" "Date: Fri, 16 Oct 2009 23:05:07 GMT";
  }

  server {
@@ -1078,54 +1044,29 @@ http {
    pagespeed DisableFilters remove_comments,add_instrumentation;
  }

-  # Start remote config servers.
+  # Test that pagespeed is disabled when sendfile headers are present.
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
-    server_name remote-config.example.com;
+    server_name uses-sendfile.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT1@@/remote.cfg";
-    pagespeed RemoteConfigurationTimeoutMs 1500;
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+    add_header 'X-Sendfile' 'blablabla';
  }
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
-    server_name remote-config-partially-invalid.example.com;
+    server_name doesnt-sendfile.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT2@@/remote.cfg";
-    pagespeed RemoteConfigurationTimeoutMs 1500;
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
  }
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
-    server_name remote-config-invalid.example.com;
+    server_name uses-xaccelredirect.example.com;
    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT3@@/remote.cfg";
-    pagespeed RemoteConfigurationTimeoutMs 1500;
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name remote-config-failed-fetch.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT5@@/remote.cfg";
-    pagespeed RemoteConfigurationTimeoutMs 1500;
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name remote-config-slow-fetch.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT6@@/remote.cfg";
-    pagespeed RemoteConfigurationTimeoutMs 1500;
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name remote-config-experiment.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed RemoteConfigurationUrl "http://127.0.0.1:@@RCPORT7@@/remote.cfg";
-    pagespeed RemoteConfigurationTimeoutMs 1500;
+    pagespeed EnableFilters inline_javascript,rewrite_javascript;
+    add_header 'X-Accel-Redirect' 'blablabla';
  }

  # Proxy + IPRO a gzip'd file for testing Issue 896.
@@ -1220,20 +1161,6 @@ http {
    pagespeed FileCachePath "@@FILE_CACHE@@";
  }

-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name script-filters.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
-    set $filters "";
-    if ($http_X_Script) {
-        set $filters "add_instrumentation";
-    }
-    pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters $filters;
-  }
-
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
@@ -1243,7 +1170,7 @@ http {
    pagespeed EnableCachePurge on;

    pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@";
    pagespeed FileCachePath "@@FILE_CACHE@@_purge";
    pagespeed DisableFilters add_instrumentation;
    pagespeed RewriteLevel PassThrough;
@@ -1253,61 +1180,16 @@ http {
  server {
    listen @@SECONDARY_PORT@@;
    listen [::]:@@SECONDARY_PORT@@;
-    server_name psoff-dir-on.example.com;
-
-    # Test purging individual URLs without flushing the entire metadata cache.
-    pagespeed EnableCachePurge on;
-
-    pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
-    pagespeed FileCachePath "@@FILE_CACHE@@_dir_on";
-    pagespeed DisableFilters add_instrumentation;
-    pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters rewrite_css;
-    pagespeed off;
-    location / {
-      pagespeed on;
-    }
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name  cc-header-origin.example.com;
-    pagespeed InPlaceResourceOptimization off;
-    pagespeed off;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
+    server_name headers.example.com;
    root "@@SERVER_ROOT@@";
-    location /mod_pagespeed_test/nostore {
-      add_header "Cache-Control" "max-age=12345";
-      add_header "Cache-Control" "public, no-store";
-      add_header "Cache-Control" "max-age=14";
-    }
-  }
+    pagespeed FileCachePath "@@FILE_CACHE@@_purge";
+    pagespeed RewriteLevel CoreFilters;

-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name  enable-filter-header-origin.example.com;
-    pagespeed InPlaceResourceOptimization off;
-    pagespeed off;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
-    location / {
-      add_header "PageSpeedFilters" "add_instrumentation";
-    }
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name  pagespeed-off-header-origin.example.com;
-    pagespeed InPlaceResourceOptimization off;
-    pagespeed off;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
-    location / {
-      add_header "PageSpeed" "off";
+    pagespeed InPlaceRewriteDeadlineMs -1;
+    pagespeed LoadFromFile "http://headers.example.com/"
+              "@@SERVER_ROOT@@/";
+    location /mod_pagespeed_test/ {
+      more_set_headers "Server: overriden";
    }
  }

@@ -1355,10 +1237,6 @@ http {
      add_header 'Cache-Control' 'no-transform';
    }

-    location /mod_pagespeed_test/no_transform {
-      add_header 'Cache-Control' 'no-transform';
-    }
-
    # uncomment the following two lines if you're testing memcached
    #pagespeed MemcachedServers "localhost:11211";
    #pagespeed MemcachedThreads 1;
@@ -1373,8 +1251,7 @@ http {
    #pagespeed ExperimentSpec "id=2;percent=50";

    pagespeed Library 43 1o978_K0_LNE5_ystNklf
-      http://www.modpagespeed.com/rewrite_javascript.js;
-    pagespeed RetainComment " google_ad_section*";
+    http://www.modpagespeed.com/rewrite_javascript.js;

    add_header X-Extra-Header 1;

@@ -1400,8 +1277,9 @@ http {
    pagespeed AvoidRenamingIntrospectiveJavascript off;

    location /mod_pagespeed_test/nostore {
-      proxy_pass http://localhost:@@SECONDARY_PORT@@;
-      proxy_set_header Host "cc-header-origin.example.com";
+      add_header "Cache-Control" "max-age=12345";
+      add_header "Cache-Control" "public, no-store";
+      add_header "Cache-Control" "max-age=14";
    }

    location /mod_pagespeed_test/forbid_all_disabled/disabled {
@@ -53,13 +53,6 @@ PRIMARY_PORT="$1"
 SECONDARY_PORT="$2"
 MOD_PAGESPEED_DIR="$3"
 NGINX_EXECUTABLE="$4"
-RCPORT1=9991
-RCPORT2=9992
-RCPORT3=9993
-RCPORT4=9994
-RCPORT5=9995
-RCPORT6=9996
-RCPORT7=9997

 this_dir="$( cd $(dirname "$0") && pwd)"

@@ -70,13 +63,6 @@ function run_test_checking_failure() {
    MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
    NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
    RUN_TESTS="$RUN_TESTS" \
-    RCPORT1="$RCPORT1" \
-    RCPORT2="$RCPORT2" \
-    RCPORT3="$RCPORT3" \
-    RCPORT4="$RCPORT4" \
-    RCPORT5="$RCPORT5" \
-    RCPORT6="$RCPORT6" \
-    RCPORT7="$RCPORT7" \
    bash "$this_dir/nginx_system_test.sh"
  STATUS=$?
  echo "With $@ setup."
Author	SHA1	Message	Date
Jeff Kaufman	8b371c0af1	1.9.32.13 -> 1.9.32.14	2016-03-28 09:10:35 -04:00
Jeff Kaufman	fcbce7acc2	avoid warning	2016-03-28 09:10:05 -04:00
Jeff Kaufman	ad74f4412b	1.9.32.11 -> 1.9.32.13	2016-02-02 13:50:27 -05:00
Jeffrey Crowell	79f98b197e	release version 1.9.32.10 -> 1.9.32.11	2015-12-10 10:05:43 -05:00
Jeffrey Crowell	4103004cee	update config for 1.9.32.10	2015-10-08 17:26:44 -04:00
Jeff Kaufman	0f3f268b36	Revert "Revert "Handle DoneAndSetHeaders() new second argument that wants to know whether the response is complete."" Needed for 1.9.32.6 to build. This reverts commit `19ebf69bad`.	2015-07-30 07:27:19 -04:00
Jeffrey Crowell	30fdd02d0c	Merge pull request #999 from pagespeed/release-1.9.32.6-beta release: version 1.9.32.4 -> 1.9.32.6	2015-07-29 16:07:45 -04:00
Jeffrey Crowell	0572a5bebf	release: version 1.9.32.4 -> 1.9.32.6	2015-07-29 14:33:57 -04:00
Jeff Kaufman	4bd5b7c663	Merge pull request #997 from pagespeed/jefftk-backport-864-fix backport #864 fix for bugfix release	2015-07-28 13:26:45 -04:00
Otto van der Schaaf	a9c292d8dc	server-header: improve handling of server header - Fixes an issue in the html flow that would prevent overriding the value of the 'Server' response-header. - Add tests that ensure we emit a single and correct server header in all flows when not overriding it. - Add tests that ensure overriding the 'Server' response header works. The resource and IPRO flow are added to the expected failures as those are not working yet (and will be adressed in a follow-up). Fixed https://github.com/pagespeed/ngx_pagespeed/issues/864 (html flow)	2015-07-28 11:32:55 -04:00
Jeffrey Crowell	907c57a217	add test servers for sending x-sendfile and x-accel-redirect	2015-07-23 17:36:54 -04:00
Jan-Willem Maessen	a7bef40eac	Fix pagespeed_libraries_generator.sh to fetch pagespeed_libraries.conf from github rather than svn.	2015-07-09 10:19:37 -04:00
Jeffrey Crowell	af5c568a92	Merge remote-tracking branch 'origin/release-1.9.32.4-beta'	2015-06-17 14:15:19 -04:00
Jeffrey Crowell	5cc1ffb352	release: version 1.9.32.3 -> 1.9.32.4	2015-06-17 14:08:47 -04:00
Jeffrey Crowell	d01093ba9b	release: version 1.9.32.3 -> 1.9.32.4	2015-06-17 11:40:05 -04:00
Jeffrey Crowell	a578057e2f	fix tab/space.	2015-06-10 13:01:26 -04:00
Jeffrey Crowell	5bd43dce9a	fix backport of fix for #965 was missing ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers).	2015-06-10 11:44:46 -04:00
Jeffrey Crowell	19ebf69bad	Revert "Handle DoneAndSetHeaders() new second argument that wants to know whether the response is complete." This reverts commit `4df7460d62`.	2015-06-10 11:33:05 -04:00
Jeff Kaufman	4df7460d62	Handle DoneAndSetHeaders() new second argument that wants to know whether the response is complete.	2015-05-15 11:30:04 -04:00
Jeffrey Crowell	c5c443f256	add psol tar.gz to gitignore	2015-05-13 11:56:56 -04:00
Jeffrey Crowell	b9ca5d865d	backport @oschaff fix of #965 fix issue of PageSpeed silently crashing returning partial web pages backported from commit `a5411a1c7c`	2015-05-13 11:48:43 -04:00
Jeffrey Crowell	0cf3f1c6d7	Merge pull request #951 from pagespeed/crowell-build_clang initialize uninitialized variables.	2015-04-01 15:44:56 -04:00
Jeffrey Crowell	7857bab7fb	Merge pull request #949 from pagespeed/crowell-with_threads add support for nginx 1.7.11 --with-threads	2015-04-01 15:09:49 -04:00
Jeffrey Crowell	b11ab687c1	initialize uninitialized variables.	2015-04-01 14:49:31 -04:00
Jeffrey Crowell	a81cc997a9	add support for nginx 1.7.11 --with-threads	2015-04-01 14:11:00 -04:00
Otto van der Schaaf	4c9298446c	Merge pull request #916 from pagespeed/oschaaf-date-header-32-bits-issue date-header: fix 32 bits issue	2015-02-17 15:25:37 +01:00
Otto van der Schaaf	7355f2e207	date-header: fix 32 bits issue Should fix https://github.com/pagespeed/ngx_pagespeed/issues/913	2015-02-16 23:42:54 +01:00
Jeffrey Crowell	9da85bb9d5	Replace CHECK with return failure and log. Fixes #888 along with https://code.google.com/p/modpagespeed/source/detail?r=4533	2015-01-30 10:29:05 -05:00
Otto van der Schaaf	90ac91fe9e	Merge pull request #816 from pagespeed/oschaaf-gzip-vs-etag gzip-vs-etag: Don't rename the ETag when gzip isn't ./configured	2015-01-08 18:00:42 +01:00
Jeffrey Crowell	da466c1487	release: 1.9.32.2 -> 1.9.32.3	2014-12-22 10:10:05 -05:00
Jeff Kaufman	2a409777dc	Merge pull request #859 from We-Amp/keesspoelstra-gzip-init-fix NgxGZipSetter fix, at config reload Init did not work as expected (Issue 844)	2014-12-10 07:44:46 -05:00
keesspoelstra	c3f41512cf	NgxGZipSetter fix, at config reload Init did not work as expected (Issue 844)	2014-12-09 20:47:48 +01:00
Jeff Kaufman	0980633dd2	Merge pull request #858 from vlajos/typofixes-vlajos-20141204 typofixes - https://github.com/vlajos/misspell_fixer	2014-12-05 08:28:55 -05:00
Veres Lajos	c1c83aa69b	typofixes - https://github.com/vlajos/misspell_fixer	2014-12-04 22:56:03 +00:00
Otto van der Schaaf	b037cc2b3e	gzip-vs-etag: Don't rename the ETag when gzip isn't ./configured We should not rename the ETag header to work around the gzip module clearing it, when the gzip module actually isn't built. In that case we will fail to inject the header filter that renames the header back to 'ETag' before it gets send out. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/770	2014-11-14 10:42:49 +01:00
Otto van der Schaaf	4885d44f69	native-fetcher: remove a DCHECK that possibly fires on shutdown.	2014-11-10 13:24:17 -05:00
Otto van der Schaaf	7b84f92adf	Merge pull request #838 from pagespeed/oschaaf-gzip-rollback-message gzip-rollback-message: Demote a log message from info to debug	2014-10-30 09:48:50 +01:00
Jeffrey Crowell	a403e62074	Check type of resolver_ctx->addrs. Check the type of resolver_ctx->addrs and make sure that it is ngx_addr_t* instead of in_addr_t*. addresses issue #839	2014-10-29 15:14:22 -04:00
Jeffrey Crowell	b5dc1083f4	Readd pthread_mutex.h include on ngx_fetch.cc. Messed up the merge of `0290f52a88` originally, add the include back.	2014-10-27 14:51:45 -04:00
Otto van der Schaaf	ab9929e5a5	native fetcher: Support http keep-alive Based on @dinic his work, add keep-alive support for the native fetcher. Adds a new option, usable at the http{} level in configuration: pagespeed NativeFetcherMaxKeepaliveRequests 50; The default value is 100 (aligned to nginx). Setting the value to 1 turns off keep-alive requests altogether). Most notable changes: - Request keep-alive by adding the appropriate request header - Fixes connections getting reused while they are servicing other requests: - Remove connection from the pool of available connections for keepalive when applicable - Disable keepalive in more appropriate situations - Response parsing fixes - Remove connections that timeout from the k.a. pool - Add a few sanity (D)CHECKS - Emit debug messages for traceability - Fix for ignoring ipv6 addresses returned from dns queries when ipv6 is enabled. - Bump the fetch timeout in test configuration to deflake tests that require dns lookups (which will be done via 8.8.8.8 currently for the native fetcher) Conflicts: src/ngx_fetch.cc	2014-10-24 16:23:45 -04:00
Jeffrey Crowell	9832a049fe	release: version 1.9.32.1 -> 1.9.32.2	2014-10-24 15:49:48 -04:00
Otto van der Schaaf	14822570c4	gzip-rollback-message: Demote a log message from info to debug Fixes https://github.com/pagespeed/ngx_pagespeed/issues/832	2014-10-21 05:57:25 +02:00
Otto van der Schaaf	ccd800a0b1	Merge pull request #818 from pagespeed/oschaaf-fix-stdio-logging logging: init logging early, hand ProxyFetchFactory correct server log	2014-10-16 15:58:57 +02:00
Otto van der Schaaf	e2e21474ce	logging: init logging early, hand ProxyFetchFactory correct server log - Prevent logging to stdout/stderr, make sure we log to error.log for early messages during initialization. Note that nginx is still working to setup its logging configuration, so these early messages will go through its defaults. Which means that only warnings or worse will pass for early logging messages. - Make sure we init ProxyFetchFactories's NgxMessageHandler to the correct server{} specific log so it will write to the error_log configured in the server{} block instead of the global error_log. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/808 Helps https://github.com/pagespeed/ngx_pagespeed/issues/817	2014-10-01 11:09:20 +02:00
Otto van der Schaaf	7680a159a7	Merge pull request #814 from pagespeed/oschaaf-gcc-4.1.2-segfault gcc-4.1.2-segfault: Make sure we init all ps_request_ctx_t members	2014-09-26 16:23:30 +02:00
Otto van der Schaaf	7aa5bc4a99	gcc-4.1.2-segfault: Make sure we init all ps_request_ctx_t members When built with gcc-4.1.2, this change prevents segmentation faults caused by undefined behaviour via uninitialized members. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/813	2014-09-26 00:17:11 +02:00
Jeffrey Crowell	462acc9eee	update version number in the config file	2014-09-17 11:10:56 -04:00
Jeffrey Crowell	4678a62be8	Fix config file based on trunk-tracking instead of master	2014-09-17 11:05:55 -04:00
Jeffrey Crowell	525e331c0c	Fix config file Config was improperly merged from master, fixing.	2014-09-17 10:26:11 -04:00
Otto van der Schaaf	dc40c902ec	connection_read_handler: fix potential double free In connection_read_handler(), make sure we act accordingly when r->connection->error is set (indicating the the current request has been finalized). Reproduction of what happens when we don't: enable IPRO+SPDY, and rapidly refresh a page with chrome. These rapid abortions will eventually trigger a segfault/hang/misc bad behaviour.	2014-09-15 10:37:42 -04:00
Otto van der Schaaf	6a8a1c5bf7	IPRO: Fix a race in processing writes() from NgxBaseFetch For FetchInPlaceResource, NgxBaseFetch would send two bytes down its pipe, one upon HeaderComplete() and one upon HandleDone(). We need only one to resume processing on the nginx side. There is a race between ps_connection_read_handler() and processing of the byte send by NgxBaseFetch::HandleDone(). ps_connection_read_handler() clears the pipe when the request is finalized, and also drains it on each event - so two writes could be processed as one when lucky, masking the problem). One concrete problem this solved for me was that SPDY + IPRO + proxy_pass would segfault, hang, and/or pass on 5xx/404 responses from IPRO lookup fetches to the browser, next to alerts about r->count being zero in nginx's error.log Might fix https://github.com/pagespeed/ngx_pagespeed/issues/788 Fixes https://github.com/pagespeed/ngx_pagespeed/issues/792	2014-09-12 10:59:16 -04:00
Jeffrey Crowell	5bcd9e277e	release: version 1.8.31.4 -> 1.9.32.1	2014-09-11 13:47:00 -04:00
Jeffrey Crowell	554577bbf2	Merge branch 'master' into HEAD Preparing for 1.9.32.1 release Conflicts: config src/ngx_pagespeed.cc src/ngx_rewrite_driver_factory.h src/ngx_rewrite_options.cc test/nginx_system_test.sh test/pagespeed_test.conf.template	2014-09-11 13:45:22 -04:00
Jeffrey Crowell	458a2f5236	Merge pull request #789 from gmszone/patch-1 Update config	2014-09-03 09:59:38 -04:00
Phodal Huang	71e89efc64	Update config	2014-09-02 13:49:49 +08:00
Otto van der Schaaf	bbdfb5b429	Merge pull request #744 from pagespeed/oschaaf-double-location location-header: tweak location header handling	2014-07-16 00:50:25 +02:00
Otto van der Schaaf	1f02f368e1	location-header: tweak location header handling - Fix potentially sending the location header into PSOL twice. - Be more thorough when unsetting the location header Attempts to fix https://github.com/pagespeed/ngx_pagespeed/issues/725	2014-06-30 00:54:58 +02:00
Otto van der Schaaf	0c1cbbdf64	Merge pull request #736 from pagespeed/oschaaf-lff-script-support script-variables: Support LoadFromFileXXX	2014-06-28 00:01:03 +02:00
Otto van der Schaaf	75a4481750	script-variables: Support LoadFromFileXXX Add support for script variables in LoadFromFileXXX configuration, which can be enabled by adding "pagespeed ProcessScriptVariables on" in the http{} block. Note that tests currently can't pass because of a failing unrelated test: "start_test PageSpeed CSS loaded in fallback mode is always chunked" I have tested this by commenting that test, but re-enabled the test before making the pull to keep the diff clean. Fixes https://github.com/pagespeed/ngx_pagespeed/issues/549 Docs: https://github.com/pagespeed/ngx_pagespeed/wiki/Script-variables-in-LoadFromFile-configuration	2014-06-27 23:55:34 +02:00
Jeff Kaufman	b1e188bdb6	readme: reference doc on developers.google.com instead of duplicating it. Conflicts: README.md	2014-06-23 10:41:47 -04:00
Jan-Willem Maessen	4d940fb84e	Update for 1.8.31.4	2014-06-17 13:32:51 -04:00
Jan-Willem Maessen	a6a2da765b	Merge remote-tracking branch 'origin/release-1.8.31.3-beta' Pulling in upstream merge to local master.	2014-05-29 15:06:03 -04:00
Jan-Willem Maessen	70038bc64a	Merge pull request #714 from pagespeed/jmaessen-prepare-release-1.8.31.3-beta Jmaessen prepare release 1.8.31.3 beta	2014-05-29 15:01:45 -04:00
Jan-Willem Maessen	815fa29985	Clarify test name and test comment.	2014-05-29 14:51:36 -04:00
Jan-Willem Maessen	c82bb2c7a4	Version 1.8.31.2 -> 1.8.31.3	2014-05-29 14:32:18 -04:00
Jan-Willem Maessen	e9eaa23356	Test fixups because fallback css now always uses chunked encoding in 1.8.31.3	2014-05-29 14:30:03 -04:00
Jeff Kaufman	168df6ae68	readme: with 1.8 we need different default config	2014-05-28 16:34:12 -04:00
Jan-Willem Maessen	a7969a6382	Merge pull request #710 from pagespeed/jefftk-unbackport-roo-test fixup: unbackport the RequestOptionOverride tests from `8246c03f`	2014-05-28 11:17:06 -04:00
Jeff Kaufman	243d4e1931	fixup: unbackport the RequestOptionOverride tests from `8246c03f`	2014-05-28 11:14:14 -04:00
Jeff Kaufman	8246c03fda	options: support StaticAssetPrefix Backporting this to master for 1.8.31.3. Fixes #705.	2014-05-27 11:47:36 -04:00
Jeff Kaufman	99ac026f00	readme: 1.4.6 -> 1.6.0	2014-05-12 10:02:19 -04:00
Jeff Kaufman	90e4c40d86	Allow admin paths to be set at server scope.	2014-05-12 09:35:54 -04:00
Jan-Willem Maessen	0de4e20be1	release: version 1.7.30.4 -> 1.8.31.2	2014-05-09 11:28:04 -04:00