We need -lstdc++ to come after psol.a (#1273)

This is needed because the current GURL implementation there --- and hence
google_url.h needs an extra compat header that's there.

config

@@ -29,8 +29,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.10.33.7.tar.gz"
-    echo "     $ tar -xzvf 1.10.33.7.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.11.33.4.tar.gz"
+    echo "     $ tar -xzvf 1.11.33.4.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -80,10 +80,8 @@ ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
 ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
 if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
   LINK=$ps_maybe_gpp
-  NGX_TEST_LD_OPT="$NGX_TEST_LD_OPT -lstdc++"
-else
-  pagespeed_libs="-lstdc++"
 fi
+pagespeed_libs="-lstdc++"
 
 # The compiler needs to know that __sync_add_and_fetch_4 is ok,
 # and this requires an instruction that didn't exist on i586 or i386.
@@ -159,17 +157,18 @@ pagespeed_include="\
   $mod_pagespeed_dir/third_party/chromium/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/src \
   $mod_pagespeed_dir/third_party/google-sparsehash/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/protobuf/src \
+  $mod_pagespeed_dir/third_party/protobuf/src/src \
   $mod_pagespeed_dir/third_party/re2/src \
   $mod_pagespeed_dir/out/$buildtype/obj/gen \
   $mod_pagespeed_dir/out/$buildtype/obj/gen/protoc_out/instaweb \
   $mod_pagespeed_dir/third_party/apr/src/include \
   $mod_pagespeed_dir/third_party/aprutil/src/include \
   $mod_pagespeed_dir/third_party/apr/gen/arch/$os_name/$arch_name/include \
-  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
+  $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include \
+  $mod_pagespeed_dir/url"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="$pagespeed_libs $psol_binary -lrt -pthread -lm"
+pagespeed_libs="$psol_binary $pagespeed_libs -lrt -pthread -lm"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
 

src/ngx_base_fetch.cc

@@ -12,9 +12,12 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Author: jefftk@google.com (Jeff Kaufman)
  */
 
-// Author: jefftk@google.com (Jeff Kaufman)
+#include "ngx_pagespeed.h"  // Must come first, see comments in CollectHeaders.
+
 #include <unistd.h> //for usleep
 
 #include "ngx_base_fetch.h"
@@ -23,6 +26,8 @@
 
 #include "ngx_pagespeed.h"
 
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "net/instaweb/rewriter/public/rewrite_stats.h"
 #include "pagespeed/kernel/base/google_message_handler.h"
 #include "pagespeed/kernel/base/message_handler.h"
@@ -38,14 +43,18 @@ const char kDone = 'D';
 NgxEventConnection* NgxBaseFetch::event_connection = NULL;
 int NgxBaseFetch::active_base_fetches = 0;
 
-NgxBaseFetch::NgxBaseFetch(ngx_http_request_t* r,
+NgxBaseFetch::NgxBaseFetch(StringPiece url,
+                           ngx_http_request_t* r,
                            NgxServerContext* server_context,
                            const RequestContextPtr& request_ctx,
                            PreserveCachingHeaders preserve_caching_headers,
-                           NgxBaseFetchType base_fetch_type)
+                           NgxBaseFetchType base_fetch_type,
+                           const RewriteOptions* options)
     : AsyncFetch(request_ctx),
+      url_(url.data(), url.size()),
       request_(r),
       server_context_(server_context),
+      options_(options),
       done_called_(false),
       last_buf_sent_(false),
       references_(2),
@@ -249,6 +258,13 @@ ngx_int_t NgxBaseFetch::CollectAccumulatedWrites(ngx_chain_t** link_ptr) {
 }
 
 ngx_int_t NgxBaseFetch::CollectHeaders(ngx_http_headers_out_t* headers_out) {
+  // nginx defines _FILE_OFFSET_BITS to 64, which changes the size of off_t.
+  // If a standard header is accidentally included before the nginx header,
+  // on a 32-bit system off_t will be 4 bytes and we don't assign all the
+  // bits of content_length_n. Sanity check that did not happen.
+  // This could use static_assert, but this file is not built with --std=c++11.
+  bool sanity_check_off_t[sizeof(off_t) == 8 ? 1 : -1] __attribute__ ((unused));
+
   const ResponseHeaders* pagespeed_headers = response_headers();
 
   if (content_length_known()) {
@@ -331,4 +347,9 @@ void NgxBaseFetch::HandleDone(bool success) {
   DecrefAndDeleteIfUnreferenced();
 }
 
+bool NgxBaseFetch::IsCachedResultValid(const ResponseHeaders& headers) {
+  return OptionsAwareHTTPCacheCallback::IsCacheValid(
+      url_, *options_, request_context(), headers);
+}
+
 }  // namespace net_instaweb

src/ngx_base_fetch.h

@@ -43,6 +43,9 @@
 //    events it handles.
 //
 // When the last reference is dropped, this class will delete itself.
+//
+// TODO(jmarantz): consider sharing the cache-invalidation infrastructure
+// with ApacheFetch, using a common base class.
 
 #ifndef NGX_BASE_FETCH_H_
 #define NGX_BASE_FETCH_H_
@@ -59,6 +62,7 @@ extern "C" {
 #include "ngx_server_context.h"
 
 #include "net/instaweb/http/public/async_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
 #include "pagespeed/kernel/base/string.h"
 #include "pagespeed/kernel/http/headers.h"
 
@@ -74,10 +78,12 @@ enum NgxBaseFetchType {
 
 class NgxBaseFetch : public AsyncFetch {
  public:
-  NgxBaseFetch(ngx_http_request_t* r, NgxServerContext* server_context,
+  NgxBaseFetch(StringPiece url, ngx_http_request_t* r,
+               NgxServerContext* server_context,
                const RequestContextPtr& request_ctx,
                PreserveCachingHeaders preserve_caching_headers,
-               NgxBaseFetchType base_fetch_type);
+               NgxBaseFetchType base_fetch_type,
+               const RewriteOptions* options);
   virtual ~NgxBaseFetch();
 
   // Statically initializes event_connection, require for PSOL and nginx to
@@ -125,6 +131,8 @@ class NgxBaseFetch : public AsyncFetch {
   ngx_http_request_t* request() { return request_; }
   NgxBaseFetchType base_fetch_type() { return base_fetch_type_; }
 
+  virtual bool IsCachedResultValid(const ResponseHeaders& headers);
+
  private:
   virtual bool HandleWrite(const StringPiece& sp, MessageHandler* handler);
   virtual bool HandleFlush(MessageHandler* handler);
@@ -156,9 +164,11 @@ class NgxBaseFetch : public AsyncFetch {
   // Live count of NgxBaseFetch instances that are currently in use.
   static int active_base_fetches;
 
+  GoogleString url_;
   ngx_http_request_t* request_;
   GoogleString buffer_;
   NgxServerContext* server_context_;
+  const RewriteOptions* options_;
   bool done_called_;
   bool last_buf_sent_;
   // How many active references there are to this fetch. Starts at two,

src/ngx_fetch.cc

@@ -315,7 +315,9 @@ bool NgxFetch::Init() {
   }
 
   if (!ParseUrl()) {
-    message_handler_->Message(kError, "NgxFetch: ParseUrl() failed");
+    message_handler_->Message(kError, 
+                              "NgxFetch: ParseUrl() failed for [%s]:%s",
+                              str_url_.c_str(), url_.err);
     return false;
   }
 
@@ -938,8 +940,8 @@ void NgxFetch::FixUserAgent() {
     user_agent += "NgxNativeFetcher";
   }
   GoogleString version = StrCat(
-      " ", kModPagespeedSubrequestUserAgent,
-      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+      " (", kModPagespeedSubrequestUserAgent,
+      "/" MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING ")");
   if (!StringPiece(user_agent).ends_with(version)) {
     user_agent += version;
   }

src/ngx_pagespeed.cc

@@ -466,7 +466,8 @@ void copy_response_headers_from_ngx(const ngx_http_request_t* r,
 
   // When we don't have a date header, set one with the current time.
   if (headers->Lookup1(HttpAttributes::kDate) == NULL) {
-    headers->SetDate(ngx_current_msec);
+    PosixTimer timer;
+    headers->SetDate(timer.NowMs());
   }
 
   // TODO(oschaaf): ComputeCaching should be called in setupforhtml()?
@@ -576,6 +577,10 @@ ngx_int_t copy_response_headers_to_ngx(
       continue;
     } else if (STR_EQ_LITERAL(name, "Transfer-Encoding")) {
       continue;
+    } else if (STR_EQ_LITERAL(name, "Vary") && value.len
+        && STR_EQ_LITERAL(value, "Accept-Encoding")) {
+      ps_request_ctx_t* ctx = ps_get_request_context(r);
+      ctx->psol_vary_accept_only = true;
     }
 
     ngx_table_elt_t* header = static_cast<ngx_table_elt_t*>(
@@ -678,6 +683,13 @@ char* ps_main_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_srv_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 char* ps_loc_configure(ngx_conf_t* cf, ngx_command_t* cmd, void* conf);
 
+// We want NGX_CONF_MULTI for some very old versions:
+//   https://github.com/pagespeed/ngx_pagespeed/commit/66f1b9aa
+// but it's gone in recent revisions, so provide a compat #define if needed
+#ifndef NGX_CONF_MULTI
+#define NGX_CONF_MULTI 0
+#endif
+
 // TODO(jud): Verify that all the offsets should be NGX_HTTP_SRV_CONF_OFFSET and
 // not NGX_HTTP_LOC_CONF_OFFSET or NGX_HTTP_MAIN_CONF_OFFSET.
 ngx_command_t ps_commands[] = {
@@ -1082,7 +1094,7 @@ char* ps_merge_srv_conf(ngx_conf_t* cf, void* parent, void* child) {
 
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-  cfg_m->driver_factory->set_main_conf(parent_cfg_s->options);
+  cfg_m->driver_factory->SetMainConf(parent_cfg_s->options);
   cfg_s->server_context = cfg_m->driver_factory->MakeNgxServerContext(
       "dummy_hostname", dummy_port);
   // The server context sets some options when we call global_options(). So
@@ -1270,6 +1282,7 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
   ctx->driver->Cleanup();
   ctx->driver = NULL;
   ctx->location_field_set = false;
+  ctx->psol_vary_accept_only = false;
 
   // re init ctx
   ctx->html_rewrite = true;
@@ -1576,10 +1589,12 @@ void ps_release_base_fetch(ps_request_ctx_t* ctx) {
 }
 
 // TODO(chaizhenhua): merge into NgxBaseFetch ctor
-void ps_create_base_fetch(ps_request_ctx_t* ctx,
-                               RequestContextPtr request_context,
-                               RequestHeaders* request_headers,
-                               NgxBaseFetchType type) {
+void ps_create_base_fetch(StringPiece url,
+                          ps_request_ctx_t* ctx,
+                          RequestContextPtr request_context,
+                          RequestHeaders* request_headers,
+                          NgxBaseFetchType type,
+                          const RewriteOptions* options) {
   CHECK(ctx->base_fetch == NULL) << "Pre-existing base fetch!";
 
   ngx_http_request_t* r = ctx->r;
@@ -1589,9 +1604,9 @@ void ps_create_base_fetch(ps_request_ctx_t* ctx,
   // it, and call Done() on the associated parent (Proxy or Resource) fetch. If
   // we fail before creating the associated fetch then we need to call Done() on
   // the BaseFetch ourselves.
-  ctx->base_fetch = new NgxBaseFetch(r, cfg_s->server_context,
-                                     request_context,
-                                     ctx->preserve_caching_headers, type);
+  ctx->base_fetch = new NgxBaseFetch(url, r, cfg_s->server_context, request_context,
+                                     ctx->preserve_caching_headers, type,
+                                     options);
   ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
 }
 
@@ -1840,6 +1855,7 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     ctx->recorder = NULL;
     ctx->url_string = url_string;
     ctx->location_field_set = false;
+    ctx->psol_vary_accept_only = false;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1855,16 +1871,17 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   if (pagespeed_resource) {
     // TODO(jefftk): Set using_spdy appropriately.  See
     // ProxyInterface::ProxyRequestCallback
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kPageSpeedResource);
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kPageSpeedResource,
+                         options);
     ResourceFetch::Start(
         url,
         custom_options.release() /* null if there aren't custom options */,
         false /* using_spdy */, cfg_s->server_context, ctx->base_fetch);
     return ps_async_wait_response(r);
   } else if (is_an_admin_handler) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kAdminPage);
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kAdminPage, options);
     QueryParams query_params;
     query_params.ParseFromUrl(url);
 
@@ -1912,8 +1929,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
     if (options->domain_lawyer()->MapOriginUrl(
             url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
-      ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                          kPageSpeedProxy);
+      ps_create_base_fetch(url.Spec(), ctx, request_context,
+                           request_headers.release(), kPageSpeedProxy, options);
 
       RewriteDriver* driver;
       if (custom_options.get() == NULL) {
@@ -1937,8 +1954,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   }
 
   if (html_rewrite) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kHtmlTransform);
+    ps_create_base_fetch(url.Spec(), ctx, request_context,
+                         request_headers.release(), kHtmlTransform, options);
     // Do not store driver in request_context, it's not safe.
     RewriteDriver* driver;
 
@@ -1982,8 +1999,8 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
   if (options->in_place_rewriting_enabled() &&
       options->enabled() &&
       options->IsAllowed(url.Spec())) {
-    ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                         kIproLookup);
+    ps_create_base_fetch(url.Spec(), ctx, request_context, request_headers.release(),
+                         kIproLookup, options);
 
     // Do not store driver in request_context, it's not safe.
     RewriteDriver* driver;
@@ -2157,6 +2174,13 @@ ngx_int_t ps_etag_header_filter(ngx_http_request_t* r) {
       break;
     }
   }
+
+  ps_request_ctx_t* ctx = ps_get_request_context(r);
+#if (NGX_HTTP_GZIP)
+  if (ctx && ctx->psol_vary_accept_only) {
+    r->gzip_vary = 0;
+  }
+#endif
   return ngx_http_ef_next_header_filter(r);
 }
 
@@ -2918,7 +2942,7 @@ ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
 ngx_int_t ps_etag_filter_init(ngx_conf_t* cf) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_conf_get_module_main_conf(cf, ngx_pagespeed));
-  if (cfg_m->driver_factory != NULL) {
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
     ngx_http_ef_next_header_filter = ngx_http_top_header_filter;
     ngx_http_top_header_filter = ps_etag_header_filter;
   }
@@ -3000,6 +3024,11 @@ ngx_int_t ps_init_module(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
 
+  // See https://github.com/pagespeed/ngx_pagespeed/issues/1220
+  if (cfg_m == NULL) {
+    return NGX_OK;
+  }
+
   ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_http_core_module));
   ngx_http_core_srv_conf_t** cscfp = static_cast<ngx_http_core_srv_conf_t**>(
@@ -3067,7 +3096,7 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
   NgxBaseFetch::Terminate();
-  if (cfg_m->driver_factory != NULL) {
+  if (cfg_m != NULL && cfg_m->driver_factory != NULL) {
     cfg_m->driver_factory->ShutDown();
   }
 }
@@ -3077,7 +3106,7 @@ void ps_exit_child_process(ngx_cycle_t* cycle) {
 ngx_int_t ps_init_child_process(ngx_cycle_t* cycle) {
   ps_main_conf_t* cfg_m = static_cast<ps_main_conf_t*>(
       ngx_http_cycle_get_module_main_conf(cycle, ngx_pagespeed));
-  if (cfg_m->driver_factory == NULL) {
+  if (cfg_m == NULL || cfg_m->driver_factory == NULL) {
     return NGX_OK;
   }
 

src/ngx_pagespeed.h

@@ -108,6 +108,7 @@ typedef struct {
   // we should mirror that when we write it back. nginx may absolutify
   // Location: headers that start with '/' without regarding X-Forwarded-Proto.
   bool location_field_set;
+  bool psol_vary_accept_only;
 } ps_request_ctx_t;
 
 ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

src/ngx_rewrite_driver_factory.cc

@@ -69,7 +69,6 @@ NgxRewriteDriverFactory::NgxRewriteDriverFactory(
     SystemThreadSystem* system_thread_system, StringPiece hostname, int port)
     : SystemRewriteDriverFactory(process_context, system_thread_system,
         NULL /* default shared memory runtime */, hostname, port),
-      main_conf_(NULL),
       threads_started_(false),
       ngx_message_handler_(
           new NgxMessageHandler(timer(), thread_system()->NewMutex())),
@@ -148,10 +147,19 @@ NamedLockManager* NgxRewriteDriverFactory::DefaultLockManager() {
 
 RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptions() {
   NgxRewriteOptions* options = new NgxRewriteOptions(thread_system());
+  // TODO(jefftk): figure out why using SetDefaultRewriteLevel like
+  // mod_pagespeed does in mod_instaweb.cc:create_dir_config() isn't enough here
+  // -- if you use that instead then ngx_pagespeed doesn't actually end up
+  // defaulting CoreFilters.
+  // See: https://github.com/pagespeed/ngx_pagespeed/issues/1190
   options->SetRewriteLevel(RewriteOptions::kCoreFilters);
   return options;
 }
 
+RewriteOptions* NgxRewriteDriverFactory::NewRewriteOptionsForQuery() {
+  return new NgxRewriteOptions(thread_system());
+}
+
 bool NgxRewriteDriverFactory::CheckResolver() {
   if (use_native_fetcher_ && resolver_ == NULL) {
     return false;
@@ -208,6 +216,14 @@ void NgxRewriteDriverFactory::StartThreads() {
   threads_started_ = true;
 }
 
+void NgxRewriteDriverFactory::SetMainConf(NgxRewriteOptions* main_options) {
+  // Propagate process-scope options from the copy we had during nginx option
+  // parsing to our own.
+  if (main_options != NULL) {
+    default_options()->MergeOnlyProcessScopeOptions(*main_options);
+  }
+}
+
 void NgxRewriteDriverFactory::LoggingInit(
     ngx_log_t* log, bool may_install_crash_handler) {
   log_ = log;

src/ngx_rewrite_driver_factory.h

@@ -63,8 +63,9 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
   virtual Timer* DefaultTimer();
   virtual NamedLockManager* DefaultLockManager();
   // Create a new RewriteOptions.  In this implementation it will be an
-  // NgxRewriteOptions.
+  // NgxRewriteOptions, and it will have CoreFilters explicitly set.
   virtual RewriteOptions* NewRewriteOptions();
+  virtual RewriteOptions* NewRewriteOptionsForQuery();
   virtual ServerContext* NewDecodingServerContext();
   // Check resolver configured or not.
   bool CheckResolver();
@@ -90,7 +91,7 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
     InitStats(statistics);
   }
 
-  void set_main_conf(NgxRewriteOptions* main_conf) {  main_conf_ = main_conf; }
+  void SetMainConf(NgxRewriteOptions* main_conf);
 
   void set_resolver(ngx_resolver_t* resolver) {
     resolver_ = resolver;
@@ -133,10 +134,6 @@ class NgxRewriteDriverFactory : public SystemRewriteDriverFactory {
  private:
   Timer* timer_;
 
-  // main_conf will have only options set in the main block.  It may be NULL,
-  // and we do not take ownership.
-  NgxRewriteOptions* main_conf_;
-
   bool threads_started_;
   NgxMessageHandler* ngx_message_handler_;
   NgxMessageHandler* ngx_html_parse_message_handler_;

test/nginx_system_test.sh

@@ -46,7 +46,23 @@ POSITION_AUX="${POSITION_AUX:-unset}"
 PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
 SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT"
 
-SERVER_ROOT="$MOD_PAGESPEED_DIR/src/install/"
+this_dir="$( cd $(dirname "$0") && pwd)"
+echo this_dir=$this_dir
+TEST_TMP="$this_dir/tmp"
+rm -rf "$TEST_TMP"
+mkdir -p "$TEST_TMP"
+echo TEST_TMP=$TEST_TMP
+
+APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/src/install/"
+SERVER_ROOT="$TEST_TMP/root"
+echo SERVER_ROOT=$SERVER_ROOT
+rm -rf "$SERVER_ROOT"
+mkdir -p "$SERVER_ROOT"
+export APACHE_DOC_ROOT="$SERVER_ROOT"
+
+mkdir -p "$APACHE_DOC_ROOT"
+make -f "$APACHE_DOC_SRC/Makefile.tests" setup_doc_root \
+  INSTALL_DATA_DIR="$APACHE_DOC_SRC"
 
 # We need check and check_not before we source SYSTEM_TEST_FILE that provides
 # them.
@@ -136,8 +152,6 @@ function fire_ab_load() {
   sleep 2
  }
 
-this_dir="$( cd $(dirname "$0") && pwd)"
-
 # stop nginx/valgrind
 killall -s KILL nginx
 # TODO(oschaaf): Fix waiting for valgrind on 32 bits systems.
@@ -145,8 +159,6 @@ killall -s KILL memcheck-amd64-
 while pgrep nginx > /dev/null; do sleep 1; done
 while pgrep memcheck > /dev/null; do sleep 1; done
 
-TEST_TMP="$this_dir/tmp"
-rm -r "$TEST_TMP"
 check_simple mkdir -p "$TEST_TMP"
 PROXY_CACHE="$TEST_TMP/proxycache"
 TMP_PROXY_CACHE="$TEST_TMP/tmpproxycache"
@@ -646,14 +658,14 @@ check touch "$SECONDARY_CACHE/cache.flush"
 check touch "$IPRO_CACHE/cache.flush"
 sleep 1
 
-CACHE_TESTING_DIR="$SERVER_ROOT/mod_pagespeed_test/cache_flush/"
+CACHE_TESTING_DIR="$SERVER_ROOT/cache_flush/"
 CACHE_TESTING_TMPDIR="$CACHE_TESTING_DIR/$$"
 mkdir "$CACHE_TESTING_TMPDIR"
 cp "$CACHE_TESTING_DIR/cache_flush_test.html" "$CACHE_TESTING_TMPDIR/"
 CSS_FILE="$CACHE_TESTING_TMPDIR/update.css"
 echo ".class myclass { color: $COLOR0; }" > "$CSS_FILE"
 
-URL_PATH="mod_pagespeed_test/cache_flush/$$/cache_flush_test.html"
+URL_PATH="cache_flush/$$/cache_flush_test.html"
 
 URL="$SECONDARY_HOSTNAME/$URL_PATH"
 CACHE_A="--header=Host:cache_a.example.com"
@@ -1249,6 +1261,28 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1) || tr
 # We ignored the exit code, check if we got a 404 response.
 check_from "$OUT" fgrep -qi '404'
 
+if false; then
+# This test fails on 1.11, but passes on trunk.  While 2b4c097 fixed most users,
+# it's still possible to get two Vary: Accept-Encoding headers in this test, so
+# disable it for now.
+start_test Single Vary: Accept-Encoding header in IPRO flow
+URL=http://psol-vary.example.com/mod_pagespeed_example/styles/index_style.css
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+# First hit will be recorded and passed on untouched
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+# Fetch until we get a fully optimized response
+http_proxy=$SECONDARY_HOSTNAME \
+  fetch_until $URL "fgrep -c W/\"PSA" 1 --save-headers
+
+# Test the optimized response.
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP -O /dev/null -S $URL 2>&1)
+MATCHES=$(echo "$OUT" | grep -c "Vary: Accept-Encoding") || true
+check [ $MATCHES -eq 1 ]
+
+fi
+
 start_test Shutting down.
 
 # Fire up some heavy load if ab is available to test a stressed shutdown

test/pagespeed_test.conf.template

@@ -564,6 +564,10 @@ http {
     pagespeed UrlValuedAttribute img alt-src Image;
     pagespeed UrlValuedAttribute video alt-a Image;
     pagespeed UrlValuedAttribute video alt-b Image;
+    pagespeed UrlValuedAttribute link data-stylesheet Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-a Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-b Stylesheet;
+    pagespeed UrlValuedAttribute span data-stylesheet-c Stylesheet;
 
     # Also test that we can redefine spec-defined attributes.
     pagespeed UrlValuedAttribute blockquote cite Image;
@@ -703,7 +707,44 @@ http {
     pagespeed EnableFilters rewrite_images,rewrite_css;
     pagespeed EnableFilters convert_to_webp_lossless;
     pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed WebpRecompressionQuality 70;
     pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "Accept";
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-auto.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,rewrite_css;
+    pagespeed EnableFilters convert_to_webp_animated;
+    pagespeed EnableFilters convert_to_webp_lossless;
+    pagespeed EnableFilters in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    # pagespeed AllowVaryOn "Accept";  # Default is "Auto".
+    pagespeed ImageRecompressionQuality 90;
+    pagespeed JpegRecompressionQuality 75;
+    pagespeed JpegRecompressionQualityForSmallScreens 55;
+    pagespeed JpegQualityForSaveData 35;
+    pagespeed WebpRecompressionQuality 70;
+    pagespeed WebpRecompressionQualityForSmallScreens 50;
+    pagespeed WebpQualityForSaveData 30;
+    pagespeed WebpAnimatedRecompressionQuality 60;
+    pagespeed FileCachePath "@@IPRO_CACHE@@";
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name ipro-for-browser-vary-on-none.example.com;
+    root "@@SERVER_ROOT@@/mod_pagespeed_example";
+    pagespeed EnableFilters rewrite_images,in_place_optimize_for_browser;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed AllowVaryOn "None";
+    pagespeed ImageRecompressionQuality 75;
     pagespeed FileCachePath "@@IPRO_CACHE@@";
   }
 
@@ -771,7 +812,7 @@ http {
     server_name www.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -785,7 +826,7 @@ http {
     server_name origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -799,7 +840,7 @@ http {
     server_name cdn.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
 
-    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@;
+    pagespeed LoadFromFile http://cdn.example.com @@SERVER_ROOT@@/;
     pagespeed MapRewriteDomain cdn.example.com origin.example.com;
     pagespeed RewriteLevel PassThrough;
     pagespeed EnableFilters rewrite_css,rewrite_images;
@@ -1229,6 +1270,44 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name compressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 9;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name uncompressedcache.example.com;
+    pagespeed on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed EnableFilters rewrite_css;
+    pagespeed HttpCacheCompressionLevel 0;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -1291,7 +1370,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name proxy-post-origin.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
   }
 
   server {
@@ -1309,7 +1388,7 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name script-filters.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     set $filters "";
     set $domain_shards "cdn1.example.com,cdn2.example.com";
     if ($http_X_Script) {
@@ -1331,7 +1410,7 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_purge";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
@@ -1347,7 +1426,7 @@ http {
     pagespeed EnableCachePurge on;
 
     pagespeed PurgeMethod PURGE;
-    root "@@SERVER_ROOT@@/mod_pagespeed_test/purge";
+    root "@@SERVER_ROOT@@/purge";
     pagespeed FileCachePath "@@FILE_CACHE@@_dir_on";
     pagespeed DisableFilters add_instrumentation;
     pagespeed RewriteLevel PassThrough;
@@ -1365,7 +1444,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location /mod_pagespeed_test/nostore {
       add_header "Cache-Control" "max-age=12345";
       add_header "Cache-Control" "public, no-store";
@@ -1380,7 +1459,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location / {
       add_header "PageSpeedFilters" "add_instrumentation";
     }
@@ -1393,7 +1472,7 @@ http {
     pagespeed InPlaceResourceOptimization off;
     pagespeed off;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    root "@@SERVER_ROOT@@";
+    root "@@SERVER_ROOT@@/";
     location / {
       add_header "PageSpeed" "off";
     }
@@ -1471,7 +1550,24 @@ http {
     pagespeed GlobalAdminDomains
      Allow everything-explicitly-allowed.example.com;
   }
+  server {
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name psol-vary.example.com;
+    pagespeed on;
+    pagespeed InPlaceResourceOptimization on;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
 
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name debug-filters.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed EnableFilters debug;
+  }
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;
@@ -1710,7 +1806,7 @@ http {
     # optimized.
     location /mod_pagespeed_test/auth/ {
        auth_basic "Restricted";
-       auth_basic_user_file "@@SERVER_ROOT@@mod_pagespeed_test/auth/passwd.conf";
+       auth_basic_user_file "@@SERVER_ROOT@@/mod_pagespeed_test/auth/passwd.conf";
     }
 
     location /mod_pagespeed_test/ipro/cookie/ {
@@ -1756,6 +1852,25 @@ http {
           https://www.gstatic.com/psa/static;
     }
 
+    location /mod_pagespeed_test/strip_subresource_hints/default/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel CoreFilters;
+      pagespeed DisAllow *dontrewriteme*;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/default_passthrough/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed RewriteLevel PassThrough;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_on/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints on;
+      pagespeed RewriteLevel CoreFilters;
+    }
+    location /mod_pagespeed_test/strip_subresource_hints/preserve_off/ {
+      pagespeed DisableFilters add_instrumentation;
+      pagespeed PreserveSubresourceHints off;
+      pagespeed RewriteLevel CoreFilters;
+    }
     # $host implicitly tests script variable support. I'd love to test it more
     # directly, but so far this is the best I've come up with and duplicating
     # the test doesn't seem to make sense.
@@ -1763,6 +1878,11 @@ http {
         "http://$host:@@PRIMARY_PORT@@/mod_pagespeed_test/ipro/instant/"
         "@@SERVER_ROOT@@/mod_pagespeed_test/ipro/instant/";
 
+    location /mod_pagespeed_test/public/ {
+      add_header "Cache-Control" "public, max-age=600";
+      pagespeed PreserveUrlRelativity off;
+    }
+
     pagespeed EnableFilters remove_comments;
 
     # Test LoadFromFile mapping by mapping one dir to another.

test/run_tests.sh

@@ -23,9 +23,11 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir pagespeed_test_host
+#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir ngx_binary
+#   pagespeed_test_host
 # Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed www.modpagespeed.com
+#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed /path/to/nginx/binary
+#   selfsigned.modpagespeed.com
 #
 
 # Normally we test only with the native fetcher off.  Set