Fix pagespeed_libraries_generator.sh to fetch pagespeed_libraries.conf from github rather than svn.

config

@@ -29,8 +29,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.10.33.0.tar.gz"
-    echo "     $ tar -xzvf 1.10.33.0.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.1.tar.gz"
+    echo "     $ tar -xzvf 1.9.32.1.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"
@@ -71,20 +71,6 @@ else
   buildtype=Release
 fi
 
-# If the compiler is gcc, we want to use g++ to link, if at all possible,
-# so that -static-libstdc++ works.
-# Annoyingly, the feature test doesn't even use $LINK for linking, so that
-# needs an explicit -lstdc++
-pagespeed_libs=
-ps_maybe_gpp_base=`basename $CC| sed s/gcc/g++/`
-ps_maybe_gpp="`dirname $CC`/$ps_maybe_gpp_base"
-if [ -n "$NGX_GCC_VER" -a \( -x "$ps_maybe_gpp" \) ]; then
-  LINK=$ps_maybe_gpp
-  NGX_TEST_LD_OPT="$NGX_TEST_LD_OPT -lstdc++"
-else
-  pagespeed_libs="-lstdc++"
-fi
-
 # The compiler needs to know that __sync_add_and_fetch_4 is ok,
 # and this requires an instruction that didn't exist on i586 or i386.
 if [ "$uname_arch" = "i686" ]; then
@@ -93,13 +79,6 @@ fi
 
 CFLAGS="$CFLAGS $FLAG_MARCH"
 
-# For now, standardize on gcc-4.x ABI --- if we don't set this, people building
-# with new gcc defaulting to gcc-5 C++11 ABI will have build trouble linking
-# to our libpsol.a
-# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
-CFLAGS="$CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
-CC_TEST_FLAGS="$CC_TEST_FLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
-
 case "$NGX_GCC_VER" in
   4.8*)
     # On GCC 4.8 and above, -Wall enables -Wunused-local-typedefs.  This breaks
@@ -162,10 +141,9 @@ pagespeed_include="\
   $mod_pagespeed_dir/third_party/aprutil/gen/arch/$os_name/$arch_name/include"
 ngx_feature_path="$pagespeed_include"
 
-pagespeed_libs="$pagespeed_libs $psol_binary -lrt -pthread -lm"
+pagespeed_libs="-lstdc++ $psol_binary -lrt -pthread -lm"
 ngx_feature_libs="$pagespeed_libs"
 ngx_feature_test="
-
   GoogleString output_buffer;
   net_instaweb::StringWriter write_to_string(&output_buffer);
 
@@ -218,29 +196,18 @@ if [ $ngx_found = yes ]; then
     $ps_src/ngx_rewrite_options.cc \
     $ps_src/ngx_server_context.cc \
     $ps_src/ngx_url_async_fetcher.cc"
-  # Save our sources in a separate var since we may need it in config.make
-  PS_NGX_SRCS="$NGX_ADDON_SRCS"
 
   if [ "$position_aux" = "true" ] ; then
      HTTP_AUX_FILTER_MODULES="$HTTP_AUX_FILTER_MODULES $ngx_addon_name"
   else
-    # Make pagespeed run immediately before gzip and Brotli.
-    if echo $HTTP_FILTER_MODULES | grep ngx_http_brotli_filter_module >/dev/null; then
-      module=ngx_http_brotli_filter_module
-    elif [ $HTTP_GZIP = YES ]; then
-      module=$HTTP_GZIP_FILTER_MODULE
-    else
-      module=$HTTP_RANGE_HEADER_FILTER_MODULE
-    fi
-
+    # Make pagespeed run immediately before gzip.
     HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-      sed "s/$module/$module $ngx_addon_name/")
+      sed "s/$HTTP_GZIP_FILTER_MODULE/$HTTP_GZIP_FILTER_MODULE $ngx_addon_name/")
   fi
 
-  # Make the etag header filter run immediately before range header filter.
+  # Make the etag header filter run immediately after gzip.
   HTTP_FILTER_MODULES=$(echo $HTTP_FILTER_MODULES |\
-    sed "s/$HTTP_RANGE_HEADER_FILTER_MODULE/$HTTP_RANGE_HEADER_FILTER_MODULE ngx_pagespeed_etag_filter/")
-
+    sed "s/$HTTP_GZIP_FILTER_MODULE/ngx_pagespeed_etag_filter $HTTP_GZIP_FILTER_MODULE/")
   CORE_LIBS="$CORE_LIBS $pagespeed_libs"
   CORE_INCS="$CORE_INCS $pagespeed_include"
   echo "List of modules (in reverse order of applicability): "$HTTP_FILTER_MODULES
@@ -252,46 +219,4 @@ END
   exit 1
 fi
 
-# Test whether the compiler is compatible
-ngx_feature="psol-compiler-compat"
-ngx_feature_name=""
-ngx_feature_run=no
-ngx_feature_incs=""
-ngx_feature_path=""
-ngx_feature_libs="-lstdc++"
-ngx_feature_test="
-
-#if defined(__clang__) && defined(__GLIBCXX__)
-   // See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning 
-   // for a list of various values of __GLIBCXX__. Note that they're not monotonic
-   // with respect to version numbers.
-   #if __GLIBCXX__ == 20120322  || __GLIBCXX__ == 20120614
-      #error \"clang is using libstdc++ 4.7.0 or 4.7.1, which can cause binary incompatibility.\"
-   #endif
-#endif
-
-#if !defined(__clang__) && defined(__GNUC__)
-   #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
-      #error \"GCC < 4.8 no longer supported. Please use gcc >= 4.8 or clang >= 3.3\"
-   #endif
-#endif
-
-#if defined(__clang__)
-   #if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3)
-      #error \"Please use gcc >= 4.8 or clang >= 3.3\"
-   #endif
-#endif
-"
-
-. "$ngx_addon_dir/cpp_feature"
-
-if [ $ngx_found = no ]; then
-  cat << END
-$0: error: module ngx_pagespeed requires gcc >= 4.8 or clang >= 3.3.
-See https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source for some recommendations.
-Look in objs/autoconf.err for more details.
-END
-  exit 1
-fi
-
 have=NGX_PAGESPEED . auto/have

config.make

@@ -1,7 +0,0 @@
-if [ -n "$NGX_CLANG_VER" ]; then
-  # Chromium headers assume clang is always in C++11 mode. Oblige it.
-  for ps_src_file in $PS_NGX_SRCS; do
-    ps_obj_file="$NGX_OBJS/addon/src/`basename $ps_src_file .cc`.o"
-    echo "$ps_obj_file : CFLAGS += --std=c++11" >> $NGX_MAKEFILE
-  done
-fi

scripts/pagespeed_libraries_generator.sh

@@ -15,7 +15,7 @@
 
 URL="https://github.com/pagespeed/mod_pagespeed/raw/master/"
 URL+="net/instaweb/genfiles/conf/pagespeed_libraries.conf"
-curl -L -s -S "$URL" \
+curl -L -s "$URL" \
     | grep ModPagespeedLibrary \
     | while read library size hash url ; do
   echo "  pagespeed Library $size $hash $url;"

src/log_message_handler.cc

@@ -38,7 +38,7 @@
 
 namespace {
 
-ngx_log_t* ngx_log = NULL;
+ngx_log_t* log = NULL;
 
 ngx_uint_t GetNgxLogLevel(int severity) {
   switch (severity) {
@@ -78,7 +78,7 @@ bool LogMessageHandler(int severity, const char* file, int line,
     message.resize(last_msg_character_index);
   }
 
-  ngx_log_error(this_log_level, ngx_log, 0, "[ngx_pagespeed %s] %s",
+  ngx_log_error(this_log_level, log, 0, "[ngx_pagespeed %s] %s",
                 net_instaweb::kModPagespeedVersion,
                 message.c_str());
 
@@ -99,12 +99,12 @@ namespace log_message_handler {
 
 
 void Install(ngx_log_t* log_in) {
-  ngx_log = log_in;
+  log = log_in;
   logging::SetLogMessageHandler(&LogMessageHandler);
 
   // All VLOG(2) and higher will be displayed as DEBUG logs if the nginx log
   // level is DEBUG.
-  if (ngx_log->log_level >= NGX_LOG_DEBUG) {
+  if (log->log_level >= NGX_LOG_DEBUG) {
     logging::SetMinLogLevel(-2);
   }
 }

src/ngx_base_fetch.cc

@@ -111,8 +111,6 @@ const char* BaseFetchTypeToCStr(NgxBaseFetchType type) {
       return "admin page";
     case kIproLookup:
       return "ipro lookup";
-    case kPageSpeedProxy:
-      return "pagespeed proxy";
   }
   CHECK(false);
   return "can't get here";

src/ngx_base_fetch.h

@@ -68,8 +68,7 @@ enum NgxBaseFetchType {
   kIproLookup,
   kHtmlTransform,
   kPageSpeedResource,
-  kAdminPage,
-  kPageSpeedProxy
+  kAdminPage
 };
 
 class NgxBaseFetch : public AsyncFetch {

src/ngx_pagespeed.cc

@@ -599,10 +599,7 @@ ngx_int_t copy_response_headers_to_ngx(
     } else if (STR_EQ_LITERAL(name, "Last-Modified")) {
       headers_out->last_modified = header;
     } else if (STR_EQ_LITERAL(name, "Location")) {
-      ps_request_ctx_t* ctx = ps_get_request_context(r);
-      if (ctx->location_field_set) {
-        headers_out->location = header;
-      }
+      headers_out->location = header;
     } else if (STR_EQ_LITERAL(name, "Server")) {
       headers_out->server = header;
     } else if (STR_EQ_LITERAL(name, "Content-Length")) {
@@ -610,16 +607,6 @@ ngx_int_t copy_response_headers_to_ngx(
       CHECK(pagespeed_headers.FindContentLength(&len));
       headers_out->content_length_n = len;
       headers_out->content_length = header;
-    } else if (STR_EQ_LITERAL(name, "Content-Encoding")) {
-      headers_out->content_encoding = header;
-    } else if (STR_EQ_LITERAL(name, "Refresh")) {
-      headers_out->refresh = header;
-    } else if (STR_EQ_LITERAL(name, "Content-Range")) {
-      headers_out->content_range = header;
-    } else if (STR_EQ_LITERAL(name, "Accept-Ranges")) {
-      headers_out->accept_ranges = header;
-    } else if (STR_EQ_LITERAL(name, "WWW-Authenticate")) {
-      headers_out->www_authenticate = header;
     }
   }
 
@@ -1267,7 +1254,6 @@ ngx_int_t ps_decline_request(ngx_http_request_t* r) {
 
   ctx->driver->Cleanup();
   ctx->driver = NULL;
-  ctx->location_field_set = false;
 
   // re init ctx
   ctx->html_rewrite = true;
@@ -1664,29 +1650,23 @@ RequestRouting::Response ps_route_request(ngx_http_request_t* r) {
   const NgxRewriteOptions* global_options = cfg_s->server_context->config();
 
   StringPiece path = url.PathSansQuery();
-  if (StringCaseEqual(path, global_options->statistics_path()) &&
-      global_options->StatisticsAccessAllowed(url)) {
+  if (StringCaseEqual(path, global_options->statistics_path())) {
     return RequestRouting::kStatistics;
-  } else if (StringCaseEqual(path, global_options->global_statistics_path()) &&
-             global_options->GlobalStatisticsAccessAllowed(url)) {
+  } else if (StringCaseEqual(path, global_options->global_statistics_path())) {
     return RequestRouting::kGlobalStatistics;
-  } else if (StringCaseEqual(path, global_options->console_path()) &&
-             global_options->ConsoleAccessAllowed(url)) {
+  } else if (StringCaseEqual(path, global_options->console_path())) {
     return RequestRouting::kConsole;
-  } else if (StringCaseEqual(path, global_options->messages_path()) &&
-             global_options->MessagesAccessAllowed(url)) {
+  } else if (StringCaseEqual(path, global_options->messages_path())) {
     return RequestRouting::kMessages;
   } else if (
       // The admin handlers get everything under a path (/path/*) while all the
       // other handlers only get exact matches (/path).  So match all paths
       // starting with the handler path.
       !global_options->admin_path().empty() &&
-      StringCaseStartsWith(path, global_options->admin_path()) &&
-      global_options->AdminAccessAllowed(url)) {
+      StringCaseStartsWith(path, global_options->admin_path())) {
     return RequestRouting::kAdmin;
   } else if (!global_options->global_admin_path().empty() &&
-             StringCaseStartsWith(path, global_options->global_admin_path()) &&
-             global_options->GlobalAdminAccessAllowed(url)) {
+             StringCaseStartsWith(path, global_options->global_admin_path())) {
     return RequestRouting::kGlobalAdmin;
   } else if (global_options->enable_cache_purge() &&
              !global_options->purge_method().empty() &&
@@ -1837,7 +1817,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
 
     ctx->recorder = NULL;
     ctx->url_string = url_string;
-    ctx->location_field_set = false;
 
     // Set up a cleanup handler on the request.
     ngx_http_cleanup_t* cleanup = ngx_http_cleanup_add(r, 0);
@@ -1903,35 +1882,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
     }
 
     return ps_async_wait_response(r);
-  } else if (!html_rewrite && response_category == RequestRouting::kResource) {
-    bool is_proxy = false;
-    GoogleString mapped_url;
-    GoogleString host_header;
-
-    if (options->domain_lawyer()->MapOriginUrl(
-            url, &mapped_url, &host_header, &is_proxy) && is_proxy) {
-      ps_create_base_fetch(ctx, request_context, request_headers.release(),
-                          kPageSpeedProxy);
-
-      RewriteDriver* driver;
-      if (custom_options.get() == NULL) {
-        driver = cfg_s->server_context->NewRewriteDriver(
-            ctx->base_fetch->request_context());
-      } else {
-        driver = cfg_s->server_context->NewCustomRewriteDriver(
-            custom_options.release(), ctx->base_fetch->request_context());
-      }
-
-      driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
-      driver->set_pagespeed_query_params(pagespeed_query_params);
-      driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
-      cfg_s->proxy_fetch_factory->StartNewProxyFetch(
-          mapped_url, ctx->base_fetch, driver, NULL /*property_callback*/,
-          NULL /*original_content_fetch*/);
-
-      return ps_async_wait_response(r);
-    }
-
   }
 
   if (html_rewrite) {
@@ -1953,6 +1903,11 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
           custom_options.release(), ctx->base_fetch->request_context());
     }
 
+    StringPiece user_agent = ctx->base_fetch->request_headers()->Lookup1(
+        HttpAttributes::kUserAgent);
+    if (!user_agent.empty()) {
+      driver->SetUserAgent(user_agent);
+    }
     driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
     driver->set_pagespeed_query_params(pagespeed_query_params);
     driver->set_pagespeed_option_cookies(pagespeed_option_cookies);
@@ -1973,7 +1928,6 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
         url_string, ctx->base_fetch, driver,
         property_callback,
         NULL /* original_content_fetch */);
-    ctx->proxy_fetch->set_trusted_input(true);
     return NGX_OK;
   }
 
@@ -1994,7 +1948,13 @@ ngx_int_t ps_resource_handler(ngx_http_request_t* r,
           custom_options.release(), ctx->base_fetch->request_context());
     }
 
+    StringPiece user_agent = ctx->base_fetch->request_headers()->Lookup1(
+        HttpAttributes::kUserAgent);
+    if (!user_agent.empty()) {
+      driver->SetUserAgent(user_agent);
+    }
     driver->SetRequestHeaders(*ctx->base_fetch->request_headers());
+
     ctx->driver = driver;
 
     cfg_s->server_context->message_handler()->Message(
@@ -2238,8 +2198,6 @@ ngx_int_t ps_html_rewrite_header_filter(ngx_http_request_t* r) {
   }
 
   ps_strip_html_headers(r);
-  // See https://github.com/pagespeed/ngx_pagespeed/issues/819
-  ctx->location_field_set = r->headers_out.location != NULL;
 
   // TODO(jefftk): is this thread safe?
   copy_response_headers_from_ngx(r, ctx->base_fetch->response_headers());

src/ngx_pagespeed.h

@@ -103,11 +103,6 @@ typedef struct {
   // We need to remember the URL here as well since we may modify what NGX
   // gets by stripping our special query params and honoring X-Forwarded-Proto.
   GoogleString url_string;
-
-  // We need to remember if the upstream had headers_out->location set, because
-  // we should mirror that when we write it back. nginx may absolutify
-  // Location: headers that start with '/' without regarding X-Forwarded-Proto.
-  bool location_field_set;
 } ps_request_ctx_t;
 
 ps_request_ctx_t* ps_get_request_context(ngx_http_request_t* r);

src/ngx_rewrite_options.cc

@@ -280,12 +280,7 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
   // as browsers might be able to manipulate its natural use-case: $http_host.
   if (!StringCaseStartsWith(directive, "LoadFromFile") &&
       !StringCaseEqual(directive, "EnableFilters") &&
-      !StringCaseEqual(directive, "DisableFilters") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeLocationPrefix") &&
-      !StringCaseEqual(directive, "DownstreamCachePurgeMethod") &&
-      !StringCaseEqual(directive,
-                       "DownstreamCacheRewrittenPercentageThreshold") &&
-      !StringCaseEqual(directive, "ShardDomain")){
+      !StringCaseEqual(directive, "DisableFilters")) {
     compile_scripts = false;
   }
 

src/ngx_server_context.h

@@ -40,9 +40,8 @@ class NgxServerContext : public SystemServerContext {
       NgxRewriteDriverFactory* factory, StringPiece hostname, int port);
   virtual ~NgxServerContext();
 
-  // We don't allow ProxyFetch to fetch HTML via MapProxyDomain. We will call
-  // set_trusted_input() on any ProxyFetches we use to transform internal HTML.
-  virtual bool ProxiesHtml() const { return false; }
+  // We expect to use ProxyFetch with HTML.
+  virtual bool ProxiesHtml() const { return true; }
 
   // Call only when you need an NgxRewriteOptions.  If you don't need
   // nginx-specific behavior, call global_options() instead which doesn't

test/nginx_system_test.sh

@@ -40,7 +40,6 @@
 : ${RCPORT7:?"Set RCPORT7"}
 : ${MOD_PAGESPEED_DIR:?"Set MOD_PAGESPEED_DIR"}
 : ${NGINX_EXECUTABLE:?"Set NGINX_EXECUTABLE"}
-: ${PAGESPEED_TEST_HOST:?"Set PAGESPEED_TEST_HOST"}
 POSITION_AUX="${POSITION_AUX:-unset}"
 
 PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
@@ -209,7 +208,6 @@ cat $PAGESPEED_CONF_TEMPLATE \
   | sed 's#@@RCPORT5@@#'"$RCPORT5"'#' \
   | sed 's#@@RCPORT6@@#'"$RCPORT6"'#' \
   | sed 's#@@RCPORT7@@#'"$RCPORT7"'#' \
-  | sed 's#@@PAGESPEED_TEST_HOST@@#'"$PAGESPEED_TEST_HOST"'#' \
   >> $PAGESPEED_CONF
 # make sure we substituted all the variables
 check_not_simple grep @@ $PAGESPEED_CONF
@@ -300,16 +298,11 @@ fi
 
 PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
 BEACON_HANDLER="ngx_pagespeed_beacon"
-STATISTICS_HANDLER="ngx_pagespeed_statistics"
-GLOBAL_STATISTICS_HANDLER="ngx_pagespeed_global_statistics"
-MESSAGES_HANDLER="ngx_pagespeed_message"
 STATISTICS_URL=http://$PRIMARY_HOSTNAME/ngx_pagespeed_statistics
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
 ~Cache purging with PageSpeed off in vhost, but on in directory.~
-~2-pass ipro with long ModPagespeedInPlaceRewriteDeadline~
-~3-pass ipro with short ModPagespeedInPlaceRewriteDeadline~
 "
 
 if [ "$POSITION_AUX" = "true" ] ; then
@@ -324,8 +317,8 @@ fi
 #
 # TODO(sligicki): When the prioritize critical css race condition is fixed, the
 # two prioritize_critical_css tests no longer need to be listed here.
-# TODO(oschaaf): Now that we wait after we send a SIGHUP for the new worker
-# process to handle requests, check if we can remove more from the expected
+# TODO(oschaaf): Now that we wait after we send a SIGHUP for the new worker 
+# process to handle requests, check if we can remove more from the expected 
 # failures here under valgrind.
 if $USE_VALGRIND; then
     PAGESPEED_EXPECTED_FAILURES+="
@@ -516,17 +509,6 @@ check $WGET_DUMP -O $FETCHED $HEADERS $URL
 # When enabled, we respect X-Forwarded-Proto and thus list base as https.
 check fgrep -q '<base href="https://' $FETCHED
 
-start_test Relative redirects starting with a forward slash survive.
-URL=http://xfp.example.com/redirect
-# wget seems a bit hairy here, when it comes to handling (relative) redirects.
-# I could not get this test going with wget, and that is why curl is used here.
-# TODO(oschaaf): debug wget some more and swap out curl here.
-OUT=$(curl -v --proxy $SECONDARY_HOSTNAME $URL 2>&1)
-check_from "$OUT" egrep -q '301 Moved Permanently'
-# The important part is that we don't end up with an absolute location here.
-check_from "$OUT" grep -q 'Location: /mod_pagespeed_example'
-check_not_from "$OUT" grep -q 'Location: http'
-
 # Test that loopback route fetcher works with vhosts not listening on
 # 127.0.0.1
 start_test IP choice for loopback fetches.
@@ -579,7 +561,7 @@ check test $(scrape_stat image_rewrite_total_original_bytes) -ge 10000
 start_test "Reload config"
 
 # Fire up some heavy load if ab is available to test a stressed reload.
-# TODO(oschaaf): make sure we wait for the new worker to get ready to accept
+# TODO(oschaaf): make sure we wait for the new worker to get ready to accept 
 # requests.
 fire_ab_load
 
@@ -1166,19 +1148,6 @@ HEADERS="--header=Host:script-filters.example.com"
 OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
 check_not_from "$OUT" fgrep -qi 'addInstrumentationInit'
 
-start_test Test that we can modify domain sharding via script variables.
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
-HEADERS="--header=Host:script-filters.example.com"
-OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
-check_from "$OUT" fgrep "http://cdn1.example.com"
-check_from "$OUT" fgrep "http://cdn2.example.com"
-
-URL="http://$SECONDARY_HOSTNAME/mod_pagespeed_example/rewrite_images.html"
-HEADERS="--header=Host:script-filters.example.com --header=X-Script:1"
-OUT=$($WGET_DUMP -S $HEADERS $URL 2>&1)
-check_not_from "$OUT" fgrep "http://cdn1.example.com"
-check_not_from "$OUT" fgrep "http://cdn2.example.com"
-
 if [ "$NATIVE_FETCHER" != "on" ]; then
   start_test Test that we can rewrite an HTTPS resource.
   fetch_until $TEST_ROOT/https_fetch/https_fetch.html \
@@ -1310,9 +1279,6 @@ OUT=$(cat "$ERROR_LOG" \
     | grep -v "\\[error\\].*Could not create directories*" \
     | grep -v "\\[error\\].*opening temp file: No such file or directory.*" \
     | grep -v "\\[error\\].*remote\.cfg.*" \
-    | grep -v "\\[error\\].*Slow read operation on file.*" \
-    | grep -v "\\[error\\].*Slow ReadFile operation on file.*" \
-    | grep -v "\\[error\\].*Slow write operation on file.*" \
     | grep -v "\\[warn\\].*remote\.cfg.*" \
     | grep -v "\\[warn\\].*end token not received.*" \
     | grep -v "\\[warn\\].*failed to hook next event.*" \

test/pagespeed_test.conf.template

@@ -24,9 +24,6 @@ http {
                    '"$http_user_agent"';
   access_log "@@ACCESS_LOG@@" cache;
 
-  # Don't put entries in the error log for 403s and 404s.
-  log_not_found off;
-
   proxy_cache_path "@@PROXY_CACHE@@"  levels=1:2   keys_zone=htmlcache:60m inactive=90m  max_size=50m;
   proxy_temp_path "@@TMP_PROXY_CACHE@@";
 
@@ -286,31 +283,12 @@ http {
 
     pagespeed RunExperiment on;
     pagespeed AnalyticsID "123-45-6734";
-    pagespeed UseAnalyticsJs false;
     pagespeed ExperimentVariable 2;
     pagespeed ExperimentSpec
       "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50";
     pagespeed ExperimentSpec "id=2;enable=recompress_images;percent=50";
     pagespeed ExperimentSpec "id=3;default;percent=0";
   }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name contentexperiment.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed InPlaceResourceOptimization off;
-
-    pagespeed RunExperiment on;
-    pagespeed AnalyticsID "123-45-6734";
-    pagespeed UseAnalyticsJs false;
-    pagespeed ExperimentVariable 2;
-    pagespeed ExperimentSpec
-      "id=7;enable=recompress_images;disable=convert_jpeg_to_progressive;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=111";
-    pagespeed ExperimentSpec
-      "id=2;enable=recompress_images;percent=50;options=ContentExperimentID=ID:H1BpS2TCRFmJzZgjwyeBHQ,ContentExperimentVariantID=222";
-    pagespeed ExperimentSpec
-      "id=3;default;percent=0;options=ContentExperimentID=123,ContentExperimentVariantID=333";
-  }
   server {
     listen @@SECONDARY_PORT@@;
     listen [::]:@@SECONDARY_PORT@@;
@@ -621,23 +599,8 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name xfp.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed RespectXForwardedProto on;
 
-    location /redirecting_origin {
-      pagespeed off;
-      # Hack: we clear the response headers using headers_more.
-      # If we don't, nginx will add an extra empty Location: headers here.
-      # It is kind of hard to get nginx to generate a relative location header
-      # that starts with "/".
-      more_clear_headers 'Location';
-      add_header Location /mod_pagespeed_example;
-      return 301;
-    }
-    location /redirect {
-      proxy_method GET;
-      proxy_pass http://127.0.0.1:@@SECONDARY_PORT@@/redirecting_origin;
-      proxy_set_header "Host" "xfp.example.com";
-    }
+    pagespeed RespectXForwardedProto on;
   }
 
   server {
@@ -853,13 +816,6 @@ http {
     pagespeed ConsolePath /custom_pagespeed_console;
     pagespeed MessagesPath /custom_pagespeed_message;
     pagespeed AdminPath /custom_pagespeed_admin;
-
-    pagespeed StatisticsDomains Allow *;
-    pagespeed GlobalStatisticsDomains Allow *;
-    pagespeed MessagesDomains Allow *;
-    pagespeed ConsoleDomains Allow *;
-    pagespeed AdminDomains Allow *;
-    pagespeed GlobalAdminDomains Allow *;
   }
 
   server {
@@ -867,13 +823,6 @@ http {
     listen [::]:@@SECONDARY_PORT@@;
     server_name inherit-paths.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed StatisticsDomains Allow *;
-    pagespeed GlobalStatisticsDomains Allow *;
-    pagespeed MessagesDomains Allow *;
-    pagespeed ConsoleDomains Allow *;
-    pagespeed AdminDomains Allow *;
-    pagespeed GlobalAdminDomains Allow *;
   }
 
   server {
@@ -1179,31 +1128,6 @@ http {
     pagespeed RemoteConfigurationTimeoutMs 1500;
   }
 
-  # Test that pagespeed is disabled when sendfile headers are present.
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name uses-sendfile.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed EnableFilters inline_javascript,rewrite_javascript;
-    add_header 'X-Sendfile' 'blablabla';
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name doesnt-sendfile.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed EnableFilters inline_javascript,rewrite_javascript;
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name uses-xaccelredirect.example.com;
-    pagespeed FileCachePath "@@SECONDARY_CACHE@@";
-    pagespeed EnableFilters inline_javascript,rewrite_javascript;
-    add_header 'X-Accel-Redirect' 'blablabla';
-  }
-
   # Proxy + IPRO a gzip'd file for testing Issue 896.
   server {
     listen @@SECONDARY_PORT@@;
@@ -1303,15 +1227,11 @@ http {
     pagespeed FileCachePath "@@FILE_CACHE@@";
     root "@@SERVER_ROOT@@";
     set $filters "";
-    set $domain_shards "cdn1.example.com,cdn2.example.com";
     if ($http_X_Script) {
         set $filters "add_instrumentation";
-        set $domain_shards "";
     }
     pagespeed RewriteLevel PassThrough;
-    pagespeed EnableFilters rewrite_domains;
     pagespeed EnableFilters $filters;
-    pagespeed ShardDomain script-filters.example.com "$domain_shards";
   }
 
   server {
@@ -1391,79 +1311,6 @@ http {
     }
   }
 
-  pagespeed MessagesDomains Allow messages-allowed.example.com;
-  pagespeed MessagesDomains Allow cleared-inherited.example.com;
-  pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
-  pagespeed MessagesDomains Allow more-messages-allowed.example.com;
-  pagespeed MessagesDomains Allow anything-*-wildcard.example.com;
-  pagespeed MessagesDomains Allow localhost;
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name messages-allowed.example.com
-                messages-not-allowed.example.com
-                more-messages-allowed.example.com
-                anything-a-wildcard.example.com
-                anything-b-wildcard.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name messages-still-not-allowed.example.com
-                but-this-message-allowed.example.com
-                and-this-one.example.com;
-    pagespeed MessagesDomains Allow but-this-message-allowed.example.com;
-    pagespeed MessagesDomains Allow and-this-one.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name cleared-inherited.example.com
-                cleared-inherited-reallowed.example.com
-                messages-allowed-at-vhost.example.com
-                messages-not-allowed-at-vhost.example.com
-                anything-c-wildcard.example.com;
-    pagespeed MessagesDomains Disallow *;
-    pagespeed MessagesDomains Allow cleared-inherited-reallowed.example.com;
-    pagespeed MessagesDomains Allow messages-allowed-at-vhost.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name cleared-inherited-unlisted.example.com;
-    pagespeed MessagesDomains Allow *;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-  server {
-    server_name nothing-allowed.example.com;
-    pagespeed MessagesDomains Disallow *;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-  server {
-    server_name nothing-explicitly-allowed.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-  }
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name everything-explicitly-allowed.example.com
-                everything-explicitly-allowed-but-aliased.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-
-    pagespeed StatisticsDomains Allow everything-explicitly-allowed.example.com;
-    pagespeed GlobalStatisticsDomains
-      Allow everything-explicitly-allowed.example.com;
-    pagespeed MessagesDomains Allow everything-explicitly-allowed.example.com;
-    pagespeed ConsoleDomains Allow everything-explicitly-allowed.example.com;
-    pagespeed AdminDomains Allow everything-explicitly-allowed.example.com;
-    pagespeed GlobalAdminDomains
-     Allow everything-explicitly-allowed.example.com;
-  }
-
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;
@@ -1529,14 +1376,6 @@ http {
       http://www.modpagespeed.com/rewrite_javascript.js;
     pagespeed RetainComment " google_ad_section*";
 
-    # Test proxying of non-.pagespeed. resources.
-    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/modpagespeed_http
-                           http://@@PAGESPEED_TEST_HOST@@/do_not_modify;
-    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_present
-                           http://@@PAGESPEED_TEST_HOST@@:8091;
-    pagespeed MapProxyDomain http://localhost:@@PRIMARY_PORT@@/content_type_absent
-                           http://@@PAGESPEED_TEST_HOST@@:8092;
-
     add_header X-Extra-Header 1;
 
     # Establish a proxy mapping where the current server proxies an image
@@ -1655,30 +1494,6 @@ http {
       expires 5m;
     }
 
-    location /mod_pagespeed_test/ipro/wait/ {
-      # TODO(jmarantz): ModPagespeedInPlaceWaitForOptimized should be superfluous,
-      # or made equivalent to ModPagespeedInPlaceRewriteDeadlineMs -1, which waits
-      # forever.  Otherwise ModPagespeedInPlaceRewriteDeadlineMs should just have
-      # the specified deadline.
-      # # See https://github.com/pagespeed/mod_pagespeed/issues/1171 for more
-      # detailed discussion.
-      pagespeed InPlaceWaitForOptimized on;
-    }
-
-    location /mod_pagespeed_test/ipro/wait/long/ {
-      # Make the deadline long here for valgrind tests.  We could
-      # conditionalize this.
-      pagespeed InPlaceRewriteDeadlineMs 10000;
-    }
-
-    location /mod_pagespeed_test/ipro/wait/short/ {
-      pagespeed EnableFilters in_place_optimize_for_browser;
-
-      # Make the deadline short here as we expect to always miss it
-      # in tests.
-      pagespeed InPlaceRewriteDeadlineMs 1;
-    }
-
     location /mod_pagespeed_test/ipro/instant/wait/ {
       pagespeed InPlaceWaitForOptimized on;
 

test/run_tests.sh

@@ -23,9 +23,9 @@
 # Exits with status 2 if command line args are wrong.
 #
 # Usage:
-#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir pagespeed_test_host
+#   ./run_tests.sh primary_port secondary_port mod_pagespeed_dir
 # Example:
-#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed www.modpagespeed.com
+#   ./run_tests.sh 8050 8051 /path/to/mod_pagespeed
 #
 
 # Normally we test only with the native fetcher off.  Set
@@ -43,7 +43,7 @@ RUN_TESTS=${RUN_TESTS:-true}
 # true.
 USE_VALGRIND=${USE_VALGRIND:-false}
 
-if [ "$#" -ne 5 ] ; then
+if [ "$#" -ne 4 ] ; then
   echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
   echo "  nginx_executable"
   exit 2
@@ -53,7 +53,6 @@ PRIMARY_PORT="$1"
 SECONDARY_PORT="$2"
 MOD_PAGESPEED_DIR="$3"
 NGINX_EXECUTABLE="$4"
-PAGESPEED_TEST_HOST="$5"
 RCPORT1=9991
 RCPORT2=9992
 RCPORT3=9993
@@ -70,7 +69,6 @@ function run_test_checking_failure() {
     SECONDARY_PORT="$SECONDARY_PORT" \
     MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \
     NGINX_EXECUTABLE="$NGINX_EXECUTABLE" \
-    PAGESPEED_TEST_HOST="$PAGESPEED_TEST_HOST" \
     RUN_TESTS="$RUN_TESTS" \
     RCPORT1="$RCPORT1" \
     RCPORT2="$RCPORT2" \