Update PSOL_BINARY_URL

Kernels before 2.6.35 do not have F_SETPIPE_SZ defined.

Fixes https://github.com/pagespeed/ngx_pagespeed/issues/1503

.gitmodules

@@ -1,18 +0,0 @@
-[submodule "testing-dependencies/mod_pagespeed"]
-	path = testing-dependencies/mod_pagespeed
-	url = https://github.com/pagespeed/mod_pagespeed.git
-[submodule "testing-dependencies/ngx_cache_purge"]
-	path = testing-dependencies/ngx_cache_purge
-	url = https://github.com/FRiCKLE/ngx_cache_purge.git
-[submodule "testing-dependencies/nginx"]
-	path = testing-dependencies/nginx
-	url = https://github.com/nginx/nginx.git
-[submodule "testing-dependencies/set-misc-nginx-module"]
-	path = testing-dependencies/set-misc-nginx-module
-	url = https://github.com/openresty/set-misc-nginx-module
-[submodule "testing-dependencies/ngx_devel_kit"]
-	path = testing-dependencies/ngx_devel_kit
-	url = https://github.com/simpl/ngx_devel_kit
-[submodule "testing-dependencies/headers-more-nginx-module"]
-	path = testing-dependencies/headers-more-nginx-module
-	url = https://github.com/openresty/headers-more-nginx-module

PSOL_BINARY_URL

@@ -1,4 +1 @@
-In a release this file would contain the URL to download the pre-compiled PSOL
-binary, but on development branches (like this one) you have to build PSOL from
-source yourself.  See:
-  https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source
+https://dl.google.com/dl/page-speed/psol/1.13.35.2-$BIT_SIZE_NAME.tar.gz

config

@@ -15,9 +15,11 @@
 # Environment Variables (Optional):
 #   MOD_PAGESPEED_DIR: absolute path to the mod_pagespeed/src directory
 #   PSOL_BINARY: absolute path to pagespeed_automatic.a
+#   PSOL_BUILDTYPE: Release or Debug
 
 mod_pagespeed_dir="${MOD_PAGESPEED_DIR:-unset}"
 position_aux="${POSITION_AUX:-unset}"
+psol_buildtype="${PSOL_BUILDTYPE:-unset}"
 
 if [ "$mod_pagespeed_dir" = "unset" ] ; then
   mod_pagespeed_dir="$ngx_addon_dir/psol/include"
@@ -75,12 +77,16 @@ else
   exit 1
 fi
 
-if [ "$NGX_DEBUG" = "YES" ]; then
-  buildtype=Debug
-  # If we're using a psol tarball that doesn't contain Debug/ (which is the case
-  # from 1.12 onward) then this will be overriden to buildtype=Release below.
+if [ "$psol_buildtype" = "unset" ] ; then
+  if [ "$NGX_DEBUG" = "YES" ]; then
+    buildtype=Debug
+    # If we're using a psol tarball that doesn't contain Debug/ (which is the case
+    # from 1.12 onward) then this will be overriden to buildtype=Release below.
+  else
+    buildtype=Release
+  fi
 else
-  buildtype=Release
+  buildtype=$psol_buildtype
 fi
 
 # If the compiler is gcc, we want to use g++ to link, if at all possible,

scripts/build_ngx_pagespeed.sh

@@ -486,7 +486,7 @@ add support for dynamic modules in a way compatible with ngx_pagespeed until
       status "Detected debian-based distro."
 
       install_dependencies "apt-get install" debian_is_installed \
-        "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip"
+        build-essential zlib1g-dev libpcre3 libpcre3-dev unzip uuid-dev
 
       if gcc_too_old; then
         if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then
@@ -504,7 +504,7 @@ add support for dynamic modules in a way compatible with ngx_pagespeed until
       status "Detected redhat-based distro."
 
       install_dependencies "yum install" redhat_is_installed \
-        "gcc-c++ pcre-devel zlib-devel make unzip wget"
+        gcc-c++ pcre-devel zlib-devel make unzip wget libuuid-devel
       if gcc_too_old; then
         if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then
           redhat_major_version=$(
@@ -634,7 +634,13 @@ Not deleting $directory; name is suspiciously short.  Something is wrong."
       run make apache_debug_psol
       PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a"
     else
-      run install/build_psol.sh --skip_tests --skip_packaging
+      if "$DO_DEPS_CHECK"; then
+        skip_deps_arg=""
+      else
+        skip_deps_arg="--skip_deps"
+      fi
+
+      run install/build_psol.sh --skip_tests --skip_packaging "$skip_deps_arg"
       PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a"
     fi
     run popd

src/ngx_event_connection.cc

@@ -52,6 +52,14 @@ bool NgxEventConnection::Init(ngx_cycle_t* cycle) {
   } else {
     pipe_read_fd_ = file_descriptors[0];
     pipe_write_fd_ = file_descriptors[1];
+    // Attempt to bump the pipe capacity, because running out of buffer space
+    // can potentially lead up to writes spinning on EAGAIN.
+    // See https://github.com/pagespeed/ngx_pagespeed/issues/1380
+    // TODO(oschaaf): Consider implementing a queueing mechanism for retrying
+    // failed writes.
+#ifdef F_SETPIPE_SZ
+    fcntl(pipe_write_fd_, F_SETPIPE_SZ, 200*1024 /* minimal amount of bytes */);
+#endif
     return true;
   }
   close(file_descriptors[0]);

src/ngx_pagespeed.cc

@@ -84,6 +84,7 @@ extern ngx_module_t ngx_pagespeed;
 
 // Needed for SystemRewriteDriverFactory to use shared memory.
 #define PAGESPEED_SUPPORT_POSIX_SHARED_MEM
+#define NGINX_1_13_4 1013004
 
 net_instaweb::NgxRewriteDriverFactory* active_driver_factory = NULL;
 
@@ -2703,8 +2704,7 @@ ngx_int_t ps_simple_handler(ngx_http_request_t* r,
     }
   }
 
-  send_out_headers_and_body(r, response_headers, output);
-  return NGX_OK;
+  return send_out_headers_and_body(r, response_headers, output);
 }
 
 void ps_beacon_handler_helper(ngx_http_request_t* r,
@@ -3019,13 +3019,17 @@ ngx_int_t ps_preaccess_handler(ngx_http_request_t* r) {
   ph = cmcf->phase_engine.handlers;
 
   i = r->phase_handler;
+
   // move handlers before try_files && content phase
+  // As of nginx 1.13.4 we will be right before the try_files module
+  #if (nginx_version < NGINX_1_13_4)
   while (ph[i + 1].checker != ngx_http_core_try_files_phase &&
          ph[i + 1].checker != ngx_http_core_content_phase) {
     ph[i] = ph[i + 1];
     ph[i].next--;
     i++;
   }
+  #endif
 
   // insert ps phase handler
   ph[i].checker = ps_phase_handler;
@@ -3081,8 +3085,17 @@ ngx_int_t ps_init(ngx_conf_t* cf) {
     ngx_http_core_main_conf_t* cmcf = static_cast<ngx_http_core_main_conf_t*>(
         ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module));
 
+    int phase = NGX_HTTP_PREACCESS_PHASE;
+
+    // As of nginx 1.13.4, try_files has changed.
+    // https://github.com/nginx/nginx/commit/129b06dc5dfab7b4513a4f274b3778cd9b8a6a22
+#if (nginx_version >= NGINX_1_13_4)
+    phase = NGX_HTTP_PRECONTENT_PHASE;
+#endif
+
     ngx_http_handler_pt* h = static_cast<ngx_http_handler_pt*>(
-        ngx_array_push(&cmcf->phases[NGX_HTTP_PREACCESS_PHASE].handlers));
+        ngx_array_push(&cmcf->phases[phase].handlers));
+
     if (h == NULL) {
       return NGX_ERROR;
     }

test/pagespeed_test.conf.template

@@ -85,11 +85,19 @@ http {
   pagespeed StatisticsLogging on;
   pagespeed LogDir "@@TEST_TMP@@/logdir";
 
-  pagespeed MaxFetchRedirects 3;
-
   # Expanded to CentralControllerPort directive when RUN_CONTROLLER_TEST=on
   @@CONTROLLER@@
 
+  server {
+    # This is the default vhost for @@SECONDARY_PORT@@
+    # Your request will land here when you pass in an unknown host.
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name default.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel CoreFilters;
+  }
+
   server {
     # Sets up a logical home-page server on
     # max-cacheable-content-length.example.com.  This server is only used to
@@ -1674,50 +1682,6 @@ http {
     pagespeed MessagesDomains Disallow *;
     pagespeed FileCachePath "@@FILE_CACHE@@";
   }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name redirecting-fetch.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed Domain redirecting-fetch.example.com:@@SECONDARY_PORT@@;
-    pagespeed MapOriginDomain 127.0.0.1:@@SECONDARY_PORT@@
-    	      redirecting-fetch.example.com:@@SECONDARY_PORT@@;
-    pagespeed Disallow *styles/disallowed.css;
-    pagespeed InPlaceResourceOptimization off;
-
-    location /redir_to_test/styles/blue.css {
-      return 301 /mod_pagespeed_example/styles/yellow.css;
-    }
-    location /redir_to_test/styles/1.css {
-      return 301 /redir_to_test/styles/2.css;
-    }
-    location /redir_to_test/styles/2.css {
-      return 301 /mod_pagespeed_example/styles/yellow.css;
-    }
-    location /redir_to_test/styles/redirtodisallowed.css {
-      return 301 /redir_to_test/styles/disallowed.css;
-    }
-  }
-
-  server {
-    listen @@SECONDARY_PORT@@;
-    listen [::]:@@SECONDARY_PORT@@;
-    server_name redirecting-fetch-single-only.example.com;
-    pagespeed FileCachePath "@@FILE_CACHE@@";
-    pagespeed MaxFetchRedirects 1;
-    pagespeed Domain redirecting-fetch.example.com:@@SECONDARY_PORT@@;
-    pagespeed MapOriginDomain 127.0.0.1:@@SECONDARY_PORT@@ 
-       redirecting-fetch-single-only.example.com:@@SECONDARY_PORT@@;
-
-    location /redir_to_test/styles/1.css {
-      return 301 /redir_to_test/styles/2.css;
-    }
-    location /redir_to_test/styles/2.css {
-      return 301 /mod_pagespeed_example/styles/yellow.css;
-    }
-  }
-
   server {
     server_name nothing-explicitly-allowed.example.com;
     pagespeed FileCachePath "@@FILE_CACHE@@";
@@ -1831,6 +1795,25 @@ http {
     pagespeed DisableFilters add_instrumentation;
     pagespeed CriticalImagesBeaconEnabled false;
   }
+  server {
+    pagespeed on;
+    listen @@SECONDARY_PORT@@;
+    listen [::]:@@SECONDARY_PORT@@;
+    server_name image-rewrite-with-flush.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+    pagespeed RewriteLevel PassThrough;
+    pagespeed UrlValuedAttribute li data-thumb image;
+    pagespeed EnableFilters core;
+    location ~ \.php$ {
+      fastcgi_param SCRIPT_FILENAME $request_filename;
+      fastcgi_param QUERY_STRING $query_string;
+      fastcgi_param REQUEST_METHOD $request_method;
+      fastcgi_param CONTENT_TYPE $content_type;
+      fastcgi_param CONTENT_LENGTH $content_length;
+      fastcgi_pass 127.0.0.1:9000;
+      fastcgi_buffering off;
+    }
+  }
   server {
     listen @@PRIMARY_PORT@@;
     listen [::]:@@PRIMARY_PORT@@;