Update for 1.8.31.4

Pulling in upstream merge to local master.

README.md

@@ -37,11 +37,11 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
 
    ```bash
    $ cd ~
-   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.7.30.4-beta.zip
-   $ unzip v1.7.30.4-beta.zip # or unzip v1.7.30.4-beta
-   $ cd ngx_pagespeed-1.7.30.4-beta/
-   $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz
-   $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/
+   $ wget https://github.com/pagespeed/ngx_pagespeed/archive/v1.8.31.4-beta.zip
+   $ unzip v1.8.31.4-beta.zip # or unzip v1.8.31.4-beta
+   $ cd ngx_pagespeed-1.8.31.4-beta/
+   $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz
+   $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/
    ```
 
 3. Download and build nginx:
@@ -49,10 +49,10 @@ recompiling Tengine](https://github.com/pagespeed/ngx_pagespeed/wiki/Using-ngx_p
    ```bash
    $ cd ~
    $ # check http://nginx.org/en/download.html for the latest version
-   $ wget http://nginx.org/download/nginx-1.4.6.tar.gz
-   $ tar -xvzf nginx-1.4.6.tar.gz
-   $ cd nginx-1.4.6/
-   $ ./configure --add-module=$HOME/ngx_pagespeed-1.7.30.4-beta
+   $ wget http://nginx.org/download/nginx-1.6.0.tar.gz
+   $ tar -xvzf nginx-1.6.0.tar.gz
+   $ cd nginx-1.6.0/
+   $ ./configure --add-module=$HOME/ngx_pagespeed-1.8.31.4-beta
    $ make
    $ sudo make install
    ```
@@ -82,12 +82,8 @@ In every server block where pagespeed is enabled add:
 #  Ensure requests for pagespeed optimized resources go to the pagespeed
 #  handler and no extraneous headers get set.
 location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" { add_header "" ""; }
-location ~ "^/ngx_pagespeed_static/" { }
+location ~ "^/pagespeed_static/" { }
 location ~ "^/ngx_pagespeed_beacon$" { }
-location /ngx_pagespeed_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_global_statistics { allow 127.0.0.1; deny all; }
-location /ngx_pagespeed_message { allow 127.0.0.1; deny all; }
-location /pagespeed_console { allow 127.0.0.1; deny all; }
 ```
 
 To confirm that the module is loaded, fetch a page and check that you see the
@@ -95,7 +91,7 @@ To confirm that the module is loaded, fetch a page and check that you see the
 
 ```bash
 $ curl -I 'http://localhost:8050/some_page/' | grep X-Page-Speed
-X-Page-Speed: 1.7.30.4-...
+X-Page-Speed: 1.8.31.4-...
 ```
 
 Looking at the source of a few pages you should see various changes, such as

config

@@ -27,8 +27,8 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then
     echo "   You need to separately download the pagespeed library:"
     echo ""
     echo "     $ cd /path/to/ngx_pagespeed"
-    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.7.30.4.tar.gz"
-    echo "     $ tar -xzvf 1.7.30.4.tar.gz # expands to psol/"
+    echo "     $ wget https://dl.google.com/dl/page-speed/psol/1.8.31.4.tar.gz"
+    echo "     $ tar -xzvf 1.8.31.4.tar.gz # expands to psol/"
     echo ""
     echo "   Or see the installation instructions:"
     echo "     https://github.com/pagespeed/ngx_pagespeed#how-to-build"

src/ngx_rewrite_options.cc

@@ -46,9 +46,10 @@ const char kGlobalAdminPath[] = "GlobalAdminPath";
 
 // These options are copied from mod_instaweb.cc, where APACHE_CONFIG_OPTIONX
 // indicates that they can not be set at the directory/location level. They set
-// options in the RewriteDriverFactory, so they do not appear in RewriteOptions.
-// They are not alphabetized on purpose, but rather left in the same order as in
-// mod_instaweb.cc in case we end up needing to compare.
+// options in the RewriteDriverFactory, so they're entirely global and do not
+// appear in RewriteOptions.  They are not alphabetized on purpose, but rather
+// left in the same order as in mod_instaweb.cc in case we end up needing to
+// compare.
 // TODO(oschaaf): this duplication is a short term solution.
 const char* const server_only_options[] = {
   "FetcherTimeoutMs",
@@ -61,6 +62,7 @@ const char* const server_only_options[] = {
   "MessageBufferSize",
   "NumRewriteThreads",
   "NumExpensiveRewriteThreads",
+  "StaticAssetPrefix",
   "TrackOriginalContentLength",
   "UsePerVHostStatistics",  // TODO(anupama): What to do about "No longer used"
   "BlockingRewriteRefererUrls",
@@ -102,20 +104,20 @@ void NgxRewriteOptions::AddProperties() {
   // Nginx-specific options.
   add_ngx_option(
       "", &NgxRewriteOptions::statistics_path_, "nsp", kStatisticsPath,
-      kProcessScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
+      kServerScope, "Set the statistics path. Ex: /ngx_pagespeed_statistics");
   add_ngx_option(
       "", &NgxRewriteOptions::global_statistics_path_, "ngsp",
       kGlobalStatisticsPath, kProcessScope,
       "Set the global statistics path. Ex: /ngx_pagespeed_global_statistics");
   add_ngx_option(
-      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kProcessScope,
+      "", &NgxRewriteOptions::console_path_, "ncp", kConsolePath, kServerScope,
       "Set the console path. Ex: /pagespeed_console");
   add_ngx_option(
       "", &NgxRewriteOptions::messages_path_, "nmp", kMessagesPath,
-      kProcessScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
+      kServerScope, "Set the messages path.  Ex: /ngx_pagespeed_message");
   add_ngx_option(
       "", &NgxRewriteOptions::admin_path_, "nap", kAdminPath,
-      kProcessScope, "Set the admin path.  Ex: /pagespeed_admin");
+      kServerScope, "Set the admin path.  Ex: /pagespeed_admin");
   add_ngx_option(
       "", &NgxRewriteOptions::global_admin_path_, "ngap", kGlobalAdminPath,
       kProcessScope, "Set the global admin path.  Ex: /pagespeed_global_admin");
@@ -309,8 +311,11 @@ const char* NgxRewriteOptions::ParseAndSetOptions(
       result = ParseAndSetOptionHelper<SystemRewriteDriverFactory>(
           arg, driver_factory,
           &SystemRewriteDriverFactory::set_track_original_content_length);
+    } else if (IsDirective(directive, "StaticAssetPrefix")) {
+      driver_factory->set_static_asset_prefix(arg);
+      result = RewriteOptions::kOptionOk;
     } else {
-      result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
+      result = ParseAndSetOptionFromName1(directive, arg, &msg, handler);
     }
   } else if (n_args == 3) {
     // Short-term special handling, until this moves to common code.

test/nginx_system_test.sh

@@ -259,7 +259,7 @@ if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
   exit 2
 fi
 
-PSA_JS_LIBRARY_URL_PREFIX="pagespeed_static"
+PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static"
 
 # An expected failure can be indicated like: "~In-place resource optimization~"
 PAGESPEED_EXPECTED_FAILURES="
@@ -522,6 +522,60 @@ if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then
   AUTH=""
 fi
 
+start_test "Custom statistics paths in server block"
+
+# Served on normal paths by default.
+URL="inherit-paths.example.com/ngx_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep cache_time_us
+
+URL="inherit-paths.example.com/ngx_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Info
+
+URL="inherit-paths.example.com/pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep console_div
+
+URL="inherit-paths.example.com/pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Admin
+
+# Not served on normal paths when overriden.
+URL="custom-paths.example.com/ngx_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep cache_time_us
+
+URL="custom-paths.example.com/ngx_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep Info
+
+URL="custom-paths.example.com/pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep console_div
+
+URL="custom-paths.example.com/pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_not_from "$OUT" grep Admin
+
+# Served on custom paths when overriden
+URL="custom-paths.example.com/custom_pagespeed_statistics"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep cache_time_us
+
+URL="custom-paths.example.com/custom_pagespeed_message"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Info
+
+URL="custom-paths.example.com/custom_pagespeed_console"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep console_div
+
+URL="custom-paths.example.com/custom_pagespeed_admin/"
+OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)
+check_from "$OUT" grep Admin
+
+
 WGET_ARGS=""
 function gunzip_grep_0ff() {
   gunzip - | fgrep -q "color:#00f"
@@ -2150,7 +2204,7 @@ keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\
 
 start_test keepalive with static resources
 keepalive_test "keepalive-static.example.com"\
-  "/pagespeed_static/js_defer.0.js" ""
+  "/pagespeed_custom_static/js_defer.0.js" ""
 
 # Test for MaxCombinedCssBytes. The html used in the test, 'combine_css.html',
 # has 4 CSS files in the following order.
@@ -2185,7 +2239,7 @@ CONNECTION=$(extract_headers $FETCH_UNTIL_OUTFILE | fgrep "Connection:")
 check_not_from "$CONNECTION" fgrep -qi "Keep-Alive, Keep-Alive"
 check_from "$CONNECTION" fgrep -qi "Keep-Alive"
 
-start_test pagespeed_static defer js served with correct headers.
+start_test pagespeed_custom_static defer js served with correct headers.
 # First, determine which hash js_defer is served with. We need a correct hash
 # to get it served up with an Etag, which is one of the things we want to test.
 URL="$HOSTNAME/mod_pagespeed_example/defer_javascript.html?PageSpeed=on&PageSpeedFilters=defer_javascript"
@@ -2342,7 +2396,9 @@ test_decent_browsers "New Opera" \
 WGETRC=$OLD_WGETRC
 WGET_ARGS=""
 
-JS_URL="$HOSTNAME/pagespeed_static/js_defer.$HASH.js"
+start_test JS gzip headers
+
+JS_URL="$HOSTNAME/pagespeed_custom_static/js_defer.$HASH.js"
 JS_HEADERS=$($WGET -O /dev/null -q -S --header='Accept-Encoding: gzip' \
   $JS_URL 2>&1)
 check_from "$JS_HEADERS" egrep -qi 'HTTP/1[.]. 200 OK'
@@ -2536,12 +2592,15 @@ OUTFILE=$OUTDIR/etags
 $WGET -o $OUTFILE -O /dev/null --header "If-None-Match: $ETAG" $URL
 check fgrep -q "awaiting response... 304" $OUTFILE
 
-start_test PageSpeed resources should have a content length.
+start_test PageSpeed CSS loaded in fallback mode is always chunked
+# This is because the domains in urls may be rewritten, changing content
+# length from what's in the cache.
 URL="$EXAMPLE_ROOT/styles/W.rewrite_css_images.css.pagespeed.cf.Hash.css"
-OUT=$($WGET_DUMP --save-headers $URL)
-check_from "$OUT" egrep -q $'^Content-Length: ([0-9])*\r$'
-check_not_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
-check_not_from "$OUT" egrep -iq $'^Connection: close\r$'
+fetch_until -save $URL "egrep -c Transfer-Encoding:" 1 "--save-headers"
+OUT=$(cat $FETCH_UNTIL_OUTFILE)
+check_from "$OUT" egrep -iq $'^Transfer-Encoding: chunked\r$'
+check_not_from "$OUT" egrep -iq '^Content-Length:'
+check_not_from "$OUT" egrep -iq '^Connection: close'
 
 start_test Downstream cache integration caching headers.
 URL="http://downstreamcacheresource.example.com/mod_pagespeed_example/images/"

test/pagespeed_test.conf.template

@@ -34,6 +34,8 @@ http {
   pagespeed AdminPath /pagespeed_admin;
   pagespeed GlobalAdminPath /pagespeed_global_admin;
 
+  pagespeed StaticAssetPrefix /pagespeed_custom_static/;
+
   root "@@SERVER_ROOT@@";
 
   # Block 5a: Decide on Cache-Control header value to use for outgoing
@@ -711,6 +713,23 @@ http {
     }
   }
 
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name custom-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+
+    pagespeed StatisticsPath /custom_pagespeed_statistics;
+    pagespeed ConsolePath /custom_pagespeed_console;
+    pagespeed MessagesPath /custom_pagespeed_message;
+    pagespeed AdminPath /custom_pagespeed_admin;
+  }
+
+  server {
+    listen @@SECONDARY_PORT@@;
+    server_name inherit-paths.example.com;
+    pagespeed FileCachePath "@@FILE_CACHE@@";
+  }
+
   server {
     listen @@SECONDARY_PORT@@;
     server_name notransform.example.com;