From 852e376130d498abb5b84af426700ad8f7898d5f Mon Sep 17 00:00:00 2001 From: Jeff Kaufman Date: Wed, 19 Nov 2014 09:48:42 -0500 Subject: [PATCH] Move duplicate tests from the nginx-specific system test to system/system_test As system/system_test.sh is in the mod_pagespeed repo, this is just a bunch of deletes. --- test/nginx_system_test.sh | 1690 +---------------------------- test/pagespeed_test.conf.template | 63 +- 2 files changed, 56 insertions(+), 1697 deletions(-) diff --git a/test/nginx_system_test.sh b/test/nginx_system_test.sh index 7530cb4c7..587e42d1b 100644 --- a/test/nginx_system_test.sh +++ b/test/nginx_system_test.sh @@ -268,6 +268,8 @@ if [ ! -e "$SYSTEM_TEST_FILE" ] ; then fi PSA_JS_LIBRARY_URL_PREFIX="pagespeed_custom_static" +BEACON_HANDLER="ngx_pagespeed_beacon" +STATISTICS_URL=http://$PRIMARY_HOSTNAME/ngx_pagespeed_statistics # An expected failure can be indicated like: "~In-place resource optimization~" PAGESPEED_EXPECTED_FAILURES=" @@ -294,31 +296,6 @@ fi set -- "$PRIMARY_HOSTNAME" source $SYSTEM_TEST_FILE -STATISTICS_URL=$PRIMARY_SERVER/ngx_pagespeed_statistics - -# Define a mechanism to start a test before the cache-flush and finish it -# after the cache-flush. This mechanism is preferable to flushing cache -# within a test as that requires waiting 5 seconds for the poll, so we'd -# like to limit the number of cache flushes and exploit it on behalf of -# multiple tests. - -# Variable holding a space-separated lists of bash functions to run after -# flushing cache. -post_cache_flush_test="" - -# Adds a new function to run after cache flush. -function on_cache_flush() { - post_cache_flush_test+=" $1" -} - -# Called after cache-flush to run all the functions specified to -# on_cache_flush. -function run_post_cache_flush() { - for test in $post_cache_flush_test; do - $test - done -} - # nginx-specific system tests start_test Test pagespeed directive inside if block inside location block. @@ -410,127 +387,6 @@ else fi -start_test Check for correct default X-Page-Speed header format. -OUT=$($WGET_DUMP $EXAMPLE_ROOT/combine_css.html) -check_from "$OUT" egrep -q \ - '^X-Page-Speed: [0-9]+[.][0-9]+[.][0-9]+[.][0-9]+-[0-9]+' - -start_test pagespeed is defaulting to more than PassThrough -fetch_until $TEST_ROOT/bot_test.html 'fgrep -c .pagespeed.' 2 - -start_test 404s are served and properly recorded. -NUM_404=$(scrape_stat resource_404_count) -echo "Initial 404s: $NUM_404" -WGET_ERROR=$(check_not $WGET -O /dev/null $BAD_RESOURCE_URL 2>&1) -check_from "$WGET_ERROR" fgrep -q "404 Not Found" - -# Check that the stat got bumped. -NUM_404_FINAL=$(scrape_stat resource_404_count) -echo "Final 404s: $NUM_404_FINAL" -check [ $(expr $NUM_404_FINAL - $NUM_404) -eq 1 ] - -# Check that the stat doesn't get bumped on non-404s. -URL="$PRIMARY_SERVER/mod_pagespeed_example/styles/" -URL+="W.rewrite_css_images.css.pagespeed.cf.Hash.css" -OUT=$(wget -O - -q $URL) -check_from "$OUT" grep background-image -NUM_404_REALLY_FINAL=$(scrape_stat resource_404_count) -check [ $NUM_404_FINAL -eq $NUM_404_REALLY_FINAL ] - -start_test Non-local access to statistics fails. - -# This test only makes sense if you're running tests against localhost. -if [ "$HOSTNAME" = "localhost:$PRIMARY_PORT" ] ; then - NON_LOCAL_IP=$(ifconfig | egrep -o 'inet addr:[0-9]+.[0-9]+.[0-9]+.[0-9]+' \ - | awk -F: '{print $2}' | grep -v ^127 | head -n 1) - - # Make sure pagespeed is listening on NON_LOCAL_IP. - URL="http://$NON_LOCAL_IP:$PRIMARY_PORT/mod_pagespeed_example/styles/" - URL+="W.rewrite_css_images.css.pagespeed.cf.Hash.css" - OUT=$(wget -O - -q $URL) - check_from "$OUT" grep background-image - - # Make sure we can't load statistics from NON_LOCAL_IP. - ALT_STAT_URL=$(echo $STATISTICS_URL | sed s#localhost#$NON_LOCAL_IP#) - - echo "wget $ALT_STAT_URL >& $TESTTMP/alt_stat_url" - check_error_code 8 wget $ALT_STAT_URL >& "$TESTTMP/alt_stat_url" - rm -f "$TESTTMP/alt_stat_url" - - ALT_CE_URL="$ALT_STAT_URL.pagespeed.ce.8CfGBvwDhH.css" - check_error_code 8 wget -O - $ALT_CE_URL >& "$TESTTMP/alt_ce_url" - check_error_code 8 wget -O - --header="Host: $HOSTNAME" $ALT_CE_URL \ - >& "$TESTTMP/alt_ce_url" - rm -f "$TESTTMP/alt_ce_url" - - # Even though we don't have a cookie, we will conservatively avoid - # optimizing resources with Vary:Cookie set on the response, so we - # will not get the instant response, of "body{background:#9370db}": - # 24 bytes, but will get the full original text: - # "body {\n background: MediumPurple;\n}\n" - # This will happen whether or not we send a cookie. - # - # Testing this requires proving we'll never optimize something, which - # can't be distinguished from the not-yet-optimized case, except by the - # ipro_not_rewritable stat, so we loop by scraping that stat and seeing - # when it changes. - - # Executes commands until ipro_no_rewrite_count changes. The - # command-line options are all passed to WGET_DUMP. Leaves command - # wget output in $IPRO_OUTPUT. - function ipro_expect_no_rewrite() { - ipro_no_rewrite_count_start=$(scrape_stat ipro_not_rewritable) - ipro_no_rewrite_count=$ipro_no_rewrite_count_start - iters=0 - while [ $ipro_no_rewrite_count -eq $ipro_no_rewrite_count_start ]; do - if [ $iters -ne 0 ]; then - sleep 0.1 - if [ $iters -gt 100 ]; then - echo TIMEOUT - exit 1 - fi - fi - IPRO_OUTPUT=$($WGET_DUMP "$@") - ipro_no_rewrite_count=$(scrape_stat ipro_not_rewritable) - iters=$((iters + 1)) - done - } - - start_test ipro with vary:cookie with no cookie set - ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie/vary_cookie.css - check_from "$IPRO_OUTPUT" fgrep -q ' background: MediumPurple;' - check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie' - - start_test ipro with vary:cookie with cookie set - ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie/vary_cookie.css \ - --header=Cookie:cookie-data - check_from "$IPRO_OUTPUT" fgrep -q ' background: MediumPurple;' - check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie' - - start_test ipro with vary:cookie2 with no cookie2 set - ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie2/vary_cookie2.css - check_from "$IPRO_OUTPUT" fgrep -q ' background: MediumPurple;' - check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie2' - - start_test ipro with vary:cookie2 with cookie2 set - ipro_expect_no_rewrite $TEST_ROOT/ipro/cookie2/vary_cookie2.css \ - --header=Cookie2:cookie2-data - check_from "$IPRO_OUTPUT" fgrep -q ' background: MediumPurple;' - check_from "$IPRO_OUTPUT" fgrep -q 'Vary: Cookie2' - - start_test authorized resources do not get cached and optimized. - URL="$TEST_ROOT/auth/medium_purple.css" - AUTH="Authorization:Basic dXNlcjE6cGFzc3dvcmQ=" - not_cacheable_start=$(scrape_stat ipro_recorder_not_cacheable) - echo $WGET_DUMP --header="$AUTH" "$URL" - OUT=$($WGET_DUMP --header="$AUTH" "$URL") - check_from "$OUT" fgrep -q 'background: MediumPurple;' - not_cacheable=$(scrape_stat ipro_recorder_not_cacheable) - check [ $not_cacheable = $((not_cacheable_start + 1)) ] - URL="" - AUTH="" -fi - start_test "Custom statistics paths in server block" # Served on normal paths by default. @@ -605,36 +461,13 @@ OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET -q -O - \ http://ipro-proxy.example.com/A.big.css.pagespeed.cf.0.css) check_from "$OUT" fgrep -q "big{color:#00f}" -start_test Accept bad query params and headers - -# The examples page should have this EXPECTED_EXAMPLES_TEXT on it. -EXPECTED_EXAMPLES_TEXT="PageSpeed Examples Directory" -OUT=$(wget -O - $EXAMPLE_ROOT) -check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT" - -# It should still be there with bad query params. -OUT=$(wget -O - $EXAMPLE_ROOT?PageSpeedFilters=bogus) -check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT" - -# And also with bad request headers. -OUT=$(wget -O - --header=PageSpeedFilters:bogus $EXAMPLE_ROOT) -check_from "$OUT" fgrep -q "$EXPECTED_EXAMPLES_TEXT" - -# Tests that an origin header with a Vary header other than Vary:Accept-Encoding -# loses that header when we are not respecting vary. -start_test Vary:User-Agent on resources is held by our cache. -URL="$TEST_ROOT/vary/no_respect/index.html" -fetch_until -save $URL 'fgrep -c .pagespeed.cf.' 1 - -# Extract out the rewritten CSS file from the HTML saved by fetch_until -# above (see -save and definition of fetch_until). Fetch that CSS -# file with headers and make sure the Vary is stripped. -CSS_URL=$(grep stylesheet $FETCH_UNTIL_OUTFILE | cut -d\" -f 4) -CSS_URL="$TEST_ROOT/vary/no_respect/$(basename $CSS_URL)" -echo CSS_URL=$CSS_URL -CSS_OUT=$($WGET_DUMP $CSS_URL) -check_from "$CSS_OUT" fgrep -q "Vary: Accept-Encoding" -check_not_from "$CSS_OUT" fgrep -q "User-Agent" +start_test Respect X-Forwarded-Proto when told to +FETCHED=$OUTDIR/x_forwarded_proto +URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?PageSpeedFilters=add_base_tag +HEADERS="--header=X-Forwarded-Proto:https --header=Host:xfp.example.com" +check $WGET_DUMP -O $FETCHED $HEADERS $URL +# When enabled, we respect X-Forwarded-Proto and thus list base as https. +check fgrep -q '&1) -echo JS_HEADERS=$JS_HEADERS -check_200_http_response "$JS_HEADERS" -check_from "$JS_HEADERS" fgrep -qi 'Content-Encoding: gzip' -check_from "$JS_HEADERS" fgrep -qi 'Vary: Accept-Encoding' -check_from "$JS_HEADERS" egrep -qi '(Etag: W/"0")|(Etag: W/"0-gzip")' -check_from "$JS_HEADERS" fgrep -qi 'Last-Modified:' - -start_test Respect X-Forwarded-Proto when told to -FETCHED=$OUTDIR/x_forwarded_proto -URL=$SECONDARY_HOSTNAME/mod_pagespeed_example/?PageSpeedFilters=add_base_tag -HEADERS="--header=X-Forwarded-Proto:https --header=Host:xfp.example.com" -check $WGET_DUMP -O $FETCHED $HEADERS $URL -# When enabled, we respect X-Forwarded-Proto and thus list base as https. -check fgrep -q ' $FETCHED -sleep .1 # Give pagespeed time to transform the image if it's going to. -wget -O - $URL $WGET_ARGS > $FETCHED -# Make sure that the URLs in the html are not rewritten -check_not fgrep -q '.pagespeed.' $FETCHED -URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/no_transform/BikeCrashIcn.png" -wget -O - -S $URL $WGET_ARGS &> $FETCHED -# Make sure that the no-transfrom header is still there -check grep -q 'Cache-Control:.*no-transform' $FETCHED - -start_test respect vary user-agent -URL="$SECONDARY_HOSTNAME/mod_pagespeed_test/vary/index.html" -URL+="?PageSpeedFilters=inline_css" -FETCH_CMD="$WGET_DUMP --header=Host:respectvary.example.com $URL" -OUT=$($FETCH_CMD) -# We want to verify that css is not inlined, but if we just check once then -# pagespeed doesn't have long enough to be able to inline it. -sleep .1 -OUT=$($FETCH_CMD) -check_not_from "$OUT" fgrep "' -test_optimize_for_bandwidth css_urls/rewrite_css.html \ - '.blue{foreground-color:blue}body{background:url(arrow.png)}' \ - '.yellow{background-color:#ff0}' - -# To make sure that we can reconstruct the proxied content by going back -# to the origin, we must avoid hitting the output cache. -# Note that cache-flushing does not affect the cache of rewritten resources; -# only input-resources and metadata. To avoid hitting that cache and force -# us to rewrite the resource from origin, we grab this resource from a -# virtual host attached to a different cache. -# -# With the proper hash, we'll get a long cache lifetime. -SECONDARY_HOST="http://mpd.example.com/gstatic_images" -PROXIED_IMAGE="$SECONDARY_HOST$PAGESPEED_GIF" - -start_test $PROXIED_IMAGE expecting one year cache. -http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \ - "grep -c max-age=31536000" 1 --save-headers - -# With the wrong hash, we'll get a short cache lifetime (and also no output -# cache hit. -WRONG_HASH="0" -PROXIED_IMAGE="$SECONDARY_HOST/1.gif.pagespeed.ce.$WRONG_HASH.jpg" -start_test Fetching $PROXIED_IMAGE expecting short private cache. -http_proxy=$SECONDARY_HOSTNAME fetch_until $PROXIED_IMAGE \ - "grep -c max-age=300,private" 1 --save-headers - -start_test ShowCache without URL gets a form, inputs, preloaded UA. -ADMIN_CACHE=$PRIMARY_SERVER/pagespeed_admin/cache -OUT=$($WGET_DUMP $ADMIN_CACHE) -check_from "$OUT" fgrep -q "
" -check_from "$OUT" fgrep -q "200k. But by enabling resizing & compression 73 -# as specified in the HTML domain, and transmitting that configuration via -# image URL query param, the image file (including headers) is 8341 bytes. -# We check against 10000 here so this test isn't sensitive to -# image-compression tweaks (we have enough of those elsewhere). -check_file_size "$WGET_DIR/256x192xPuz*.pagespeed.*iq=*.ic.*" -lt 10000 - -# The CSS file gets rewritten with embedded options, and will have an -# embedded image in it as well. -check_file_size "$WGET_DIR/*rewrite_css_images.css.pagespeed.*+ii+*+iq=*.cf.*" \ - -lt 600 - -# The JS file is rewritten but has no related options set, so it will -# not get the embedded options between "pagespeed" and "jm". -check_file_size "$WGET_DIR/rewrite_javascript.js.pagespeed.jm.*.js" -lt 500 - -# Count how many bytes there are of body, skipping the initial headers -function body_size { - fname="$1" - tail -n+$(($(extract_headers $fname | wc -l) + 1)) $fname | wc -c -} - -# One flaw in the above test is that it short-circuits the decoding -# of the query-params because when pagespeed responds to the recursive -# wget fetch of the image, it finds the rewritten resource in the -# cache. The two vhosts are set up with the same cache. If they -# had different caches we'd have a different problem, which is that -# the first load of the image-rewrite from the resource vhost would -# not be resized. To make sure the decoding path works, we'll -# "finish" this test below after performing a cache flush, saving -# the encoded image and expected size. -EMBED_CONFIGURATION_IMAGE="http://embed-config-resources.example.com/images/" -EMBED_CONFIGURATION_IMAGE_TAIL=$(ls $WGET_DIR | grep 256x192xPuz | grep iq=) -EMBED_CONFIGURATION_IMAGE+="$EMBED_CONFIGURATION_IMAGE_TAIL" -EMBED_CONFIGURATION_IMAGE_LENGTH=$( - body_size "$WGET_DIR/$EMBED_CONFIGURATION_IMAGE_TAIL") - -# Grab the URL for the CSS file. -EMBED_CONFIGURATION_CSS_LEAF=$(ls $WGET_DIR | \ - grep '\.pagespeed\..*+ii+.*+iq=.*\.cf\..*') -EMBED_CONFIGURATION_CSS_LENGTH=$( - body_size $WGET_DIR/$EMBED_CONFIGURATION_CSS_LEAF) - -EMBED_CONFIGURATION_CSS_URL="http://embed-config-resources.example.com/styles" -EMBED_CONFIGURATION_CSS_URL+="/$EMBED_CONFIGURATION_CSS_LEAF" - -# Grab the URL for that embedded image; it should *also* have the embedded -# configuration options in it, though wget/recursive will not have pulled -# it to a file for us (wget does not parse CSS) so we'll have to request it. -EMBED_CONFIGURATION_CSS_IMAGE=$WGET_DIR/*images.css.pagespeed.*+ii+*+iq=*.cf.* -EMBED_CONFIGURATION_CSS_IMAGE_URL=$(egrep -o \ - 'http://.*iq=[0-9]*\.ic\..*\.jpg' \ - $EMBED_CONFIGURATION_CSS_IMAGE) -# fetch that file and make sure it has the right cache-control -http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \ - $EMBED_CONFIGURATION_CSS_IMAGE_URL > "$WGET_DIR/img" -CSS_IMAGE_HEADERS=$(head -10 "$WGET_DIR/img") -check_from "$CSS_IMAGE_HEADERS" fgrep -q "Cache-Control: max-age=31536000" -EMBED_CONFIGURATION_CSS_IMAGE_LENGTH=$(body_size "$WGET_DIR/img") - -function embed_image_config_post_flush() { - # Finish off the url-params-.pagespeed.-resource tests with a clear - # cache. We split the test like this to avoid having multiple - # places where we flush cache, which requires sleeps since the - # cache-flush is poll driven. - start_test Embed image/css configuration decoding with clear cache. - echo Looking for $EMBED_CONFIGURATION_IMAGE expecting \ - $EMBED_CONFIGURATION_IMAGE_LENGTH bytes - http_proxy=$SECONDARY_HOSTNAME fetch_until "$EMBED_CONFIGURATION_IMAGE" \ - "wc -c" $EMBED_CONFIGURATION_IMAGE_LENGTH - - echo Looking for $EMBED_CONFIGURATION_CSS_IMAGE_URL expecting \ - $EMBED_CONFIGURATION_CSS_IMAGE_LENGTH bytes - http_proxy=$SECONDARY_HOSTNAME fetch_until \ - "$EMBED_CONFIGURATION_CSS_IMAGE_URL" \ - "wc -c" $EMBED_CONFIGURATION_CSS_IMAGE_LENGTH - - echo Looking for $EMBED_CONFIGURATION_CSS_URL expecting \ - $EMBED_CONFIGURATION_CSS_LENGTH bytes - http_proxy=$SECONDARY_HOSTNAME fetch_until \ - "$EMBED_CONFIGURATION_CSS_URL" \ - "wc -c" $EMBED_CONFIGURATION_CSS_LENGTH -} -on_cache_flush embed_image_config_post_flush - # Several cache flushing tests. -start_test Touching cache.flush flushes the cache. +start_test Cache flushing works by touching cache.flush in cache directory. # If we write fixed values into the css file here, there is a risk that # we will end up seeing the 'right' value because an old process hasn't @@ -1523,87 +735,6 @@ check [ $ERRS -ge 1 ] #check grep "URL http://modpagespeed.com:1023/someimage.png active for " \ # $FETCHER_REFUSED_PATH -# http://code.google.com/p/modpagespeed/issues/detail?id=494 -- test -# that fetching a css with embedded relative images from a different -# VirtualHost, accessing the same content, and rewrite-mapped to the -# primary domain, delivers results that are cached for a year, which -# implies the hash matches when serving vs when rewriting from HTML. -# -# This rewrites the CSS, absolutifying the embedded relative image URL -# reference based on the the main server host. -start_test Relative images embedded in a CSS file served from a mapped domain -DIR="mod_pagespeed_test/map_css_embedded" -URL="http://www.example.com/$DIR/issue494.html" -MAPPED_PREFIX="$DIR/A.styles.css.pagespeed.cf" -http_proxy=$SECONDARY_HOSTNAME fetch_until $URL \ - "grep -c cdn.example.com/$MAPPED_PREFIX" 1 -MAPPED_CSS=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL | \ - grep -o "$MAPPED_PREFIX..*.css") - -# Now fetch the resource using a different host, which is mapped to the first -# one. To get the correct bytes, matching hash, and long TTL, we need to do -# apply the domain mapping in the CSS resource fetch. -URL="http://origin.example.com/$MAPPED_CSS" -echo http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL -CSS_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL) -check_from "$CSS_OUT" fgrep -q "Cache-Control: max-age=31536000" - -# Test ForbidFilters, which is set in the config for the VHost -# forbidden.example.com, where we've forbidden remove_quotes, remove_comments, -# collapse_whitespace, rewrite_css, and resize_images; we've also disabled -# inline_css so the link doesn't get inlined since we test that it still has all -# its quotes. -FORBIDDEN_TEST_ROOT=http://forbidden.example.com/mod_pagespeed_test -function test_forbid_filters() { - QUERYP="$1" - HEADER="$2" - URL="$FORBIDDEN_TEST_ROOT/forbidden.html" - OUTFILE="$TESTTMP/test_forbid_filters" - echo http_proxy=$SECONDARY_HOSTNAME $WGET $HEADER $URL$QUERYP - http_proxy=$SECONDARY_HOSTNAME $WGET -q -O $OUTFILE $HEADER $URL$QUERYP - check egrep -q '&1) -check_from "$OUT" fgrep -q "200 OK" -# .cf. is forbidden -FORBIDDEN=$FORBIDDEN_STYLES_ROOT/A.all_styles.css.pagespeed.cf.UH8L-zY4b4.css -OUT=$(http_proxy=$SECONDARY_HOSTNAME check_not $WGET -O /dev/null $FORBIDDEN \ - 2>&1) -check_from "$OUT" fgrep -q "404 Not Found" -# The image will be optimized but NOT resized to the much smaller size, -# so it will be >200k (optimized) rather than <20k (resized). -# Use a blocking fetch to force all -allowed- rewriting to be done. -RESIZED=$FORBIDDEN_IMAGES_ROOT/256x192xPuzzle.jpg.pagespeed.ic.8AB3ykr7Of.jpg -HEADERS="$WGET_DIR/headers" -http_proxy=$SECONDARY_HOSTNAME $WGET -q --server-response -O /dev/null \ - --header 'X-PSA-Blocking-Rewrite: psatest' $RESIZED >& $HEADERS -LENGTH=$(grep '^ *Content-Length:' $HEADERS | sed -e 's/.*://') -check test -n "$LENGTH" -check test $LENGTH -gt 200000 -CCONTROL=$(grep '^ *Cache-Control:' $HEADERS | sed -e 's/.*://') -check_from "$CCONTROL" grep -w max-age=300 -check_from "$CCONTROL" grep -w private - start_test Blocking rewrite enabled. # We assume that blocking_rewrite_test_dont_reuse_1.jpg will not be # rewritten on the first request since it takes significantly more time to @@ -1678,102 +809,6 @@ HEADER="--header=PageSpeedFilters:" HEADER="${HEADER}+remove_quotes,+remove_comments,+collapse_whitespace" test_forbid_all_disabled "" $HEADER -# Test that we work fine with an explicitly configured SHM metadata cache. -start_test Using SHM metadata cache -HOST_NAME="http://shmcache.example.com" -URL="$HOST_NAME/mod_pagespeed_example/rewrite_images.html" -http_proxy=$SECONDARY_HOSTNAME fetch_until $URL 'grep -c .pagespeed.ic' 2 - -# Test max_cacheable_response_content_length. There are two Javascript files -# in the html file. The smaller Javascript file should be rewritten while -# the larger one shouldn't. -start_test Maximum length of cacheable response content. -HOST_NAME="http://max-cacheable-content-length.example.com" -DIR_NAME="mod_pagespeed_test/max_cacheable_content_length" -HTML_NAME="test_max_cacheable_content_length.html" -URL=$HOST_NAME/$DIR_NAME/$HTML_NAME -RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header \ - 'X-PSA-Blocking-Rewrite: psatest' $URL) -check_from "$RESPONSE_OUT" fgrep -qi small.js.pagespeed. -check_not_from "$RESPONSE_OUT" fgrep -qi large.js.pagespeed. - -# This test checks that the PageSpeedXHeaderValue directive works. -start_test PageSpeedXHeaderValue directive - -RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \ - http://xheader.example.com/mod_pagespeed_example) -check_from "$RESPONSE_OUT" fgrep -q "X-Page-Speed: UNSPECIFIED VERSION" - -# This test checks that the DomainRewriteHyperlinks directive -# can turn off. See mod_pagespeed_test/rewrite_domains.html: it has -# one URL, one URL, and one url, all referencing -# src.example.com. Only the url should be rewritten. -start_test RewriteHyperlinks off directive -HOST_NAME="http://domain-hyperlinks-off.example.com" -RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \ - $HOST_NAME/mod_pagespeed_test/rewrite_domains.html) -MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com) -check [ $MATCHES -eq 1 ] - -# This test checks that the DomainRewriteHyperlinks directive -# can turn on. See mod_pagespeed_test/rewrite_domains.html: it has -# one URL, one URL, and one url, all referencing -# src.example.com. They should all be rewritten to dst.example.com. -start_test RewriteHyperlinks on directive -HOST_NAME="http://domain-hyperlinks-on.example.com" -RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \ - $HOST_NAME/mod_pagespeed_test/rewrite_domains.html) -MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com) -check [ $MATCHES -eq 4 ] - -# Test to make sure dynamically defined url-valued attributes are rewritten by -# rewrite_domains. See mod_pagespeed_test/rewrite_domains.html: in addition to -# having one URL, one URL, and one url it also has one URL, one
URL, one
URL, and one -#
URL, all referencing src.example.com. The first three -# should be rewritten because of hardcoded rules, the span.src and hr.imgsrc -# should be rewritten because of UrlValuedAttribute directives, the hr.src -# should be left unmodified, and the blockquote.src should be rewritten as an -# image because of a UrlValuedAttribute override. The rewritten ones should all -# be rewritten to dst.example.com. -HOST_NAME="http://url-attribute.example.com" -TEST="$HOST_NAME/mod_pagespeed_test" -REWRITE_DOMAINS="$TEST/rewrite_domains.html" -UVA_EXTEND_CACHE="$TEST/url_valued_attribute_extend_cache.html" -UVA_EXTEND_CACHE+="?PageSpeedFilters=core,+left_trim_urls" - -start_test Rewrite domains in dynamically defined url-valued attributes. - -RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $REWRITE_DOMAINS) -MATCHES=$(echo "$RESPONSE_OUT" | fgrep -c http://dst.example.com) -check [ $MATCHES -eq 6 ] -MATCHES=$(echo "$RESPONSE_OUT" | \ - fgrep -c '
') -check [ $MATCHES -eq 1 ] - -start_test Additional url-valued attributes are fully respected. - -function count_exact_matches() { - # Needed because "fgrep -c" counts lines with matches, not pure matches. - fgrep -o "$1" | wc -l -} - -# There are ten resources that should be optimized -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.' 10 - -# Make sure isn't modified at all, but that everything else is -# recognized as a url and rewritten from ../foo to /foo. This means that only -# one reference to ../mod_pagespeed should remain, . -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until $UVA_EXTEND_CACHE 'grep -c d=.[.][.]/mod_pa' 1 -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until $UVA_EXTEND_CACHE 'fgrep -c ../mod_pa' 1 - -# There are ten images that should be optimized. -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until $UVA_EXTEND_CACHE 'count_exact_matches .pagespeed.ic' 10 - # Test the experiment framework (Furious). start_test PageSpeedExperiment cookie is set. @@ -1813,13 +848,13 @@ check_not_from "$OUT" fgrep 'PageSpeedExperiment=' start_test The beacon should include the experiment id. OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \ --header='Cookie: PageSpeedExperiment=2' $EXP_EXTEND_CACHE) -BEACON_CODE="pagespeed.addInstrumentationInit('/ngx_pagespeed_beacon', 'load'," +BEACON_CODE="pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load'," BEACON_CODE+=" '&exptid=2', 'http://experiment.example.com/" BEACON_CODE+="mod_pagespeed_example/extend_cache.html');" check_from "$OUT" grep "$BEACON_CODE" OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP --header='Cookie: PageSpeedExperiment=7' \ $EXP_EXTEND_CACHE) -BEACON_CODE="pagespeed.addInstrumentationInit('/ngx_pagespeed_beacon', 'load'," +BEACON_CODE="pagespeed.addInstrumentationInit('/$BEACON_HANDLER', 'load'," BEACON_CODE+=" '&exptid=7', 'http://experiment.example.com/" BEACON_CODE+="mod_pagespeed_example/extend_cache.html');" check_from "$OUT" grep "$BEACON_CODE" @@ -1903,65 +938,9 @@ function check_failures_and_exit() { exit 0 } -start_test Make sure nostore on a subdirectory is retained -URL=$TEST_ROOT/nostore/nostore.html -HTML_HEADERS=$($WGET_DUMP $URL) -check_from "$HTML_HEADERS" egrep -q \ - 'Cache-Control: max-age=0, no-cache, no-store' - -start_test Custom headers remain on resources, but cache should be 1 year. -URL="$TEST_ROOT/compressed/hello_js.custom_ext.pagespeed.ce.HdziXmtLIV.txt" -echo $WGET_DUMP $URL -RESOURCE_HEADERS=$($WGET_DUMP $URL) -check_from "$RESOURCE_HEADERS" egrep -q 'X-Extra-Header: 1' -# The extra header should only be added once, not twice. -check_not_from "$RESOURCE_HEADERS" egrep -q 'X-Extra-Header: 1, 1' -check [ "$(echo "$RESOURCE_HEADERS" | grep -c '^X-Extra-Header: 1')" = 1 ] -check_from "$RESOURCE_HEADERS" egrep -q 'Cache-Control: max-age=31536000' - -# Test critical CSS beacon injection, beacon return, and computation. This -# requires UseBeaconResultsInFilters() to be true in rewrite_driver_factory. -# NOTE: must occur after cache flush, which is why it's in this embedded -# block. The flush removes pre-existing beacon results from the pcache. -test_filter prioritize_critical_css -fetch_until -save $URL 'fgrep -c pagespeed.criticalCssBeaconInit' 1 -check [ $(fgrep -o ".very_large_class_name_" $FETCH_FILE | wc -l) -eq 36 ] -CALL_PAT=".*criticalCssBeaconInit(" -SKIP_ARG="[^,]*," -CAPTURE_ARG="'\([^']*\)'.*" -BEACON_PATH=$(sed -n "s/${CALL_PAT}${CAPTURE_ARG}/\1/p" $FETCH_FILE) -ESCAPED_URL=$( \ - sed -n "s/${CALL_PAT}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE) -OPTIONS_HASH=$( \ - sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" $FETCH_FILE) -NONCE=$( \ - sed -n "s/${CALL_PAT}${SKIP_ARG}${SKIP_ARG}${SKIP_ARG}${CAPTURE_ARG}/\1/p" \ - $FETCH_FILE) -BEACON_URL="http://${HOSTNAME}${BEACON_PATH}?url=${ESCAPED_URL}" -BEACON_DATA="oh=${OPTIONS_HASH}&n=${NONCE}&cs=.big,.blue,.bold,.foo" - -# See the comments about 204 responses and --no-http-keep-alive above. -OUT=$(wget -q --save-headers -O - --no-http-keep-alive \ - --post-data "$BEACON_DATA" "$BEACON_URL") -check_from "$OUT" grep '^HTTP/1.1 204' - -# Now make sure we see the correct critical css rules. -fetch_until $URL \ - 'grep -c ' 1 -fetch_until $URL \ - 'grep -c ' 1 -fetch_until $URL \ - 'grep -c ' 1 -fetch_until -save $URL \ - 'grep -c ' 1 -# The last one should also have the other 3, too. -check [ `grep -c '' $FETCH_UNTIL_OUTFILE` = 1 ] -check [ `grep -c '' $FETCH_UNTIL_OUTFILE` = 1 ] -check [ `grep -c '' \ - $FETCH_UNTIL_OUTFILE` = 1 ] - -# Now repeat the critical_css_filter test on a host that processes post data via +# Repeat the critical_css_filter test on a host that processes post data via # temp files to test that ngx_pagespeed specific code path. +filter_spec_method="headers" test_filter prioritize_critical_css Able to read POST data from temp file. URL="http://beacon-post-temp-file.example.com/mod_pagespeed_example/prioritize_critical_css.html" http_proxy=$SECONDARY_HOSTNAME\ @@ -2005,202 +984,6 @@ check [ `grep -c '' $FETCH_UNTIL_OUTFILE` = 1 ] check [ `grep -c '' \ $FETCH_UNTIL_OUTFILE` = 1 ] -# This test checks that the ClientDomainRewrite directive can turn on. -start_test ClientDomainRewrite on directive -HOST_NAME="http://client-domain-rewrite.example.com" -RESPONSE_OUT=$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP \ - $HOST_NAME/mod_pagespeed_test/rewrite_domains.html) -MATCHES=$(echo "$RESPONSE_OUT" | grep -c pagespeed\.clientDomainRewriterInit) -check [ $MATCHES -eq 1 ] - -# Verify rendered image dimensions test. -start_test resize_rendered_image_dimensions with critical images beacon -HOST_NAME="http://renderedimagebeacon.example.com" -URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/image_resize_using_rendered_dimensions.html" -http_proxy=$SECONDARY_HOSTNAME\ - fetch_until -save -recursive $URL 'fgrep -c "pagespeed_url_hash"' 2 \ - '--header=X-PSA-Blocking-Rewrite:psatest' -check [ $(grep -c "^pagespeed\.CriticalImages\.Run" \ - $WGET_DIR/image_resize_using_rendered_dimensions.html) = 1 ]; -OPTIONS_HASH=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' \ - $WGET_DIR/image_resize_using_rendered_dimensions.html) -NONCE=$(awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' \ - $WGET_DIR/image_resize_using_rendered_dimensions.html) - -# Send a beacon response using POST indicating that OptPuzzle.jpg is -# critical and has rendered dimensions. -BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon" -BEACON_URL+="?url=http%3A%2F%2Frenderedimagebeacon.example.com%2Fmod_pagespeed_test%2F" -BEACON_URL+="image_rewriting%2Fimage_resize_using_rendered_dimensions.html" -BEACON_DATA="oh=$OPTIONS_HASH&n=$NONCE&ci=1344500982&rd=%7B%221344500982%22%3A%7B%22rw%22%3A150%2C%22rh%22%3A100%2C%22ow%22%3A256%2C%22oh%22%3A192%7D%7D" -OUT=$(env http_proxy=$SECONDARY_HOSTNAME \ - $WGET_DUMP --no-http-keep-alive --post-data "$BEACON_DATA" "$BEACON_URL") -check_from "$OUT" egrep -q "HTTP/1[.]. 204" -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until -save -recursive $URL \ - 'fgrep -c 150x100xOptPuzzle.jpg.pagespeed.ic.' 1 - -# Verify that downstream caches and rebeaconing interact correctly for images. -start_test lazyload_images,rewrite_images with downstream cache rebeaconing -HOST_NAME="http://downstreamcacherebeacon.example.com" -URL="$HOST_NAME/mod_pagespeed_test/downstream_caching.html" -URL+="?PageSpeedFilters=lazyload_images" -# 1. Even with blocking rewrite, we don't get an instrumented page when the -# PS-ShouldBeacon header is missing. -OUT1=$(http_proxy=$SECONDARY_HOSTNAME \ - $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL) -check_not_from "$OUT1" egrep -q 'pagespeed\.CriticalImages\.Run' -check_from "$OUT1" grep -q "Cache-Control: private, max-age=3000" -# 2. We get an instrumented page if the correct key is present. -OUT2=$(http_proxy=$SECONDARY_HOSTNAME \ - $WGET_DUMP $WGET_ARGS \ - --header="X-PSA-Blocking-Rewrite: psatest" \ - --header="PS-ShouldBeacon: random_rebeaconing_key" $URL) -check_from "$OUT2" egrep -q "pagespeed\.CriticalImages\.Run" -check_from "$OUT2" grep -q "Cache-Control: max-age=0, no-cache" -# 3. We do not get an instrumented page if the wrong key is present. -OUT3=$(http_proxy=$SECONDARY_HOSTNAME \ - $WGET_DUMP $WGET_ARGS \ - --header="X-PSA-Blocking-Rewrite: psatest" \ - --header="PS-ShouldBeacon: wrong_rebeaconing_key" $URL) -check_not_from "$OUT3" egrep -q "pagespeed\.CriticalImages\.Run" -check_from "$OUT3" grep -q "Cache-Control: private, max-age=3000" - -# Verify that downstream caches and rebeaconing interact correctly for css. -test_filter prioritize_critical_css with rebeaconing -HOST_NAME="http://downstreamcacherebeacon.example.com" -URL="$HOST_NAME/mod_pagespeed_test/downstream_caching.html" -URL+="?PageSpeedFilters=prioritize_critical_css" -# 1. Even with blocking rewrite, we don't get an instrumented page when the -# PS-ShouldBeacon header is missing. -OUT1=$(http_proxy=$SECONDARY_HOSTNAME \ - $WGET_DUMP --header 'X-PSA-Blocking-Rewrite: psatest' $URL) -check_not_from "$OUT1" egrep -q 'pagespeed\.criticalCssBeaconInit' -check_from "$OUT1" grep -q "Cache-Control: private, max-age=3000" - -# 2. We get an instrumented page if the correct key is present. -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until -save $URL 'grep -c criticalCssBeaconInit' 2 \ - "--header=PS-ShouldBeacon:random_rebeaconing_key --save-headers" -check grep -q "Cache-Control: max-age=0, no-cache" $FETCH_UNTIL_OUTFILE - -# 3. We do not get an instrumented page if the wrong key is present. -WGET_ARGS="--header=\"PS-ShouldBeacon: wrong_rebeaconing_key\"" -OUT3=$(http_proxy=$SECONDARY_HOSTNAME check_not \ - $WGET_DUMP $WGET_ARGS $URL) -check_not_from "$OUT3" egrep -q "pagespeed\.criticalCssBeaconInit" -check_from "$OUT3" grep -q "Cache-Control: private, max-age=3000" - -# Verify that we can send a critical image beacon and that lazyload_images -# does not try to lazyload the critical images. -start_test lazyload_images,rewrite_images with critical images beacon -HOST_NAME="http://imagebeacon.example.com" -URL="$HOST_NAME/mod_pagespeed_test/image_rewriting/rewrite_images.html" -# There are 3 images on rewrite_images.html. Since beaconing is on but we've -# sent no beacon data, none should be lazy loaded. -# Run until we see beaconing on the page (should happen on first visit). -http_proxy=$SECONDARY_HOSTNAME\ - fetch_until -save $URL \ - 'fgrep -c "pagespeed.CriticalImages.Run"' 1 -check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ]; -# We need the options hash and nonce to send a critical image beacon, so extract -# it from injected beacon JS. -OPTIONS_HASH=$( - awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE) -NONCE=$( - awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE) -# Send a beacon response using POST indicating that Puzzle.jpg is a critical -# image. -BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon" -BEACON_URL+="?url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F" -BEACON_URL+="image_rewriting%2Frewrite_images.html" -BEACON_DATA="oh=$OPTIONS_HASH&n=$NONCE&ci=2932493096" -# See the comments about 204 responses and --no-http-keep-alive above. -OUT=$(env http_proxy=$SECONDARY_HOSTNAME \ - wget -q --save-headers -O - --no-http-keep-alive \ - --post-data "$BEACON_DATA" "$BEACON_URL") -check_from "$OUT" egrep -q "HTTP/1[.]. 204" -# Now 2 of the images should be lazyloaded, Puzzle.jpg should not be. -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 2 - -# Now test sending a beacon with a GET request, instead of POST. Indicate that -# Puzzle.jpg and Cuppa.png are the critical images. In practice we expect only -# POSTs to be used by the critical image beacon, but both code paths are -# supported. We add query params to URL to ensure that we get an instrumented -# page without blocking. -URL="$URL?id=4" -http_proxy=$SECONDARY_HOSTNAME\ - fetch_until -save $URL \ - 'fgrep -c "pagespeed.CriticalImages.Run"' 1 -check [ $(grep -c "pagespeed_lazy_src=" $FETCH_FILE) = 0 ]; -OPTIONS_HASH=$( - awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-3)}' $FETCH_FILE) -NONCE=$( - awk -F\' '/^pagespeed\.CriticalImages\.Run/ {print $(NF-1)}' $FETCH_FILE) -BEACON_URL="$HOST_NAME/ngx_pagespeed_beacon" -BEACON_URL+="?url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F" -BEACON_URL+="image_rewriting%2Frewrite_images.html%3Fid%3D4" -BEACON_DATA="oh=$OPTIONS_HASH&n=$NONCE&ci=2932493096" -# Add the hash for Cuppa.png to BEACON_DATA, which will be used as the query -# params for the GET. -BEACON_DATA+=",2644480723" -OUT=$(env http_proxy=$SECONDARY_HOSTNAME \ - $WGET_DUMP "$BEACON_URL&$BEACON_DATA") -check_from "$OUT" egrep -q "HTTP/1[.]. 204" -# Now only BikeCrashIcn.png should be lazyloaded. -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until -save -recursive $URL 'fgrep -c pagespeed_lazy_src=' 1 - -test_filter prioritize_critical_css with unauthorized resources - -start_test no critical selectors chosen from unauthorized resources -URL="$TEST_ROOT/unauthorized/prioritize_critical_css.html" -URL+="?PageSpeedFilters=prioritize_critical_css,debug" -fetch_until -save $URL 'fgrep -c pagespeed.criticalCssBeaconInit' 3 -# Except for the occurrence in html, the gsc-completion-selected string -# should not occur anywhere else, i.e. in the selector list. -check [ $(fgrep -c "gsc-completion-selected" $FETCH_FILE) -eq 1 ] -# From the css file containing an unauthorized @import line, -# a) no selectors from the unauthorized @ import (e.g .maia-display) should -# appear in the selector list. -check_not fgrep -q "maia-display" $FETCH_FILE -# b) no selectors from the authorized @ import (e.g .interesting_color) should -# appear in the selector list because it won't be flattened. -check_not fgrep -q "interesting_color" $FETCH_FILE -# c) selectors that don't depend on flattening should appear in the selector -# list. -check [ $(fgrep -c "non_flattened_selector" $FETCH_FILE) -eq 1 ] -EXPECTED_IMPORT_FAILURE_LINE="" -check [ $(grep -o "$EXPECTED_IMPORT_FAILURE_LINE" $FETCH_FILE | wc -l) -eq 1 ] -EXPECTED_COMMENT_LINE="" -check [ $(grep -o "$EXPECTED_COMMENT_LINE" $FETCH_FILE | wc -l) -eq 1 ] - -start_test inline_unauthorized_resources allows unauthorized css selectors -HOST_NAME="http://unauthorizedresources.example.com" -URL="$HOST_NAME/mod_pagespeed_test/unauthorized/prioritize_critical_css.html" -URL+="?PageSpeedFilters=prioritize_critical_css,debug" -# gsc-completion-selected string should occur once in the html and once in the -# selector list. -http_proxy=$SECONDARY_HOSTNAME \ - fetch_until -save $URL 'fgrep -c gsc-completion-selected' 2 -# Verify that this page had beaconing javascript on it. -check [ $(fgrep -c "pagespeed.criticalCssBeaconInit" $FETCH_FILE) -eq 3 ] -# From the css file containing an unauthorized @import line, -# a) no selectors from the unauthorized @ import (e.g .maia-display) should -# appear in the selector list. -check_not fgrep -q "maia-display" $FETCH_FILE -# b) no selectors from the authorized @ import (e.g .red) should -# appear in the selector list because it won't be flattened. -check_not fgrep -q "interesting_color" $FETCH_FILE -# c) selectors that don't depend on flattening should appear in the selector -# list. -check [ $(fgrep -c "non_flattened_selector" $FETCH_FILE) -eq 1 ] -check grep -q "$EXPECTED_IMPORT_FAILURE_LINE" $FETCH_FILE - - start_test keepalive with html rewriting keepalive_test "keepalive-html.example.com"\ "/mod_pagespeed_example/rewrite_images.html" "" @@ -2213,52 +996,20 @@ keepalive_test "keepalive-resource.example.com"\ BEACON_URL="http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F" start_test keepalive with beacon get requests keepalive_test "keepalive-beacon-get.example.com"\ - "/ngx_pagespeed_beacon?ets=load:13&url=$BEACON_URL" "" + "/$BEACON_HANDLER?ets=load:13&url=$BEACON_URL" "" BEACON_DATA="url=http%3A%2F%2Fimagebeacon.example.com%2Fmod_pagespeed_test%2F" BEACON_DATA+="image_rewriting%2Frewrite_images.html" BEACON_DATA+="&oh=$OPTIONS_HASH&ci=2932493096" start_test keepalive with beacon post requests -keepalive_test "keepalive-beacon-post.example.com" "/ngx_pagespeed_beacon"\ +keepalive_test "keepalive-beacon-post.example.com" "/$BEACON_HANDLER"\ "$BEACON_DATA" start_test keepalive with static resources keepalive_test "keepalive-static.example.com"\ "/pagespeed_custom_static/js_defer.0.js" "" -# Test for MaxCombinedCssBytes. The html used in the test, 'combine_css.html', -# has 4 CSS files in the following order. -# yellow.css : 36 bytes -# blue.css : 21 bytes -# big.css : 4307 bytes -# bold.css : 31 bytes -# Because the threshold was chosen as '57', only the first two CSS files -# are combined. -test_filter combine_css Maximum size of combined CSS. -QUERY_PARAM="PageSpeedMaxCombinedCssBytes=57" -URL="$URL&$QUERY_PARAM" -# We should get the first two files to be combined... -fetch_until -save $URL 'grep -c styles/yellow.css+blue.css.pagespeed.' 1 -# ... but 3rd and 4th should be standalone -check [ $(grep -c 'styles/bold.css\"' $FETCH_UNTIL_OUTFILE) = 1 ] -check [ $(grep -c 'styles/big.css\"' $FETCH_UNTIL_OUTFILE) = 1 ] - -# Test to make sure we have a sane Connection Header. See -# https://code.google.com/p/modpagespeed/issues/detail?id=664 -# -# Note that this bug is dependent on seeing a resource for the first time in the -# InPlaceResourceOptimization path, because in that flow we are caching the -# response-headers from the server. The reponse-headers from Serf never seem to -# include the Connection header. So we have to pick a JS file that is not -# otherwise used after cache is flushed in this block. -start_test Sane Connection header -URL="$TEST_ROOT/normal.js" -fetch_until -save $URL 'grep -c W/\"PSA-aj-' 1 --save-headers -CONNECTION=$(extract_headers $FETCH_UNTIL_OUTFILE | fgrep "Connection:") -check_not_from "$CONNECTION" fgrep -qi "Keep-Alive, Keep-Alive" -check_from "$CONNECTION" fgrep -qi "Keep-Alive" - start_test pagespeed_custom_static defer js served with correct headers. # First, determine which hash js_defer is served with. We need a correct hash # to get it served up with an Etag, which is one of the things we want to test. @@ -2267,194 +1018,6 @@ OUT=$($WGET_DUMP $URL) HASH=$(echo $OUT \ | grep --only-matching "/js_defer\\.*\([^.]\)*.js" | cut -d '.' -f 2) -# Test a scenario where a multi-domain installation is using a -# single CDN for all hosts, and uses a subdirectory in the CDN to -# distinguish hosts. Some of the resources may already be mapped to -# the CDN in the origin HTML, but we want to fetch them directly -# from localhost. If we do this successfully (see the MapOriginDomain -# command in customhostheader.example.com in pagespeed conf), we will -# inline a small image. -start_test shared CDN short-circuit back to origin via host-header override -URL="http://customhostheader.example.com/map_origin_host_header.html" -http_proxy=$SECONDARY_HOSTNAME fetch_until -save "$URL" \ - "grep -c data:image/png;base64" 1 - -# Optimize in-place images for browser. Ideal test matrix (not covered yet): -# User-Agent: Accept: Image type Result -# ----------- ------- ---------- ---------------------------------- -# IE N/A photo image/jpeg, Cache-Control: private * -# : N/A synthetic image/png, no vary -# Old Opera no photo image/jpeg, Vary: Accept -# : no synthetic image/png, no vary -# : webp photo image/webp, Vary: Accept, Lossy -# : webp synthetic image/png, no vary -# Chrome or no photo image/jpeg, Vary: Accept -# Firefox or no synthetic image/png, no vary -# New Opera webp photo image/webp, Vary: Accept, Lossy -# : webp synthetic image/webp, no vary -# TODO(jmaessen): * cases currently send Vary: Accept. Fix (in progress). -# + has been rejected for now in favor of image/png, Vary: Accept. -# TODO(jmaessen): Send image/webp lossless for synthetic and alpha-channel -# images. Will require reverting to Vary: Accept for these. Stuff like -# animated webp will have to remain unconverted still in IPRO mode, or switch -# to cc: private, but right now animated webp support is still pending anyway. -function test_ipro_for_browser_webp() { - IN_UA_PRETTY="$1"; shift - IN_UA="$1"; shift - IN_ACCEPT="$1"; shift - IMAGE_TYPE="$1"; shift - OUT_CONTENT_TYPE="$1"; shift - OUT_VARY="${1-}"; shift || true - OUT_CC="${1-}"; shift || true - # Remaining args are the expected headers (Name:Value), photo, or synthetic. - if [ "$IMAGE_TYPE" = "photo" ]; then - URL="http://ipro-for-browser.example.com/images/Puzzle.jpg" - else - URL="http://ipro-for-browser.example.com/images/Cuppa.png" - fi - TEST_ID="In-place optimize for " - TEST_ID+="User-Agent:${IN_UA_PRETTY:-${IN_UA:-None}}," - if [ -z "$IN_ACCEPT" ]; then - TEST_ID+=" no accept, " - else - TEST_ID+=" Accept:$IN_ACCEPT, " - fi - TEST_ID+=" $IMAGE_TYPE. Expect image/${OUT_CONTENT_TYPE}, " - if [ -z "$OUT_VARY" ]; then - TEST_ID+=" no vary, " - else - TEST_ID+=" Vary:${OUT_VARY}, " - fi - if [ -z "$OUT_CC" ]; then - TEST_ID+=" cacheable." - else - TEST_ID+=" Cache-Control:${OUT_CC}." - fi - start_test $TEST_ID - WGET_ARGS="--save-headers \ - ${IN_UA:+--user-agent $IN_UA} \ - ${IN_ACCEPT:+--header=Accept:image/$IN_ACCEPT}" - http_proxy=$SECONDARY_HOSTNAME \ - fetch_until -save $URL 'grep -c W/\"PSA-aj-' 1 - check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \ - fgrep -q "Content-Type: image/$OUT_CONTENT_TYPE" - if [ -z "$OUT_VARY" ]; then - check_not_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \ - fgrep -q "Vary:" - else - check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \ - fgrep -q "Vary: $OUT_VARY" - fi - check_from "$(extract_headers $FETCH_UNTIL_OUTFILE)" \ - grep -q "Cache-Control: ${OUT_CC:-max-age=[0-9]*}$" - # TODO: check file type of webp. Irrelevant for now. -} - -############################################################################## -# Test with testing-only user agent strings. -# UA Accept Type Out Vary CC -test_ipro_for_browser_webp "None" "" "" photo jpeg "Accept" -test_ipro_for_browser_webp "" "webp" "" photo jpeg "Accept" -test_ipro_for_browser_webp "" "webp-la" "" photo jpeg "Accept" -test_ipro_for_browser_webp "None" "" "webp" photo webp "Accept" -test_ipro_for_browser_webp "" "webp" "webp" photo webp "Accept" -test_ipro_for_browser_webp "" "webp-la" "webp" photo webp "Accept" -test_ipro_for_browser_webp "None" "" "" synth png -test_ipro_for_browser_webp "" "webp" "" synth png -test_ipro_for_browser_webp "" "webp-la" "" synth png -test_ipro_for_browser_webp "None" "" "webp" synth png -test_ipro_for_browser_webp "" "webp" "webp" synth png -test_ipro_for_browser_webp "" "webp-la" "webp" synth png -############################################################################## - -# Wordy UAs need to be stored in the WGETRC file to avoid death by quoting. -OLD_WGETRC=$WGETRC -WGETRC=$TESTTMP/wgetrc-ua -export WGETRC - -# IE 9 and later must re-validate Vary: Accept. We should send CC: private. -IE9_UA="Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))" -IE11_UA="Mozilla/5.0 (Windows NT 6.1; WOW64; ***********; rv:11.0) like Gecko" -echo "user_agent = $IE9_UA" > $WGETRC -# (no accept) Type Out Vary CC -test_ipro_for_browser_webp "IE 9" "" "" photo jpeg "" "max-age=[0-9]*,private" -test_ipro_for_browser_webp "IE 9" "" "" synth png -echo "user_agent = $IE11_UA" > $WGETRC -test_ipro_for_browser_webp "IE 11" "" "" photo jpeg "" "max-age=[0-9]*,private" -test_ipro_for_browser_webp "IE 11" "" "" synth png - -# Older Opera did not support webp. -OPERA_UA="Opera/9.80 (Windows NT 5.2; U; en) Presto/2.7.62 Version/11.01" -echo "user_agent = $OPERA_UA" > $WGETRC -# (no accept) Type Out Vary -test_ipro_for_browser_webp "Old Opera" "" "" photo jpeg "Accept" -test_ipro_for_browser_webp "Old Opera" "" "" synth png -# Slightly newer opera supports only lossy webp, sends header. -OPERA_UA="Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10" -echo "user_agent = $OPERA_UA" > $WGETRC -# Accept Type Out Vary -test_ipro_for_browser_webp "Newer Opera" "" "webp" photo webp "Accept" -test_ipro_for_browser_webp "Newer Opera" "" "webp" synth png - -function test_decent_browsers() { - echo "user_agent = $2" > $WGETRC - # UA Accept Type Out Vary - test_ipro_for_browser_webp "$1" "" "" photo jpeg "Accept" - test_ipro_for_browser_webp "$1" "" "" synthetic png - test_ipro_for_browser_webp "$1" "" "webp" photo webp "Accept" - test_ipro_for_browser_webp "$1" "" "webp" synthetic png -} -CHROME_UA="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 " -CHROME_UA+="(KHTML, like Gecko) Chrome/32.0.1700.102 Safari/537.36" -test_decent_browsers "Chrome" "$CHROME_UA" -FIREFOX_UA="Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) " -FIREFOX_UA+="Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10" -test_decent_browsers "Firefox" "$FIREFOX_UA" -test_decent_browsers "New Opera" \ - "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14" - -WGETRC=$OLD_WGETRC - -start_test Request Option Override : Correct values are passed -HOST_NAME="http://request-option-override.example.com" -OPTS="?ModPagespeed=on" -OPTS+="&ModPagespeedFilters=+collapse_whitespace,+remove_comments" -OPTS+="&PageSpeedRequestOptionOverride=abc" -URL="$HOST_NAME/mod_pagespeed_test/forbidden.html$OPTS" -OUT="$(http_proxy=$SECONDARY_HOSTNAME $WGET_DUMP $URL)" -echo wget $URL -check_not_from "$OUT" grep -q '