From 51ebf51b4e1eaa837fa03c666de151a52292fe0c Mon Sep 17 00:00:00 2001 From: Jeff Kaufman Date: Thu, 28 Mar 2013 13:39:03 -0400 Subject: [PATCH] system-test: infrastructure, host validation We are going to need some nginx-specific system tests, and this seemed like a good place to start. This adds two tests, as well as some infrastructure. With Squash-merge of my #194 and #195 --- .gitignore | 1 + README.md | 90 +++---------- test/nginx_system_test.sh | 96 +++++++++++++- test/pagespeed_test.conf.template | 203 ++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 78 deletions(-) create mode 100644 .gitignore create mode 100644 test/pagespeed_test.conf.template diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..8d5457d38 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +test/pagespeed_test.conf diff --git a/README.md b/README.md index 341aee242..85ca4833c 100644 --- a/README.md +++ b/README.md @@ -201,77 +201,25 @@ you can set your beacons to go to another site by specifying a full path: ### Testing The generic Pagespeed system test is ported, and all but three tests pass. To -run it you need to first build and configure nginx. Set it up something like: +run it you need to first build nginx. You also need to check out mod_pagespeed, +but we can take a shortcut and do this the easy way, without gyp, because we +don't need any dependencies: - ... - http { - pagespeed on; + $ svn checkout https://modpagespeed.googlecode.com/svn/trunk/ mod_pagespeed - // TODO(jefftk): this should be the default. - pagespeed RewriteLevel CoreFilters; +Then run: - # This can be anywhere on your filesystem. - pagespeed FileCachePath /path/to/ngx_pagespeed_cache; + test/nginx_system_test.sh \ + primary_port \ + secondary_port \ + mod_pagespeed_dir \ + file_cache_path \ + nginx_executable_path - # For testing that the Library command works. - pagespeed Library 43 1o978_K0_LNE5_ystNklf - http://www.modpagespeed.com/rewrite_javascript.js; +For example: - # These gzip options are needed for tests that assume that pagespeed - # always enables gzip. Which it does in apache, but not in nginx. - gzip on; - gzip_vary on; - - # Turn on gzip for all content types that should benefit from it. - gzip_types application/ecmascript; - gzip_types application/javascript; - gzip_types application/json; - gzip_types application/pdf; - gzip_types application/postscript; - gzip_types application/x-javascript; - gzip_types image/svg+xml; - gzip_types text/css; - gzip_types text/csv; - # "gzip_types text/html" is assumed. - gzip_types text/javascript; - gzip_types text/plain; - gzip_types text/xml; - - gzip_http_version 1.0; - - ... - - server { - listen 8050; - server_name localhost; - root /path/to/mod_pagespeed/src/install; - index index.html; - - add_header Cache-Control "public, max-age=600"; - - # Disable parsing if the size of the HTML exceeds 50kB. - pagespeed MaxHtmlParseBytes 50000; - - location /mod_pagespeed_test/no_cache/ { - add_header Cache-Control no-cache; - } - - location /mod_pagespeed_test/compressed/ { - add_header Cache-Control max-age=600; - add_header Content-Encoding gzip; - types { - text/javascript custom_ext; - } - } - - ... - } - } - -Then run the test, using the port you set up with `listen` in the configuration -file: - - /path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050 + $ test/nginx_system_test.sh 8050 8051 /path/to/mod_pagespeed \ + /path/to/ngx_pagespeed_cache /path/to/sbin/nginx This should print out a lot of lines like: @@ -302,16 +250,14 @@ you to [submit a bug](https://github.com/pagespeed/ngx_pagespeed/issues/new). Start an memcached server: - memcached -p 11213 + memcached -p 11211 -To the configuration above add to the main or server block: +In `ngx_pagespeed/test/pagespeed_test.conf.template` uncomment: - pagespeed MemcachedServers "localhost:11213"; + pagespeed MemcachedServers "localhost:11211"; pagespeed MemcachedThreads 1; -Then run the system test: - - /path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050 +Then run the system test as above. #### Testing with valgrind diff --git a/test/nginx_system_test.sh b/test/nginx_system_test.sh index b556eb50d..c86ccbfb7 100755 --- a/test/nginx_system_test.sh +++ b/test/nginx_system_test.sh @@ -25,16 +25,74 @@ # Exits with status 2 if command line args are wrong. # # Usage: -# Set up nginx to serve mod_pagespeed/src/install/ statically at the server -# root, then run: -# ./ngx_system_test.sh HOST:PORT +# ./ngx_system_test.sh primary_port secondary_port \ +# mod_pagespeed_dir file_cache_path # for example: -# ./ngx_system_test.sh localhost:8050 +# ./ngx_system_test.sh 8050 8051 \ +# /path/to/mod_pagespeed \ +# /path.to/ngx_pagespeed_cache # -this_dir="$( dirname "$0" )" +if [ "$#" -ne 5 ] ; then + echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir" + echo " file_cache_path nginx_executable" + exit 1 +fi -SYSTEM_TEST_FILE="$this_dir/../../mod_pagespeed/src/install/system_test.sh" +PRIMARY_PORT="$1" +SECONDARY_PORT="$2" +MOD_PAGESPEED_DIR="$3" +FILE_CACHE_PATH="$4" +NGINX_EXECUTABLE="$5" + +PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT" +SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT" + +SERVER_ROOT="$MOD_PAGESPEED_DIR/src/install/" + +# We need check and check_not before we source SYSTEM_TEST_FILE that provides +# them. +function handle_failure_simple() { + echo "FAIL" + exit 2 +} +function check_simple() { + echo " check" "$@" + "$@" || handle_failure_simple +} +function check_not_simple() { + echo " check_not" "$@" + "$@" && handle_failure_simple +} + +this_dir="$( cd $(dirname "$0") && pwd)" + +# set up the config file for the test +PAGESPEED_CONF="$this_dir/pagespeed_test.conf" +PAGESPEED_CONF_TEMPLATE="$PAGESPEED_CONF.template" +# check for config file template +check_simple test -e "$PAGESPEED_CONF_TEMPLATE" +# create PAGESPEED_CONF by substituting on PAGESPEED_CONF_TEMPLATE +echo > $PAGESPEED_CONF <> $PAGESPEED_CONF +# make sure we substituted all the variables +check_not_simple grep @@ $PAGESPEED_CONF + +# restart nginx with new config +killall nginx +sleep .1 +check_simple "$NGINX_EXECUTABLE" -c "$PAGESPEED_CONF" + +# run generic system tests +SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/install/system_test.sh" if [ ! -e "$SYSTEM_TEST_FILE" ] ; then echo "Not finding $SYSTEM_TEST_FILE -- is mod_pagespeed not in a parallel" @@ -51,5 +109,31 @@ PAGESPEED_EXPECTED_FAILURES=" ~In-place resource optimization~ " +# The existing system test takes its arguments as positional parameters, and +# wants different ones than we want, so we need to reset our positional args. +set -- "$PRIMARY_HOSTNAME" source $SYSTEM_TEST_FILE + +# nginx-specific system tests + +# When we allow ourself to fetch a resource because the Host header tells us +# that it is one of our resources, we should be fetching it from ourself. +start_test Loopback fetches go to local IPs without DNS lookup + +# If we're properly fetching from ourself we will issue loopback fetches for +# /mod_pagespeed_example/combine_javascriptN.js, which will succeed, so +# combining will work. If we're taking 'Host:www.google.com' to mean that we +# should fetch from www.google.com then those fetches will fail because +# google.com won't have /mod_pagespeed_example/combine_javascriptN.js and so +# we'll not rewrite any resources. + +URL="$HOSTNAME/mod_pagespeed_example/combine_javascript.html" +URL+="?ModPagespeed=on&ModPagespeedFilters=combine_javascript" +fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com + +# If this accepts the Host header and fetches from google.com it will fail with +# a 404. Instead it should use a loopback fetch and succeed. +URL="$HOSTNAME/mod_pagespeed_example/.pagespeed.ce.8CfGBvwDhH.css" +check wget -O /dev/null --header=Host:www.google.com "$URL" + check_failures_and_exit diff --git a/test/pagespeed_test.conf.template b/test/pagespeed_test.conf.template new file mode 100644 index 000000000..95c8f2ac3 --- /dev/null +++ b/test/pagespeed_test.conf.template @@ -0,0 +1,203 @@ +# nginx_system_test.sh makes a few substitutions to this file to generate +# pagespeed_test.conf + +#user nobody; +worker_processes 1; + +# for debugging +#daemon off; +#master_process off; + +#error_log logs/error.log; +#error_log logs/error.log notice; +#error_log logs/error.log info; +error_log logs/error.log debug; + +#pid logs/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + + root "@@SERVER_ROOT@@"; + + server { + listen @@PRIMARY_PORT@@; + server_name localhost; + pagespeed FileCachePath "@@FILE_CACHE_PATH@@"; + + # uncomment the following two lines if you're testing memcached + #pagespeed MemcachedServers "localhost:11211"; + #pagespeed MemcachedThreads 1; + + pagespeed on; + + pagespeed RewriteLevel CoreFilters; + pagespeed EnableFilters insert_ga,trim_urls; + + #pagespeed CacheFlushPollIntervalSec 1; + + #pagespeed RunExperiment on; + #pagespeed AnalyticsID "UA-XXXXXXXX-Y"; + #pagespeed ExperimentSpec "id=1;percent=50;default"; + #pagespeed ExperimentSpec "id=2;percent=50"; + + pagespeed Library 43 1o978_K0_LNE5_ystNklf + http://www.modpagespeed.com/rewrite_javascript.js; + + # Disable parsing if the size of the HTML exceeds 50kB. + pagespeed MaxHtmlParseBytes 50000; + + + location /mod_pagespeed_example/core_filters/ { + pagespeed RewriteLevel CoreFilters; + } + + location /mod_pagespeed_example/pass_through/ { + pagespeed RewriteLevel PassThrough; + } + + #location / { + # proxy_pass http://www.google.com; + #} + + location /mod_pagespeed_test/no_cache/ { + add_header Cache-Control no-cache; + } + + location /mod_pagespeed_test/compressed/ { + add_header Content-Encoding gzip; + types { + text/javascript custom_ext; + } + } + + pagespeed EnableFilters remove_comments; + + #charset koi8-r; + + #access_log logs/host.access.log main; + + index index.html; + + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + } + + sendfile on; + + keepalive_timeout 65; + + # set up gzip + gzip on; + gzip_vary on; + # Turn on gzip for all content types that should benefit from it. + gzip_types application/ecmascript; + gzip_types application/javascript; + gzip_types application/json; + gzip_types application/pdf; + gzip_types application/postscript; + gzip_types application/x-javascript; + gzip_types image/svg+xml; + gzip_types text/css; + gzip_types text/csv; + # "gzip_types text/html" is assumed. + gzip_types text/javascript; + gzip_types text/plain; + gzip_types text/xml; + + gzip_http_version 1.0; + + types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/x-javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + image/svg+xml svg svgz; + image/webp webp; + + application/java-archive jar war ear; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.ms-excel xls; + application/vnd.ms-powerpoint ppt; + application/vnd.wap.wmlc wmlc; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream eot; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; + } + default_type application/octet-stream; + +}