artox/incubator-pagespeed-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

system-test: infrastructure, host validation

Browse Source 
We are going to need some nginx-specific system tests, and this seemed like a
good place to start. This adds two tests, as well as some infrastructure. With

Squash-merge of my #194 and #195
This commit is contained in:
Jeff Kaufman
2013-03-28 13:39:03 -04:00
parent 057038d20f
commit 51ebf51b4e
4 changed files with 312 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -0,0 +1 @@
test/pagespeed_test.conf
README.md
+18 -72
View File
@@ -201,77 +201,25 @@ you can set your beacons to go to another site by specifying a full path:
### Testing
The generic Pagespeed system test is ported, and all but three tests pass. To
run it you need to first build and configure nginx. Set it up something like:
run it you need to first build nginx. You also need to check out mod_pagespeed,
but we can take a shortcut and do this the easy way, without gyp, because we
don't need any dependencies:
...
http {
pagespeed on;
$ svn checkout https://modpagespeed.googlecode.com/svn/trunk/ mod_pagespeed
// TODO(jefftk): this should be the default.
pagespeed RewriteLevel CoreFilters;
Then run:
# This can be anywhere on your filesystem.
pagespeed FileCachePath /path/to/ngx_pagespeed_cache;
test/nginx_system_test.sh \
primary_port \
secondary_port \
mod_pagespeed_dir \
file_cache_path \
nginx_executable_path
# For testing that the Library command works.
pagespeed Library 43 1o978_K0_LNE5_ystNklf
http://www.modpagespeed.com/rewrite_javascript.js;
For example:
# These gzip options are needed for tests that assume that pagespeed
# always enables gzip. Which it does in apache, but not in nginx.
gzip on;
gzip_vary on;
# Turn on gzip for all content types that should benefit from it.
gzip_types application/ecmascript;
gzip_types application/javascript;
gzip_types application/json;
gzip_types application/pdf;
gzip_types application/postscript;
gzip_types application/x-javascript;
gzip_types image/svg+xml;
gzip_types text/css;
gzip_types text/csv;
# "gzip_types text/html" is assumed.
gzip_types text/javascript;
gzip_types text/plain;
gzip_types text/xml;
gzip_http_version 1.0;
...
server {
listen 8050;
server_name localhost;
root /path/to/mod_pagespeed/src/install;
index index.html;
add_header Cache-Control "public, max-age=600";
# Disable parsing if the size of the HTML exceeds 50kB.
pagespeed MaxHtmlParseBytes 50000;
location /mod_pagespeed_test/no_cache/ {
add_header Cache-Control no-cache;
}
location /mod_pagespeed_test/compressed/ {
add_header Cache-Control max-age=600;
add_header Content-Encoding gzip;
types {
text/javascript custom_ext;
}
}
...
}
}
Then run the test, using the port you set up with `listen` in the configuration
file:
/path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050
$ test/nginx_system_test.sh 8050 8051 /path/to/mod_pagespeed \
/path/to/ngx_pagespeed_cache /path/to/sbin/nginx
This should print out a lot of lines like:
@@ -302,16 +250,14 @@ you to [submit a bug](https://github.com/pagespeed/ngx_pagespeed/issues/new).
Start an memcached server:
memcached -p 11213
memcached -p 11211
To the configuration above add to the main or server block:
In `ngx_pagespeed/test/pagespeed_test.conf.template` uncomment:
pagespeed MemcachedServers "localhost:11213";
pagespeed MemcachedServers "localhost:11211";
pagespeed MemcachedThreads 1;
Then run the system test:
/path/to/ngx_pagespeed/test/nginx_system_test.sh localhost:8050
Then run the system test as above.
#### Testing with valgrind
test/nginx_system_test.sh
+90 -6
View File
@@ -25,16 +25,74 @@
# Exits with status 2 if command line args are wrong.
#
# Usage:
# Set up nginx to serve mod_pagespeed/src/install/ statically at the server
# root, then run:
# ./ngx_system_test.sh HOST:PORT
# ./ngx_system_test.sh primary_port secondary_port \
# mod_pagespeed_dir file_cache_path
# for example:
# ./ngx_system_test.sh localhost:8050
# ./ngx_system_test.sh 8050 8051 \
# /path/to/mod_pagespeed \
# /path.to/ngx_pagespeed_cache
#
this_dir="$( dirname "$0" )"
if [ "$#" -ne 5 ] ; then
echo "Usage: $0 primary_port secondary_port mod_pagespeed_dir"
echo " file_cache_path nginx_executable"
exit 1
fi
SYSTEM_TEST_FILE="$this_dir/../../mod_pagespeed/src/install/system_test.sh"
PRIMARY_PORT="$1"
SECONDARY_PORT="$2"
MOD_PAGESPEED_DIR="$3"
FILE_CACHE_PATH="$4"
NGINX_EXECUTABLE="$5"
PRIMARY_HOSTNAME="localhost:$PRIMARY_PORT"
SECONDARY_HOSTNAME="localhost:$SECONDARY_PORT"
SERVER_ROOT="$MOD_PAGESPEED_DIR/src/install/"
# We need check and check_not before we source SYSTEM_TEST_FILE that provides
# them.
function handle_failure_simple() {
echo "FAIL"
exit 2
}
function check_simple() {
echo " check" "$@"
"$@" || handle_failure_simple
}
function check_not_simple() {
echo " check_not" "$@"
"$@" && handle_failure_simple
}
this_dir="$( cd $(dirname "$0") && pwd)"
# set up the config file for the test
PAGESPEED_CONF="$this_dir/pagespeed_test.conf"
PAGESPEED_CONF_TEMPLATE="$PAGESPEED_CONF.template"
# check for config file template
check_simple test -e "$PAGESPEED_CONF_TEMPLATE"
# create PAGESPEED_CONF by substituting on PAGESPEED_CONF_TEMPLATE
echo > $PAGESPEED_CONF <<EOF
This file is automatically generated from $PAGESPEED_CONF_TEMPLATE"
by nginx_system_test.sh; don't edit here."
EOF
cat $PAGESPEED_CONF_TEMPLATE \
| sed 's#@@FILE_CACHE_PATH@@#'"$FILE_CACHE_PATH/"'#' \
| sed 's#@@SERVER_ROOT@@#'"$SERVER_ROOT"'#' \
| sed 's#@@PRIMARY_PORT@@#'"$PRIMARY_PORT"'#' \
| sed 's#@@SECONDARY_PORT@@#'"$SECONDARY_PORT"'#' \
>> $PAGESPEED_CONF
# make sure we substituted all the variables
check_not_simple grep @@ $PAGESPEED_CONF
# restart nginx with new config
killall nginx
sleep .1
check_simple "$NGINX_EXECUTABLE" -c "$PAGESPEED_CONF"
# run generic system tests
SYSTEM_TEST_FILE="$MOD_PAGESPEED_DIR/src/install/system_test.sh"
if [ ! -e "$SYSTEM_TEST_FILE" ] ; then
echo "Not finding $SYSTEM_TEST_FILE -- is mod_pagespeed not in a parallel"
@@ -51,5 +109,31 @@ PAGESPEED_EXPECTED_FAILURES="
~In-place resource optimization~
"
# The existing system test takes its arguments as positional parameters, and
# wants different ones than we want, so we need to reset our positional args.
set -- "$PRIMARY_HOSTNAME"
source $SYSTEM_TEST_FILE
# nginx-specific system tests
# When we allow ourself to fetch a resource because the Host header tells us
# that it is one of our resources, we should be fetching it from ourself.
start_test Loopback fetches go to local IPs without DNS lookup
# If we're properly fetching from ourself we will issue loopback fetches for
# /mod_pagespeed_example/combine_javascriptN.js, which will succeed, so
# combining will work. If we're taking 'Host:www.google.com' to mean that we
# should fetch from www.google.com then those fetches will fail because
# google.com won't have /mod_pagespeed_example/combine_javascriptN.js and so
# we'll not rewrite any resources.
URL="$HOSTNAME/mod_pagespeed_example/combine_javascript.html"
URL+="?ModPagespeed=on&ModPagespeedFilters=combine_javascript"
fetch_until "$URL" "fgrep -c .pagespeed." 1 --header=Host:www.google.com
# If this accepts the Host header and fetches from google.com it will fail with
# a 404. Instead it should use a loopback fetch and succeed.
URL="$HOSTNAME/mod_pagespeed_example/.pagespeed.ce.8CfGBvwDhH.css"
check wget -O /dev/null --header=Host:www.google.com "$URL"
check_failures_and_exit
test/pagespeed_test.conf.template
+203
View File
@@ -0,0 +1,203 @@
# nginx_system_test.sh makes a few substitutions to this file to generate
# pagespeed_test.conf
#user nobody;
worker_processes 1;
# for debugging
#daemon off;
#master_process off;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
error_log logs/error.log debug;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
root "@@SERVER_ROOT@@";
server {
listen @@PRIMARY_PORT@@;
server_name localhost;
pagespeed FileCachePath "@@FILE_CACHE_PATH@@";
# uncomment the following two lines if you're testing memcached
#pagespeed MemcachedServers "localhost:11211";
#pagespeed MemcachedThreads 1;
pagespeed on;
pagespeed RewriteLevel CoreFilters;
pagespeed EnableFilters insert_ga,trim_urls;
#pagespeed CacheFlushPollIntervalSec 1;
#pagespeed RunExperiment on;
#pagespeed AnalyticsID "UA-XXXXXXXX-Y";
#pagespeed ExperimentSpec "id=1;percent=50;default";
#pagespeed ExperimentSpec "id=2;percent=50";
pagespeed Library 43 1o978_K0_LNE5_ystNklf
http://www.modpagespeed.com/rewrite_javascript.js;
# Disable parsing if the size of the HTML exceeds 50kB.
pagespeed MaxHtmlParseBytes 50000;
location /mod_pagespeed_example/core_filters/ {
pagespeed RewriteLevel CoreFilters;
}
location /mod_pagespeed_example/pass_through/ {
pagespeed RewriteLevel PassThrough;
}
#location / {
# proxy_pass http://www.google.com;
#}
location /mod_pagespeed_test/no_cache/ {
add_header Cache-Control no-cache;
}
location /mod_pagespeed_test/compressed/ {
add_header Content-Encoding gzip;
types {
text/javascript custom_ext;
}
}
pagespeed EnableFilters remove_comments;
#charset koi8-r;
#access_log logs/host.access.log main;
index index.html;
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
sendfile on;
keepalive_timeout 65;
# set up gzip
gzip on;
gzip_vary on;
# Turn on gzip for all content types that should benefit from it.
gzip_types application/ecmascript;
gzip_types application/javascript;
gzip_types application/json;
gzip_types application/pdf;
gzip_types application/postscript;
gzip_types application/x-javascript;
gzip_types image/svg+xml;
gzip_types text/css;
gzip_types text/csv;
# "gzip_types text/html" is assumed.
gzip_types text/javascript;
gzip_types text/plain;
gzip_types text/xml;
gzip_http_version 1.0;
types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
application/rss+xml rss;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg svgz;
image/webp webp;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
audio/midi mid midi kar;
audio/mpeg mp3;
audio/ogg ogg;
audio/x-m4a m4a;
audio/x-realaudio ra;
video/3gpp 3gpp 3gp;
video/mp4 mp4;
video/mpeg mpeg mpg;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-m4v m4v;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
default_type application/octet-stream;
}