From 247a821564bc932e3379caf724f33cce5fdad61a Mon Sep 17 00:00:00 2001 From: Jeff Kaufman Date: Tue, 6 Dec 2016 09:21:42 -0500 Subject: [PATCH] install: add support for building psol from source and setting up for development (#1313) * adds support for `--psol-from-source` so you don't need binary modules, and `--devel` so you can run our tests without going and getting all our dependencies * adds submodules for testing: mod_pagespeed, ngx_cache_purge etc * adds support for running as: ``` git clone git@github.com:pagespeed/ngx_pagespeed.git cd ngx_pagespeed/ git checkout scripts/build_ngx_pagespeed.sh [options] ``` * depends on the scripts @hillsp is working on so that we can just check out mod_pagespeed and ask it to build and rebuild itself * adds colors to output to make it easier to read --- .gitignore | 1 + .gitmodules | 18 + .travis.yml | 61 +-- scripts/build_ngx_pagespeed.sh | 459 ++++++++++++++---- test/nginx_system_test.sh | 10 +- test/run_tests.sh | 2 + .../headers-more-nginx-module | 1 + testing-dependencies/mod_pagespeed | 1 + testing-dependencies/nginx | 1 + testing-dependencies/ngx_cache_purge | 1 + testing-dependencies/ngx_devel_kit | 1 + testing-dependencies/set-misc-nginx-module | 1 + 12 files changed, 407 insertions(+), 150 deletions(-) create mode 100644 .gitmodules create mode 160000 testing-dependencies/headers-more-nginx-module create mode 160000 testing-dependencies/mod_pagespeed create mode 160000 testing-dependencies/nginx create mode 160000 testing-dependencies/ngx_cache_purge create mode 160000 testing-dependencies/ngx_devel_kit create mode 160000 testing-dependencies/set-misc-nginx-module diff --git a/.gitignore b/.gitignore index c1ddac92b..f03b88de6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ test/tmp psol/ psol-*.tar.gz *.*.*.*.tar.gz +nginx diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..d38a4bcd8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,18 @@ +[submodule "testing-dependencies/mod_pagespeed"] + path = testing-dependencies/mod_pagespeed + url = https://github.com/pagespeed/mod_pagespeed.git +[submodule "testing-dependencies/ngx_cache_purge"] + path = testing-dependencies/ngx_cache_purge + url = https://github.com/FRiCKLE/ngx_cache_purge.git +[submodule "testing-dependencies/nginx"] + path = testing-dependencies/nginx + url = https://github.com/nginx/nginx.git +[submodule "testing-dependencies/set-misc-nginx-module"] + path = testing-dependencies/set-misc-nginx-module + url = https://github.com/openresty/set-misc-nginx-module +[submodule "testing-dependencies/ngx_devel_kit"] + path = testing-dependencies/ngx_devel_kit + url = https://github.com/simpl/ngx_devel_kit +[submodule "testing-dependencies/headers-more-nginx-module"] + path = testing-dependencies/headers-more-nginx-module + url = https://github.com/openresty/headers-more-nginx-module diff --git a/.travis.yml b/.travis.yml index 793dc9f81..b722a04bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,50 +1,29 @@ language: c++ +env: + global: + - MAKEFLAGS=-j3 +# By default travis loads submodules serially, but we can load them in parallel +# if we install an updated git and use --jobs. Some timing numbers: +# serial: 257s +# jobs=4: 182s (29s to install new git, 153s to run the downloads) +# jobs=8: 179s (29s to install new git, 150s to run the downloads) +# We can't use --depth=1, though, because github doesn't have +# allowReachableSHA1InWant set. +git: + submodules: false +before_install: + - sudo add-apt-repository --yes ppa:git-core/ppa + - sudo apt-get update + - sudo apt-get install git + - git submodule update --init --recursive --jobs=8 install: - - mv $TRAVIS_BUILD_DIR ~/ngxpagespeed - - sudo apt-get install build-essential zlib1g-dev libpcre3 libpcre3-dev unzip g++ python gperf make devscripts fakeroot git curl netcat-traditional gcc-mozilla clang-3.4 2>&1 > /dev/null - - export PATH=/usr/lib/gcc-mozilla/bin:$PATH - - sudo ln -sf /usr/lib/gcc-mozilla/lib/libstdc++.so.6 /usr/lib/x86_64-linux-gnu/libstdc++.so.6 -# - sudo sh -c 'echo "image/webp webp" >> /etc/mime.types' -# - mkdir -p ~/bin -# - cd ~/bin -# - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git - - mkdir ~/mod_pagespeed - - cd ~/mod_pagespeed - - git clone --recursive https://github.com/pagespeed/mod_pagespeed.git src - - cd src - - python build/gyp_chromium --depth=. - - cd ~/mod_pagespeed/src/pagespeed/automatic - - make BUILDTYPE=Release -C ../../pagespeed/automatic all - - cd ~ - - git clone https://github.com/FRiCKLE/ngx_cache_purge.git - - NGX_CACHE_PURGE=$PWD/ngx_cache_purge - - wget https://openresty.org/download/ngx_openresty-1.9.7.2.tar.gz - - tar xzf ngx_openresty-*.tar.gz - - cd ngx_openresty-*/ - - ./configure --with-luajit - - make - - NGX_DEVEL_KIT=$(echo $HOME/ngx_openresty-*/build/ngx_devel_kit-*/) - - SET_MISC_MODULE=$(echo $HOME/ngx_openresty-*/build/set-misc-nginx-module-*/) - - HEADERS_MORE_MODULE=$(echo $HOME/ngx_openresty-*/build/headers-more-nginx-module-*/) - - cd ~ - - wget https://github.com/nginx/nginx/archive/branches/default.zip - - unzip default.zip - - cd nginx-branches-default - - MOD_PAGESPEED_DIR="$HOME/mod_pagespeed/src" ./auto/configure --add-module=$HOME/ngxpagespeed --add-module="$NGX_CACHE_PURGE" --add-module="$NGX_DEVEL_KIT" --add-module="$SET_MISC_MODULE" --add-module="$HEADERS_MORE_MODULE" --with-ipv6 - - make - - sudo make install + scripts/build_ngx_pagespeed.sh --devel --assume-yes script: - - echo "build successful" - - echo "cd ~/ngxpagespeed" - - echo "sudo ./test/run_tests.sh $HOME/mod_pagespeed $HOME/nginx-branches-default/objs/nginx" + test/run_tests.sh $PWD/testing-dependencies/mod_pagespeed/ \ + $PWD/nginx/sbin/nginx sudo: required compiler: - gcc notifications: email: - - cheesy@google.com - jefftk@google.com - - morlovich@google.com - - jmarantz@google.com - - huibao@google.com - - jcrowell@google.com diff --git a/scripts/build_ngx_pagespeed.sh b/scripts/build_ngx_pagespeed.sh index edf33146b..555b1dd7a 100755 --- a/scripts/build_ngx_pagespeed.sh +++ b/scripts/build_ngx_pagespeed.sh @@ -5,7 +5,16 @@ function usage() { Usage: build_ngx_pagespeed.sh [options] Installs ngx_pagespeed and its dependencies. Can optionally build and install - nginx as well. + nginx as well. Can be run either as: + + bash <(curl -f -L -sS https://ngxpagespeed.com/install) [options] + + Or: + + git clone git@github.com:pagespeed/ngx_pagespeed.git + cd ngx_pagespeed/ + git checkout + scripts/build_ngx_pagespeed.sh [options] Options: -v, --ngx-pagespeed-version @@ -14,7 +23,11 @@ Options: * latest-stable * a version number, such as 1.11.33.4 - If you don't specify a version, defaults to latest-stable. + If you don't specify a version, defaults to latest-stable unless --devel + is specified, in which case it defaults to trunk-tracking. + + This option doesn't make sense if we're running within an existing + ngx_pagespeed checkout. -n, --nginx-version What version of nginx to build. If not set, this script only prepares the @@ -36,6 +49,23 @@ Options: non-deb non-rpm system, this won't work. In that case, install the dependencies yourself and pass --no-deps-check. + -s, --psol-from-source + Build PSOL from source instead of downloading a pre-built binary module. + + -l, --devel + Sets up a development environment in ngx_pagespeed/nginx, building with + testing-only dependencies. Includes --psol-from-source, conflicts with + --nginx-version. Uses a 'git clone' checkout for ngx_pagespeed and nginx + instead of downloading a tarball. + + -t, --build-type + When building PSOL from source, what to tell it for BUILD_TYPE. Defaults + to 'Release' unless --devel is set in which case it defaults to 'Debug'. + + -y, --assume-yes + Assume the answer to all prompts is 'yes, please continue'. Intended for + automated usage, such as buildbots. + -d, --dryrun Don't make any changes to the system, just print what changes you would have made. @@ -44,10 +74,34 @@ Options: Print this message and exit." } +RED=31 +GREEN=32 +YELLOW=33 +function begin_color() { + color="$1" + echo -e -n "\e[${color}m" +} +function end_color() { + echo -e -n "\e[0m" +} + +function echo_color() { + color="$1" + shift + begin_color "$color" + echo "$@" + end_color +} + +function error() { + local error_message="$@" + echo_color "$RED" -n "Error: " >&2 + echo "$@" >&2 +} + # Prints an error message and exits with an error code. function fail() { - local error_message="$@" - echo "$@" >&2 + error "$@" # Normally I'd use $0 in "usage" here, but since most people will be running # this via curl, that wouldn't actually give something useful. @@ -56,6 +110,11 @@ function fail() { exit 1 } + +function status() { + echo_color "$GREEN" "$@" +} + # Intended to be called as: # bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) @@ -64,12 +123,19 @@ function fail() { # The run function handles exit-status checking for system-changing commands. # Additionally, this allows us to easily have a dryrun mode where we don't # actually make any changes. +INITIAL_ENV=$(printenv | sort) function run() { if "$DRYRUN"; then - echo "would run $@" + echo_color "$YELLOW" -n "would run" + echo " $@" + env_differences=$(comm -13 <(echo "$INITIAL_ENV") <(printenv | sort)) + if [ -n "$env_differences" ]; then + echo " with the following additional environment variables:" + echo "$env_differences" | sed 's/^/ /' + fi else if ! "$@"; then - echo "Failure running $@, exiting." + error "Failure running '$@', exiting." exit 1 fi fi @@ -155,9 +221,9 @@ function install_dependencies() { fi done if [ -n "$missing_dependencies" ]; then - echo "Detected that we're missing the following depencencies:" + status "Detected that we're missing the following depencencies:" echo " $missing_dependencies" - echo "Installing them:" + status "Installing them:" run sudo $install_pkg_cmd $missing_dependencies fi } @@ -176,8 +242,13 @@ function gcc_too_old() { } function continue_or_exit() { + if "$ASSUME_YES"; then + return + fi + local prompt="$1" - read -p "$prompt [Y/n] " yn + echo_color "$YELLOW" -n "$prompt" + read -p " [Y/n] " yn if [[ "$yn" == N* || "$yn" == n* ]]; then echo "Cancelled." exit 0 @@ -217,9 +288,10 @@ function build_ngx_pagespeed() { fail "Your version of getopt is too old. Exiting with no changes made." fi - opts=$(getopt -o v:n:mb:pdh \ + opts=$(getopt -o v:n:mb:pslt:ydh \ --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \ - --longoptions buildir:,no-deps-check,dryrun,help \ + --longoptions buildir:,no-deps-check,psol-from-source,devel,build-type: \ + --longoptions assume-yes,dryrun,help \ -n "$(basename "$0")" -- "$@") if [ $? != 0 ]; then usage @@ -227,10 +299,14 @@ function build_ngx_pagespeed() { fi eval set -- "$opts" - NPS_VERSION="latest-stable" + NPS_VERSION="DEFAULT" NGINX_VERSION="" BUILDDIR="$HOME" DO_DEPS_CHECK=true + PSOL_FROM_SOURCE=false + DEVEL=false + BUILD_TYPE="" + ASSUME_YES=false DRYRUN=false DYNAMIC_MODULE=false while true; do @@ -253,6 +329,19 @@ function build_ngx_pagespeed() { -p | --no-deps-check) shift DO_DEPS_CHECK=false ;; + -s | --psol-from-source) shift + PSOL_FROM_SOURCE=true + ;; + -l | --devel) shift + DEVEL=true + ;; + -t | --build-type) shift + BUILD_TYPE="$1" + shift + ;; + -y | --assume-yes) shift + ASSUME_YES="true" + ;; -d | --dryrun) shift DRYRUN="true" ;; @@ -271,10 +360,58 @@ function build_ngx_pagespeed() { esac done + USE_GIT_CHECKOUT="$DEVEL" + ALREADY_CHECKED_OUT=false + if [ -e PSOL_BINARY_URL ]; then + status "Detected that we're running in an existing ngx_pagespeed checkout." + USE_GIT_CHECKOUT=true + ALREADY_CHECKED_OUT=true + fi + + if "$ALREADY_CHECKED_OUT"; then + if [ "$NPS_VERSION" != "DEFAULT" ]; then + fail \ +"The --ngx-pagespeed-version argument doesn't make sense when running within an existing checkout." + fi + elif [ "$NPS_VERSION" = "DEFAULT" ]; then + if "$DEVEL"; then + NPS_VERSION="trunk-tracking" + else + NPS_VERSION="latest-stable" + fi + fi + if [ ! -d "$BUILDDIR" ]; then fail "Told to build in $BUILDDIR, but that directory doesn't exist." fi + BUILD_NGINX=false + if [ -n "$NGINX_VERSION" ]; then + BUILD_NGINX=true + fi + + if "$DEVEL"; then + PSOL_FROM_SOURCE=true + BUILD_NGINX=true + if [ -n "$NGINX_VERSION" ]; then + fail \ +"The --devel argument conflicts with --nginx. In devel mode we use the version of nginx that's included as a submodule." + fi + if "$DYNAMIC_MODULE"; then + fail "Can't currently build a dynamic module in --devel mode." + fi + fi + + if "$PSOL_FROM_SOURCE" && [ -z "$BUILD_TYPE" ]; then + if "$DEVEL"; then + BUILD_TYPE="Debug" + else + BUILD_TYPE="Release" + fi + elif [ -n "$BUILD_TYPE" ]; then + fail "Setting --build-type requires --psol-from-source or --devel." + fi + if [ "$NGINX_VERSION" = "latest" ]; then # When this function fails it prints the debugging information needed first # to stderr. @@ -314,16 +451,16 @@ add support for dynamic modules in a way compatible with ngx_pagespeed until # Now make sure our dependencies are installed. if "$DO_DEPS_CHECK"; then if [ -f /etc/debian_version ]; then - echo "Detected debian-based distro." + status "Detected debian-based distro." install_dependencies "apt-get install" debian_is_installed \ "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip" if gcc_too_old; then if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then - echo "Detected that gcc is older than 4.8. Installing gcc-mozilla" - echo "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't" - echo "affect your global gcc installation." + status "Detected that gcc is older than 4.8. Installing gcc-mozilla" + status "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't" + status "affect your global gcc installation." run sudo apt-get install gcc-mozilla fi @@ -332,7 +469,7 @@ add support for dynamic modules in a way compatible with ngx_pagespeed until fi elif [ -f /etc/redhat-release ]; then - echo "Detected redhat-based distro." + status "Detected redhat-based distro." install_dependencies "yum install" redhat_is_installed \ "gcc-c++ pcre-devel zlib-devel make unzip wget" @@ -350,9 +487,9 @@ Unexpected major version $redhat_major_version in /etc/redhat-release: $(cat /etc/redhat-release) Expected 5 or 6." fi - echo "Detected that gcc is older than 4.8. Scientific Linux provides" - echo "a gcc package that installs gcc-4.8 into /opt/ and doesn't" - echo "affect your global gcc installation." + status "Detected that gcc is older than 4.8. Scientific Linux" + status "provides a gcc package that installs gcc-4.8 into /opt/ and" + status "doesn't affect your global gcc installation." slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/" slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern" slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern" @@ -378,9 +515,9 @@ This doesn't appear to be a deb-based distro or an rpm-based one. Not going to be able to install dependencies. Please install dependencies manually and rerun with --no-deps-check." fi - echo "Dependencies are all set." + status "Operating system dependencies are all set." else - echo "Not checking whether dependencies are installed." + status "Not checking whether operating system dependencies are installed." fi function delete_if_already_exists() { @@ -391,7 +528,6 @@ with --no-deps-check." if [ ${#directory} -lt 8 ]; then fail " Not deleting $directory; name is suspiciously short. Something is wrong." - exit 1 fi continue_or_exit "OK to delete $directory?" @@ -399,53 +535,106 @@ Not deleting $directory; name is suspiciously short. Something is wrong." fi } - nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive" # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo, - # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION. + # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then - # We've been given a numeric version number. This has an associated tag in - # the form vVERSION-beta. - nps_url_fname="v${NPS_VERSION}-beta" + # We've been given a numeric version number. This has an associated tag + # in the form vVERSION-beta. + tag_name="v${NPS_VERSION}-beta" nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta" else # We've been given a tag name, like latest-beta. Download that directly. - nps_url_fname="$NPS_VERSION" + tag_name="$NPS_VERSION" nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}" fi - nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip" - run wget "$nps_baseurl/$nps_url_fname.zip" -O "$nps_downloaded" - nps_module_dir="$BUILDDIR/$nps_downloaded_fname" - delete_if_already_exists "$nps_module_dir" - echo "Extracting ngx_pagespeed..." - run unzip -q "$nps_downloaded" -d "$BUILDDIR" - run cd "$nps_module_dir" - - # Now we need to figure out what precompiled version of PSOL to build - # ngx_pagespeed against. - if "$DRYRUN"; then - psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz" - elif [ -e PSOL_BINARY_URL ]; then - # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us - # where to look. - psol_url="$(cat PSOL_BINARY_URL)" - if [[ "$psol_url" != https://* ]]; then - fail "Got bad psol binary location information: $psol_url" + install_dir="this-only-makes-sense-in-devel-mode" + if "$USE_GIT_CHECKOUT"; then + # We're either doing a --devel build, or someone is running us from an + # existing git checkout. + nps_module_dir="$PWD" + install_dir="$nps_module_dir" + if "$ALREADY_CHECKED_OUT"; then + run cd "$nps_module_dir" + else + status "Checking out ngx_pagespeed..." + run git clone "git@github.com:pagespeed/ngx_pagespeed.git" \ + "$nps_module_dir" + run cd "$nps_module_dir" + run git checkout "$tag_name" + fi + submodules_dir="$nps_module_dir/testing-dependencies" + if "$DEVEL"; then + status "Downloading dependencies..." + run git submodule update --init --recursive + if [[ "$CONTINUOUS_INTEGRATION" != true ]]; then + status "Switching submodules over to git protocol." + # This lets us push to github by public key. + for config in $(find .git/ -name config) ; do + run sed -i s~https://github.com/~git@github.com:~ $config ; + done + fi fi else - # For past releases we have to grep it from the config file. The url has - # always looked like this, and the config file has contained it since before - # we started tagging our ngx_pagespeed releases. - psol_url="$( - grep -o "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)" - if [ -z "$psol_url" ]; then - fail "Couldn't find PSOL url in $PWD/config" - fi + nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive" + nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip" + status "Downloading ngx_pagespeed..." + run wget "$nps_baseurl/$tag_name.zip" -O "$nps_downloaded" + nps_module_dir="$BUILDDIR/$nps_downloaded_fname" + delete_if_already_exists "$nps_module_dir" + status "Extracting ngx_pagespeed..." + run unzip -q "$nps_downloaded" -d "$BUILDDIR" + run cd "$nps_module_dir" fi - run wget "$psol_url" - echo "Extracting PSOL..." - run tar -xzf $(basename "$psol_url") # extracts to psol/ + MOD_PAGESPEED_DIR="" + PSOL_BINARY="" + if "$PSOL_FROM_SOURCE"; then + MOD_PAGESPEED_DIR="$PWD/testing-dependencies/mod_pagespeed" + git submodule update --init --recursive -- "$MOD_PAGESPEED_DIR" + run pushd "$MOD_PAGESPEED_DIR" + + if "$DEVEL"; then + if [ ! -d "$HOME/apache2" ]; then + run install/build_development_apache.sh 2.2 prefork + fi + cd devel + run make apache_debug_psol + PSOL_BINARY="$MOD_PAGESPEED_DIR/out/$BUILD_TYPE/pagespeed_automatic.a" + else + run install/build_psol.sh --skip_tests --skip_packaging + PSOL_BINARY="$MOD_PAGESPEED_DIR/pagespeed/automatic/pagespeed_automatic.a" + fi + run popd + else + # Now we need to figure out what precompiled version of PSOL to build + # ngx_pagespeed against. + if "$DRYRUN"; then + psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz" + elif [ -e PSOL_BINARY_URL ]; then + # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us + # where to look. + psol_url="$(cat PSOL_BINARY_URL)" + if [[ "$psol_url" != https://* ]]; then + fail "Got bad psol binary location information: $psol_url" + fi + else + # For past releases we have to grep it from the config file. The url has + # always looked like this, and the config file has contained it since + # before we started tagging our ngx_pagespeed releases. + psol_url="$(grep -o \ + "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)" + if [ -z "$psol_url" ]; then + fail "Couldn't find PSOL url in $PWD/config" + fi + fi + + status "Downloading PSOL binary..." + run wget "$psol_url" + + status "Extracting PSOL..." + run tar -xzf $(basename "$psol_url") # extracts to psol/ + fi if "$DYNAMIC_MODULE"; then add_module="--add-dynamic-module=$nps_module_dir" @@ -454,17 +643,37 @@ Not deleting $directory; name is suspiciously short. Something is wrong." fi configure_args=("$add_module" "${extra_flags[@]}") + if "$DEVEL"; then + configure_args=("${configure_args[@]}" + "--prefix=$install_dir/nginx" + "--add-module=$submodules_dir/ngx_cache_purge" + "--add-module=$submodules_dir/ngx_devel_kit" + "--add-module=$submodules_dir/set-misc-nginx-module" + "--add-module=$submodules_dir/headers-more-nginx-module" + "--with-ipv6" + "--with-http_v2_module") + if [ "$BUILD_TYPE" = "Debug" ]; then + configure_args=("${configure_args[@]}" "--with-debug") + fi + fi + echo - if [ -z "$NGINX_VERSION" ]; then - # They didn't specify an nginx version, so we're just preparing the - # module for them to install. - echo "ngx_pagespeed is ready to be built against nginx." - echo "When running ./configure pass in:" - echo " $(quote_arguments "${configure_args[@]}")" + if ! "$BUILD_NGINX"; then + # Just prepare the module for them to install. + status "ngx_pagespeed is ready to be built against nginx." + echo "When running ./configure:" + if "$PSOL_FROM_SOURCE"; then + echo " Set the following environment variables:" + echo " MOD_PAGESPEED_DIR=$MOD_PAGESPEED_DIR" + echo " PSOL_BINARY=$PSOL_BINARY" + fi + echo " Give ./configure the following arguments:" + echo " $(quote_arguments "${configure_args[@]}")" + echo if [ ${#extra_flags[@]} -eq 0 ]; then echo "If this is for integration with an already-built nginx, make sure" - echo "to include any other arguments you originally passed to ./configure" - echo "You can see these with 'nginx -V'." + echo "to include any other arguments you originally passed to" + echo "./configure. You can see these with 'nginx -V'." else echo "Note: because we need to set $(quote_arguments "${extra_flags[@]}")" echo "on this platform, if you want to integrate ngx_pagespeed with an" @@ -472,60 +681,98 @@ Not deleting $directory; name is suspiciously short. Something is wrong." echo "those flags set." fi else - # Download and build nginx. - nginx_leaf="nginx-${NGINX_VERSION}.tar.gz" - nginx_fname="$TEMPDIR/$nginx_leaf" - run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname" - nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/" - delete_if_already_exists "$nginx_dir" - echo "Extracting nginx..." - run tar -xzf "$nginx_fname" --directory "$BUILDDIR" - "$DRYRUN" || cd "$nginx_dir" + if "$DEVEL"; then + # Use the nginx we loaded as a submodule + nginx_dir="$submodules_dir/nginx" + configure_location="auto" + else + # Download and build the specified nginx version. + nginx_leaf="nginx-${NGINX_VERSION}.tar.gz" + nginx_fname="$TEMPDIR/$nginx_leaf" + status "Downloading nginx..." + run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname" + nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/" + delete_if_already_exists "$nginx_dir" + status "Extracting nginx..." + run tar -xzf "$nginx_fname" --directory "$BUILDDIR" + configure_location="." + fi + run cd "$nginx_dir" - configure=("./configure" "${configure_args[@]}") - echo "About to build nginx. Do you have any additional ./configure" - echo "arguments you would like to set? For example, if you would like" - echo "to build nginx with https support give --with-http_ssl_module" - echo "If you don't have any, just press enter." - read -p "> " additional_configure_args - if [ -n "$additional_configure_args" ]; then - # Split additional_configure_args respecting any internal quotation. - # Otherwise things like --with-cc-opt='-foo -bar' won't work. - eval additional_configure_args=("$additional_configure_args") - configure=("${configure[@]}" "${additional_configure_args[@]}") + configure=("$configure_location/configure" "${configure_args[@]}") + if ! "$ASSUME_YES"; then + echo "About to build nginx. Do you have any additional ./configure" + echo "arguments you would like to set? For example, if you would like" + echo "to build nginx with https support give --with-http_ssl_module" + echo "If you don't have any, just press enter." + read -p "> " additional_configure_args + if [ -n "$additional_configure_args" ]; then + # Split additional_configure_args respecting any internal quotation. + # Otherwise things like --with-cc-opt='-foo -bar' won't work. + eval additional_configure_args=("$additional_configure_args") + configure=("${configure[@]}" "${additional_configure_args[@]}") + fi fi echo "About to configure nginx with:" echo " $(quote_arguments "${configure[@]}")" continue_or_exit "Does this look right?" - run "${configure[@]}" + MOD_PAGESPEED_DIR="$MOD_PAGESPEED_DIR" \ + PSOL_BINARY="$PSOL_BINARY" \ + run "${configure[@]}" - continue_or_exit "Build nginx?" + if ! "$DEVEL"; then + continue_or_exit "Build nginx?" + fi run make - continue_or_exit "Install nginx?" - run sudo make install + if "$DEVEL"; then + run make install - echo - if "$DYNAMIC_MODULE"; then - echo "Nginx installed with ngx_pagespeed support available as a" - echo "loadable module." + status "Nginx installed with ngx_pagespeed, and set up for testing." + # TODO(jefftk): pull these out into separate scripts. + echo "To run tests, pick a pair of ports like 8050 and 8051 and run:" + echo " cd $nps_module_dir" + echo " RUN_TESTS=true \\" + echo " USE_VALGRIND=false \\" + echo " TEST_NATIVE_FETCHER=false \\" + echo " TEST_SERF_FETCHER=true \\" + echo " test/run_tests.sh \\" + echo " $MOD_PAGESPEED_DIR \\" + echo " $install_dir/nginx/sbin/nginx" echo - echo "To load the ngx_pagespeed module, you'll need to add:" - echo " load_module \"modules/ngx_pagespeed.so\";" - echo "at the top of your main nginx configuration file." + echo "To rebuild after changes:" + echo " First, if you change things in PSOL or update it:" + echo " cd $MOD_PAGESPEED_DIR/devel" + echo " make apache_debug_psol" + echo " Then, whether or not you updated PSOL, rebuild nginx:" + echo " cd $install_dir/nginx" + echo " make && make install" else - echo "Nginx installed with ngx_pagespeed support compiled-in." + continue_or_exit "Install nginx?" + run sudo make install + + echo + if "$DYNAMIC_MODULE"; then + echo "Nginx installed with ngx_pagespeed support available as a" + echo "loadable module." + echo + echo "To load the ngx_pagespeed module, you'll need to add:" + echo " load_module \"modules/ngx_pagespeed.so\";" + echo "at the top of your main nginx configuration file." + else + echo "Nginx installed with ngx_pagespeed support compiled-in." + fi + echo + echo "If this is a new installation you probably need an init script to" + echo "manage starting and stopping the nginx service. See:" + echo " http://wiki.nginx.org/InitScripts" + echo + echo "You'll also need to configure ngx_pagespeed if you haven't yet:" + echo " https://developers.google.com/speed/pagespeed/module/configuration" fi - echo - echo "If this is a new installation you probably need an init script to" - echo "manage starting and stopping the nginx service. See:" - echo " http://wiki.nginx.org/InitScripts" - echo - echo "You'll also need to configure ngx_pagespeed if you haven't yet:" - echo " https://developers.google.com/speed/pagespeed/module/configuration" fi if "$DRYRUN"; then - echo "[this was a dry run; your system is unchanged]" + echo_color "$YELLOW" "[this was a dry run; your system is unchanged]" fi } diff --git a/test/nginx_system_test.sh b/test/nginx_system_test.sh index d10d50d63..f1d7bd056 100644 --- a/test/nginx_system_test.sh +++ b/test/nginx_system_test.sh @@ -49,7 +49,11 @@ rm -rf "$TEST_TMP" mkdir -p "$TEST_TMP" echo TEST_TMP=$TEST_TMP -APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/src/install/" +if [ -d "$MOD_PAGESPEED_DIR/src" ]; then + MOD_PAGESPEED_DIR+="/src" +fi + +APACHE_DOC_SRC="$MOD_PAGESPEED_DIR/install/" SERVER_ROOT="$TEST_TMP/root" echo SERVER_ROOT=$SERVER_ROOT rm -rf "$SERVER_ROOT" @@ -311,7 +315,7 @@ SERVER_NAME=nginx RUN_CONTROLLER_TEST=${RUN_CONTROLLER_TEST:-off} # run generic system tests -PAGESPEED_DIR="$MOD_PAGESPEED_DIR/src/pagespeed" +PAGESPEED_DIR="$MOD_PAGESPEED_DIR/pagespeed" SYSTEM_TEST_FILE="$PAGESPEED_DIR/system/system_test.sh" REMOTE_CONFIG_TEST_FILE="$PAGESPEED_DIR/system/remote_config_test.sh" @@ -1420,7 +1424,7 @@ OUT=$(cat "$ERROR_LOG" \ | grep -v "\\[error\\].*forbidden.example.com*" \ | grep -v "\\[error\\].*custom-paths.example.com*" \ | grep -v "\\[error\\].*bogus_format*" \ - | grep -v "\\[error\\].*src/install/foo*" \ + | grep -v "\\[error\\].*/install/foo*" \ | grep -v "\\[error\\].*recv() failed*" \ | grep -v "\\[error\\].*send() failed*" \ | grep -v "\\[error\\].*Invalid url requested: js_defer.js.*" \ diff --git a/test/run_tests.sh b/test/run_tests.sh index f03105d8f..266aa84c8 100755 --- a/test/run_tests.sh +++ b/test/run_tests.sh @@ -58,10 +58,12 @@ NGINX_EXECUTABLE="$2" : ${CONTROLLER_PORT:=8053} : ${RCPORT:=9991} : ${PAGESPEED_TEST_HOST:=selfsigned.modpagespeed.com} +: ${PHP_PORT:=9000} this_dir="$( cd $(dirname "$0") && pwd)" function run_test_checking_failure() { + "$MOD_PAGESPEED_DIR/install/start_php.sh" "$PHP_PORT" USE_VALGRIND="$USE_VALGRIND" \ PRIMARY_PORT="$PRIMARY_PORT" \ SECONDARY_PORT="$SECONDARY_PORT" \ diff --git a/testing-dependencies/headers-more-nginx-module b/testing-dependencies/headers-more-nginx-module new file mode 160000 index 000000000..30fb25901 --- /dev/null +++ b/testing-dependencies/headers-more-nginx-module @@ -0,0 +1 @@ +Subproject commit 30fb25901cad32b342f53cb174730c6e00fb66eb diff --git a/testing-dependencies/mod_pagespeed b/testing-dependencies/mod_pagespeed new file mode 160000 index 000000000..91662d08a --- /dev/null +++ b/testing-dependencies/mod_pagespeed @@ -0,0 +1 @@ +Subproject commit 91662d08ae6653b459372daff69bd7cd3bb0589f diff --git a/testing-dependencies/nginx b/testing-dependencies/nginx new file mode 160000 index 000000000..6917d29d4 --- /dev/null +++ b/testing-dependencies/nginx @@ -0,0 +1 @@ +Subproject commit 6917d29d409b187d440d8e5012a1353398e3a365 diff --git a/testing-dependencies/ngx_cache_purge b/testing-dependencies/ngx_cache_purge new file mode 160000 index 000000000..331fe43e8 --- /dev/null +++ b/testing-dependencies/ngx_cache_purge @@ -0,0 +1 @@ +Subproject commit 331fe43e8d9a3d1fa5e0c9fec7d3201d431a9177 diff --git a/testing-dependencies/ngx_devel_kit b/testing-dependencies/ngx_devel_kit new file mode 160000 index 000000000..e44326207 --- /dev/null +++ b/testing-dependencies/ngx_devel_kit @@ -0,0 +1 @@ +Subproject commit e443262071e759c047492be60ec7e2d73c5b57ec diff --git a/testing-dependencies/set-misc-nginx-module b/testing-dependencies/set-misc-nginx-module new file mode 160000 index 000000000..72be6512c --- /dev/null +++ b/testing-dependencies/set-misc-nginx-module @@ -0,0 +1 @@ +Subproject commit 72be6512cfe330af41b44f273b26f738e6b77864