From b997d79ea7b044620f652c08aeeae12f0ab1e7f4 Mon Sep 17 00:00:00 2001 From: Jeff Kaufman Date: Wed, 21 Sep 2016 07:48:03 -0400 Subject: [PATCH] install: add script to install nginx_pagespeed (#1263) install: add script to install nginx_pagespeed Automated version of the instructions on https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source Tested on fresh vms of debian8, centos 6 and 7, rhel 6 and 7, and ubuntu lts 12, 14, and 16 Fixes https://github.com/pagespeed/ngx_pagespeed/issues/1262 --- PSOL_BINARY_URL | 4 + config | 29 +- scripts/build_ngx_pagespeed.sh | 501 +++++++++++++++++++++++++++++++++ 3 files changed, 524 insertions(+), 10 deletions(-) create mode 100644 PSOL_BINARY_URL create mode 100755 scripts/build_ngx_pagespeed.sh diff --git a/PSOL_BINARY_URL b/PSOL_BINARY_URL new file mode 100644 index 000000000..07e34fc50 --- /dev/null +++ b/PSOL_BINARY_URL @@ -0,0 +1,4 @@ +In a release this file would contain the URL to download the pre-compiled PSOL +binary, but on development branches (like this one) you have to build PSOL from +source yourself. See: + https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source \ No newline at end of file diff --git a/config b/config index d9c6cfc7f..0f523af82 100644 --- a/config +++ b/config @@ -25,18 +25,27 @@ if [ "$mod_pagespeed_dir" = "unset" ] ; then if [ ! -e "$mod_pagespeed_dir" ] ; then echo "ngx_pagespeed: pagespeed optimization library not found:" - echo "" - echo " You need to separately download the pagespeed library:" - echo "" - echo " $ cd /path/to/ngx_pagespeed" - echo " $ wget https://dl.google.com/dl/page-speed/psol/1.9.32.1.tar.gz" - echo " $ tar -xzvf 1.9.32.1.tar.gz # expands to psol/" - echo "" - echo " Or see the installation instructions:" - echo " https://github.com/pagespeed/ngx_pagespeed#how-to-build" + + psol_binary_url="$(cat $ngx_addon_dir/PSOL_BINARY_URL)" + if [[ "$psol_binary_url" != https://* ]]; then + echo " + This is a development branch of ngx_pagespeed, which means there is no + precompiled PSOL library available to link against. Either build from a + release tag, like latest-beta, or build PSOL from source: + https://github.com/pagespeed/ngx_pagespeed/wiki/Building-PSOL-From-Source" + exit 1 + fi + + echo " + You need to separately download the pagespeed library: + $ cd $ngx_addon_dir + $ wget $psol_binary_url + $ tar -xzvf $(basename $psol_binary_url) # expands to psol/ + + Or see the installation instructions: + https://developers.google.com/speed/pagespeed/module/build_ngx_pagespeed_from_source" exit 1 fi - else build_from_source=true fi diff --git a/scripts/build_ngx_pagespeed.sh b/scripts/build_ngx_pagespeed.sh new file mode 100755 index 000000000..376fc8643 --- /dev/null +++ b/scripts/build_ngx_pagespeed.sh @@ -0,0 +1,501 @@ +#!/bin/bash + +function usage() { + echo " +Usage: build_ngx_pagespeed.sh [options] + + Installs ngx_pagespeed and its dependencies. Can optionally build and install + nginx as well. + +Options: + -v, --ngx-pagespeed-version + What version of ngx_pagespeed to build. Required. + + -n, --nginx-version + What version of nginx to build. If not set, this script only prepares the + ngx_pagespeed module, and expects you to handle including it when you + build nginx. + + If you pass in 'latest' then this script scrapes the nginx download page + and attempts to determine the latest version automatically. + + -m, --dynamic-module + Build ngx_pagespeed as a dynamic module. + + -b, --builddir + Where to build. Defaults to \$HOME. + + -p, --no-deps-check + By default, this script checks for the packages it depends on and tries to + install them. If you have installed dependencies from source or are on a + non-deb non-rpm system, this won't work. In that case, install the + dependencies yourself and pass --no-deps-check. + + -d, --dryrun + Don't make any changes to the system, just print what changes you + would have made. + + -h, --help + Print this message and exit." +} + +# Prints an error message and exits with an error code. +function fail() { + local error_message="$@" + echo "$@" >&2 + + # Normally I'd use $0 in "usage" here, but since most people will be running + # this via curl, that wouldn't actually give something useful. + echo >&2 + echo "For usage information, run this script with --help" >&2 + exit 1 +} + +# Intended to be called as: +# bash <(curl dl.google.com/.../build_ngx_pagespeed.sh) + +# If we set -e or -u then users of this script will see it silently exit on +# failure. Instead we need to check the exit status of each command manually. +# The run function handles exit-status checking for system-changing commands. +# Additionally, this allows us to easily have a dryrun mode where we don't +# actually make any changes. +function run() { + if "$DRYRUN"; then + echo "would run $@" + else + if ! "$@"; then + echo "Failure running $@, exiting." + exit 1 + fi + fi +} + +function redhat_is_installed() { + local package_name="$1" + rpm -qa $package_name | grep -q . +} + +function debian_is_installed() { + local package_name="$1" + dpkg -l $package_name | grep ^ii | grep -q . +} + +function version_sort() { + # We'd rather use sort -V, but that's not available on Centos 5. This works + # for versions in the form A.B.C.D or shorter, which is enough for our use. + sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g +} + +# Compare two numeric versions in the form "A.B.C". Works with version numbers +# having up to four components, since that's enough to handle both nginx (3) and +# ngx_pagespeed (4). +function version_older_than() { + local test_version="$1" + local compare_to="$2" + + local older_version=$(echo $@ | tr ' ' '\n' | version_sort | head -n 1) + test "$older_version" != "$compare_to" +} + +function determine_latest_nginx_version() { + # Scrape nginx's download page to try to find the most recent nginx version. + + nginx_download_url="https://nginx.org/en/download.html" + function report_error() { + fail " +Couldn't automatically determine the latest nginx version: failed to $@ +$nginx_download_url" + } + + nginx_download_page=$(curl -sS --fail "$nginx_download_url") || \ + report_error "download" + + download_refs=$(echo "$nginx_download_page" | \ + grep -o '/download/nginx-[0-9.]*[.]tar[.]gz') || \ + report_error "parse" + + versions_available=$(echo "$download_refs" | \ + sed -e 's~^/download/nginx-~~' -e 's~\.tar\.gz$~~') || \ + report_error "extract versions from" + + latest_version=$(echo "$versions_available" | version_sort | tail -n 1) || \ + report_error "determine latest version from" + + if version_older_than "$latest_version" "1.11.4"; then + fail " +Expected the latest version of nginx to be at least 1.11.4 but found +$latest_version on $nginx_download_url" + fi + + echo "$latest_version" +} + +# Usage: +# install_dependencies install_pkg_cmd is_pkg_installed_cmd dep1 dep2 ... +# +# install_pkg_cmd is a command to install a dependency +# is_pkg_installed_cmd is a command that returns true if the dependency is +# already installed +# each dependency is a package name +function install_dependencies() { + local install_pkg_cmd="$1" + local is_pkg_installed_cmd="$2" + shift 2 + + local missing_dependencies="" + + for package_name in "$@"; do + if ! $is_pkg_installed_cmd $package_name; then + missing_dependencies+="$package_name " + fi + done + if [ -n "$missing_dependencies" ]; then + echo "Detected that we're missing the following depencencies:" + echo " $missing_dependencies" + echo "Installing them:" + run sudo $install_pkg_cmd $missing_dependencies + fi +} + +function gcc_too_old() { + # We need gcc >= 4.8 + local gcc_major_version=$(gcc -dumpversion | awk -F. '{print $1}') + if [ "$gcc_major_version" -lt 4 ]; then + return 0 # too old + elif [ "$gcc_major_version" -gt 4 ]; then + return 1 # plenty new + fi + # It's gcc 4.x, check if x >= 8: + local gcc_minor_version=$(gcc -dumpversion | awk -F. '{print $2}') + test "$gcc_minor_version" -lt 8 +} + +function continue_or_exit() { + local prompt="$1" + read -p "$prompt [Y/n] " yn + if [[ "$yn" == N* || "$yn" == n* ]]; then + echo "Cancelled." + exit 0 + fi +} + +function build_ngx_pagespeed() { + getopt --test + if [ "$?" != 4 ]; then + # Even Centos 5 and Ubuntu 10 LTS have new-style getopt, so I don't expect + # this to be hit in practice on systems that are actually able to run + # PageSpeed. + fail "Your version of getopt is too old. Exiting with no changes made." + fi + + opts=$(getopt -o v:n:mb:pdh \ + --longoptions ngx-pagespeed-version:,nginx-version:,dynamic-module \ + --longoptions buildir:,no-deps-check,dryrun,help \ + -n "$(basename "$0")" -- "$@") + if [ $? != 0 ]; then + usage + exit 1 + fi + eval set -- "$opts" + + NPS_VERSION="" + NGINX_VERSION="" + BUILDDIR="$HOME" + DO_DEPS_CHECK=true + DRYRUN=false + DYNAMIC_MODULE=false + while true; do + case "$1" in + -v | --ngx-pagespeed-version) shift + NPS_VERSION="$1" + shift + ;; + -n | --nginx-version) shift + NGINX_VERSION="$1" + shift + ;; + -m | --dynamic-module) shift + DYNAMIC_MODULE=true + ;; + -b | --builddir) shift + BUILDDIR="$1" + shift + ;; + -p | --no-deps-check) shift + DO_DEPS_CHECK=false + ;; + -d | --dryrun) shift + DRYRUN="true" + ;; + -h | --help) shift + usage + exit 0 + ;; + --) shift + break + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + done + + if [ -z "$NPS_VERSION" ]; then + fail "Please pass --ngx-pagespeed-version " + fi + + if [ ! -d "$BUILDDIR" ]; then + fail "Told to build in $BUILDDIR, but that directory doesn't exist." + fi + + if [ "$NGINX_VERSION" = "latest" ]; then + # When this function fails it prints the debugging information needed first + # to stderr. + NGINX_VERSION=$(determine_latest_nginx_version) || exit 1 + fi + + if "$DYNAMIC_MODULE"; then + # Check that ngx_pagespeed and nginx are recent enough to support dynamic + # modules. + if version_older_than "$NPS_VERSION" "1.10.33.5"; then + fail " +You're trying to build ngx_pagespeed $NPS_VERSION as a dynamic module, but +ngx_pagespeed didn't add support for dynamic modules until 1.10.33.5." + fi + + if [ ! -z "NGINX_VERSION" ]; then + if version_older_than "$NGINX_VERSION" "1.9.13"; then + fail " +You're trying to build nginx $NGINX_VERSION as a dynamic module but nginx didn't +add support for dynamic modules in a way compatible with ngx_pagespeed until +1.9.13." + fi + fi + fi + + if "$DRYRUN"; then + TEMPDIR="/tmp/output-of-mktemp" + else + TEMPDIR=$(mktemp -d) + function cleanup_tempdir { + rm -rf "$TEMPDIR" + } + trap cleanup_tempdir EXIT + fi + + PS_NGX_EXTRA_FLAGS="" + + # Now make sure our dependencies are installed. + if "$DO_DEPS_CHECK"; then + if [ -f /etc/debian_version ]; then + echo "Detected debian-based distro." + + install_dependencies "apt-get install" debian_is_installed \ + "build-essential zlib1g-dev libpcre3 libpcre3-dev unzip" + + if gcc_too_old; then + if [ ! -e /usr/lib/gcc-mozilla/bin/gcc ]; then + echo "Detected that gcc is older than 4.8. Installing gcc-mozilla" + echo "which installs gcc-4.8 into /usr/lib/gcc-mozilla/ and doesn't" + echo "affect your global gcc installation." + run sudo apt-get install gcc-mozilla + fi + + PS_NGX_EXTRA_FLAGS="--with-cc=/usr/lib/gcc-mozilla/bin/gcc" + PS_NGX_EXTRA_FLAGS+=" --with-ld-opt=-static-libstdc++" + fi + + elif [ -f /etc/redhat-release ]; then + echo "Detected redhat-based distro." + + install_dependencies "yum install" redhat_is_installed \ + "gcc-c++ pcre-devel zlib-devel make unzip wget" + if gcc_too_old; then + if [ ! -e /opt/rh/devtoolset-2/root/usr/bin/gcc ]; then + redhat_major_version=$( + cat /etc/redhat-release | grep -o -E '[0-9]+' | head -n 1) + if [ "$redhat_major_version" == 5 ]; then + slc_version=5 + elif [ "$redhat_major_version" == 6 ]; then + slc_version=6 + else + fail " +Unexpected major version $redhat_major_version in /etc/redhat-release: +$(cat /etc/redhat-release) Expected 5 or 6." + fi + + echo "Detected that gcc is older than 4.8. Scientific Linux provides" + echo "a gcc package that installs gcc-4.8 into /opt/ and doesn't" + echo "affect your global gcc installation." + slc_key="https://linux.web.cern.ch/linux/scientific6/docs/repository/" + slc_key+="cern/slc6X/i386/RPM-GPG-KEY-cern" + slc_key_out="$TEMPDIR/RPM-GPG-KEY-cern" + run sudo wget "$slc_key" -O "$slc_key_out" + run sudo rpm --import "$slc_key_out" + + repo_fname="/etc/yum.repos.d/slc${slc_version}-devtoolset.repo" + if [ -e "$repo_fname" ]; then + fail "Expected $repo_fname not to exist; aborting." + fi + + repo_url="https://linux.web.cern.ch/linux/scientific${slc_version}/" + repo_url+="/docs/repository/cern/devtoolset/" + repo_url+="slc${slc_version}-devtoolset.repo" + run sudo wget -O "$repo_fname" "$repo_url" + run sudo yum install devtoolset-2-gcc-c++ devtoolset-2-binutils + fi + PS_NGX_EXTRA_FLAGS="--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc" + fi + else + fail " +This doesn't appear to be a deb-based distro or an rpm-based one. Not going to +be able to install dependencies. Please install dependencies manually and rerun +with --depsinstalled." + fi + echo "Dependencies are all set." + else + echo "Not checking whether dependencies are installed." + fi + + function delete_if_already_exists() { + if "$DRYRUN"; then return; fi + + local directory="$1" + if [ -d "$directory" ]; then + if [ ${#directory} -lt 8 ]; then + fail " +Not deleting $directory; name is suspiciously short. Something is wrong." + exit 1 + fi + + continue_or_exit "OK to delete $directory?" + run rm -rf "$directory" + fi + } + + nps_baseurl="https://github.com/pagespeed/ngx_pagespeed/archive" + # In general, the zip github builds for tag foo unzips to ngx_pagespeed-foo, + # but it looks like they special case vVERSION tags to ngx_pagespeed-VERSION. + if [[ "$NPS_VERSION" =~ ^[0-9]*[.][0-9]*[.][0-9]*[.][0-9]*$ ]]; then + # We've been given a numeric version number. This has an associated tag in + # the form vVERSION-beta. + nps_url_fname="v${NPS_VERSION}-beta" + nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}-beta" + else + # We've been given a tag name, like latest-beta. Download that directly. + nps_url_fname="$NPS_VERSION" + nps_downloaded_fname="ngx_pagespeed-${NPS_VERSION}" + fi + + nps_downloaded="$TEMPDIR/$nps_downloaded_fname.zip" + run wget "$nps_baseurl/$nps_url_fname.zip" -O "$nps_downloaded" + nps_module_dir="$BUILDDIR/$nps_downloaded_fname" + delete_if_already_exists "$nps_module_dir" + echo "Extracting ngx_pagespeed..." + run unzip -q "$nps_downloaded" -d "$BUILDDIR" + run cd "$nps_module_dir" + + # Now we need to figure out what precompiled version of PSOL to build + # ngx_pagespeed against. + if "$DRYRUN"; then + psol_url="https://psol.example.com/cant-get-psol-version-in-dry-run.tar.gz" + elif [ -e PSOL_BINARY_URL ]; then + # Releases after 1.11.33.4 there is a PSOL_BINARY_URL file that tells us + # where to look. + psol_url="$(cat PSOL_BINARY_URL)" + if [[ "$psol_url" != https://* ]]; then + fail "Got bad psol binary location information: $psol_url" + fi + else + # For past releases we have to grep it from the config file. The url has + # always looked like this, and the config file has contained it since before + # we started tagging our ngx_pagespeed releases. + psol_url="$( + grep -o "https://dl.google.com/dl/page-speed/psol/[0-9.]*.tar.gz" config)" + if [ -z "$psol_url" ]; then + fail "Couldn't find PSOL url in $PWD/config" + fi + fi + + run wget "$psol_url" + echo "Extracting PSOL..." + run tar -xzf $(basename "$psol_url") # extracts to psol/ + + if "$DYNAMIC_MODULE"; then + configure_args="--add-dynamic-module=$nps_module_dir $PS_NGX_EXTRA_FLAGS" + else + configure_args="--add-module=$nps_module_dir $PS_NGX_EXTRA_FLAGS" + fi + + echo + if [ -z "$NGINX_VERSION" ]; then + # They didn't specify an nginx version, so we're just preparing the + # module for them to install. + echo "ngx_pagespeed is ready to be built against nginx." + echo "When running ./configure pass in:" + echo " $configure_args" + if [ -z "$PS_NGX_EXTRA_FLAGS" ]; then + echo "If this is for integration with an already-built nginx, make sure" + echo "to include any other arguments you originally passed to ./configure" + echo "You can see these with 'nginx -V'." + else + echo "Note: because we need to set $PS_NGX_EXTRA_FLAGS on this platform," + echo "if you want to integrate ngx_pagespeed with an already-built nginx" + echo "you're going to need to rebuild your nginx with those flags set." + fi + else + # Download and build nginx. + nginx_leaf="nginx-${NGINX_VERSION}.tar.gz" + nginx_fname="$TEMPDIR/$nginx_leaf" + run wget "http://nginx.org/download/$nginx_leaf" -O "$nginx_fname" + nginx_dir="$BUILDDIR/nginx-${NGINX_VERSION}/" + delete_if_already_exists "$nginx_dir" + echo "Extracting nginx..." + run tar -xzf "$nginx_fname" --directory "$BUILDDIR" + "$DRYRUN" || cd "$nginx_dir" + + echo "About to build nginx. Do you have any additional ./configure" + echo "arguments you would like to set? For example, if you would like" + echo "to build nginx with https support give --with-http_ssl_module" + echo "If you don't have any, just press enter." + read -p "> " additional_configure_args + + configure="./configure $configure_args $additional_configure_args" + echo "About to configure nginx with:" + echo " $configure" + continue_or_exit "Does this look right?" + run $configure + + continue_or_exit "Build nginx?" + run make + + continue_or_exit "Install nginx?" + run sudo make install + + echo + if "$DYNAMIC_MODULE"; then + echo "Nginx installed with ngx_pagespeed support available as a" + echo "loadable module." + echo + echo "To load the ngx_pagespeed module, you'll need to add:" + echo " load_module \"modules/ngx_pagespeed.so\";" + echo "at the top of your main nginx configuration file." + else + echo "Nginx installed with ngx_pagespeed support compiled-in." + fi + echo + echo "If this is a new installation you probably need an init script to" + echo "manage starting and stopping the nginx service. See:" + echo " http://wiki.nginx.org/InitScripts" + echo + echo "You'll also need to configure ngx_pagespeed if you haven't yet:" + echo " https://developers.google.com/speed/pagespeed/module/configuration" + fi + "$DRYRUN" && echo "[this was a dry run; your system is unchanged]" +} + +# Start running things from a call at the end so if this script is executed +# after a partial download it doesn't do anything. +build_ngx_pagespeed "$@"