artox/incubator-pagespeed-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build release tarball: open source it (#1474)

Browse Source 
* build release tarball: open source it

* change it from being given a branch to working on the current checkout
* change it to work with create_distro_tarball, which was already opensourced
* make it build openssl 1.0.2 if needed

* doc ubuntu 14 dep
This commit is contained in:
Jeff Kaufman
2017-01-11 15:19:44 -05:00
committed by GitHub
parent 58c175ca32
commit 6a15917500
2 changed files with 183 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devel/build_release_tarball.sh
Executable
+96
View File
@@ -0,0 +1,96 @@
#!/bin/bash
#
# Note: you might need to type your password a few times, once early, and once
# at the end.
#
# This should be run on a release branch to make sure we can make a tarball and
# at least build it on our workstations. It will also copy the tarball into
# ~/release (where the binaries usually go).
#
# Like most of our dev tools this assumes Ubuntu 14 LTS. If that isn't what you
# have, it's probably easiest to run this in a VM.
#
# Note that if this fails you may need to tweak the file list inside
# devel/create_distro_tarball.sh
set -e # exit script if any command returns an error
set -u # exit the script if any variable is uninitialized
function usage {
echo "Usage: devel/build_release_tarball.sh <beta|stable>"
exit 1
}
if [ $# -ne 1 ]; then
usage
fi
if [ ! -d net/instaweb ]; then
echo "This script must be run from the root of the mps checkout."
exit 1
fi
source net/instaweb/public/VERSION
RELEASE="$MAJOR.$MINOR.$BUILD.$PATCH"
CHANNEL="$1"
deps="libpng12-dev libicu-dev libssl-dev libjpeg-dev realpath build-essential
pkg-config gperf unzip libapr1-dev libaprutil1-dev apache2-dev"
if dpkg-query -Wf '${Status}\n' $deps 2>&1 | \
grep -v "install ok installed"; then
# Only run apt-get install if one of the deps is not already installed.
# See: http://stackoverflow.com/questions/1298066
sudo apt-get install $deps
fi
RELEASE_DIR="$HOME/release/$RELEASE"
mkdir -p "$RELEASE_DIR"
REVISION="$(build/lastchange.sh "$PWD" | sed 's/LASTCHANGE=//')"
TARBALL="$RELEASE_DIR/mod-pagespeed-$CHANNEL-$RELEASE-r$REVISION.tar.bz2"
devel/create_distro_tarball.sh "$TARBALL"
echo "Tarball should now be at $TARBALL"
# Try to build it
BUILD_DIR="$(mktemp -d)"
echo "Doing a test build inside $BUILD_DIR"
cd "$BUILD_DIR"
if openssl version | grep "^OpenSSL 1[.]0[.][01]\|^OpenSSL 0[.]"; then
echo "Your openssl version is too old to build the tarball; we need 1.0.2+"
echo "Building 1.0.2 from source..."
OPENSSL_VERSION="1.0.2j"
wget "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
tar -xzvf "openssl-${OPENSSL_VERSION}.tar.gz"
cd openssl-"${OPENSSL_VERSION}"
./config --prefix="$BUILD_DIR" shared
make
make install
export SSL_CERT_DIR=/etc/ssl/certs
export PKG_CONFIG_PATH="$BUILD_DIR/lib/pkgconfig"
export LD_LIBRARY_PATH="$BUILD_DIR/lib"
cd "$BUILD_DIR"
fi
tar xjf "$TARBALL"
cd modpagespeed*
./generate.sh -Dsystem_include_path_apr=/usr/include/apr-1.0/ \
-Dsystem_include_path_httpd=/usr/include/apache2
cd src
make -j6
out/Debug/mod_pagespeed_test
# These tests fail because they are golded against a specific version of
# compression libraries.
# TODO(sligocki): Could we change the tests to be less fragile or test in a
# different way in this case?
BROKEN_TESTS=\
ImageConverterTest.OptimizePngOrConvertToJpeg:\
ImageConverterTest.ConvertOpaqueGifToJpeg:\
JpegOptimizerTest.ValidJpegsLossy:\
JpegOptimizerTest.ValidJpegLossyAndColorSampling:\
JpegOptimizerTest.ValidJpegsProgressiveAndLossy
out/Debug/pagespeed_automatic_test --gtest_filter=-$BROKEN_TESTS
echo "Cleaning up"
rm -rf "$BUILD_DIR"
devel/create_distro_tarball.sh
+87 -119
View File
@@ -1,4 +1,4 @@
#!/bin/sh #!/bin/bash
# #
# Copyright 2011 Google Inc. # Copyright 2011 Google Inc.
# #
@@ -28,14 +28,15 @@
# the .gyp[i] files and doesn't work out of the box. The pruning was also done # the .gyp[i] files and doesn't work out of the box. The pruning was also done
# as of branch 33, so further tweaks might be required for this mode in # as of branch 33, so further tweaks might be required for this mode in
# 34 or newer. # 34 or newer.
#
# This is expected to be run from build_release_tarball.sh, on the branch you
# want a tarball for.
set -e # exit script if any command returns an error set -e # exit script if any command returns an error
set -u # exit the script if any variable is uninitialized set -u # exit the script if any variable is uninitialized
function usage { function usage {
echo -n "create_distro_tarball_debian.sh [ --minimal ] [ --checkout_dir dir ] tarball" echo "create_distro_tarball_debian.sh [ --minimal ] tarball"
echo " [ --checkout commit_ish | source_tree ]"
echo "examples of commit_ish would be 'master', '33', '1.11.33.4' or 04e3237"
exit 1 exit 1
} }
@@ -65,163 +66,130 @@ src/build/gyp_chromium -D use_system_libs=1 \$*
SCRIPT_END SCRIPT_END
} }
if [ $# -lt 2 ]; then if [ $# -lt 1 ]; then
usage usage
exit exit
fi fi
CHECKOUT_DIR=
if [ "$1" = "--checkout_dir" ]; then
if [ -z $2 ]; then
usage
fi
CHECKOUT_DIR=$2
shift 2
else
CHECKOUT_DIR=$(mktemp -d)
echo $CHECKOUT_DIR
fi
MINIMAL=0 MINIMAL=0
if [ "$1" == "--minimal" ]; then if [ "$1" == "--minimal" ]; then
MINIMAL=1 MINIMAL=1
shift 1 shift 1
fi fi
TARBALL=$1 TARBALL="$1"
if [ -z $TARBALL ]; then if [ -z "$TARBALL" ]; then
usage usage
fi fi
touch $TARBALL touch "$TARBALL"
TARBALL=$(realpath $TARBALL) TARBALL="$(realpath $TARBALL)"
MPS_CHECKOUT="$PWD"
if [ "$2" == --checkout ]; then git submodule update --init --recursive
if [ -z $3 ]; then
usage
else
GIT_TAG=$3
if [ -z $CHECKOUT_DIR ]; then
CHECKOUT_DIR=$(mktemp -d)
fi
SRC_DIR=$CHECKOUT_DIR
cd $CHECKOUT_DIR
git clone https://github.com/pagespeed/mod_pagespeed.git src
cd src
git checkout "$GIT_TAG"
git submodule update --init --recursive
fi
else
SRC_DIR=$2
fi
cd $SRC_DIR # Pick up our version info, and wrap src inside a modpagespeed-version dir.
source net/instaweb/public/VERSION
# Pick up our version info, and wrap src inside a modpagespeed-version dir, VER_STRING="$MAJOR.$MINOR.$BUILD.$PATCH"
# so unpackers don't get a src/ surprise TEMP_DIR="$(mktemp -d)"
source src/net/instaweb/public/VERSION WRAPPER_DIR="modpagespeed-$VER_STRING"
VER_STRING=$MAJOR.$MINOR.$BUILD.$PATCH mkdir "$TEMP_DIR/$WRAPPER_DIR"
DIR=modpagespeed-$VER_STRING DIR="$WRAPPER_DIR/src"
rm -rf $DIR ln -s "$MPS_CHECKOUT" "$TEMP_DIR/$DIR"
mkdir $DIR
echo ln -sf $SRC_DIR/src $DIR/src/
ln -sf $SRC_DIR/src $DIR/src
# Also create a little helper script that shows how to run gyp # Also create a little helper script that shows how to run gyp
config > $DIR/generate.sh config > "$TEMP_DIR/$WRAPPER_DIR/generate.sh"
chmod +x $DIR/generate.sh chmod +x "$TEMP_DIR/$WRAPPER_DIR/generate.sh"
# Normally, the build system runs build/lastchange.sh to figure out what # Normally, the build system runs build/lastchange.sh to figure out what
# to put into the last portion of the version number. We are, however, going to # to put into the last portion of the version number. We are, however, going to
# be getting rid of the .git dirs, so that will not work (nor would it without # be getting rid of the .git dirs, so that will not work (nor would it without
# network access). Luckily, we can provide the number via LASTCHANGE.in, # network access). Luckily, we can provide the number via LASTCHANGE.in,
# so we just compute it now, and save it there. # so we just compute it now, and save it there.
cd src/ ./build/lastchange.sh "$MPS_CHECKOUT" > LASTCHANGE.in
./build/lastchange.sh $PWD > LASTCHANGE.in
cd ..
# Things that depends on minimal or not. # Things that depends on minimal or not.
if [ $MINIMAL -eq 0 ]; then if [ $MINIMAL -eq 0 ]; then
GTEST=$DIR/src/testing GTEST=$DIR/testing
GFLAGS=$DIR/src/third_party/gflags GFLAGS=$DIR/third_party/gflags
GIFLIB=$DIR/src/third_party/giflib GIFLIB=$DIR/third_party/giflib
ICU="$DIR/src/third_party/icu/icu.gyp \ ICU="$DIR/third_party/icu/icu.gyp \
$DIR/src/third_party/icu/source/common/unicode/" $DIR/third_party/icu/source/common/unicode/"
JSONCPP=$DIR/src/third_party/jsoncpp JSONCPP=$DIR/third_party/jsoncpp
LIBWEBP=$DIR/src/third_party/libwebp LIBWEBP=$DIR/third_party/libwebp
PROTOBUF=$DIR/src/third_party/protobuf PROTOBUF=$DIR/third_party/protobuf
RE2=$DIR/src/third_party/re2 RE2=$DIR/third_party/re2
else else
GTEST="$DIR/src/testing \ GTEST="$DIR/testing \
--exclude $DIR/src/testing/gmock \ --exclude $DIR/testing/gmock \
--exclude $DIR/src/testing/gtest" --exclude $DIR/testing/gtest"
GFLAGS=$DIR/src/third_party/gflags/gflags.gyp GFLAGS=$DIR/third_party/gflags/gflags.gyp
GIFLIB=$DIR/src/third_party/giflib/giflib.gyp GIFLIB=$DIR/third_party/giflib/giflib.gyp
ICU=$DIR/src/third_party/icu/icu.gyp ICU=$DIR/third_party/icu/icu.gyp
JSONCPP=$DIR/src/third_party/jsoncpp/jsoncpp.gyp JSONCPP=$DIR/third_party/jsoncpp/jsoncpp.gyp
LIBWEBP="$DIR/src/third_party/libwebp/COPYING \ LIBWEBP="$DIR/third_party/libwebp/COPYING \
$DIR/src/third_party/libwebp/examples/gif2webp_util.*" $DIR/third_party/libwebp/examples/gif2webp_util.*"
PROTOBUF="$DIR/src/third_party/protobuf/*.gyp \ PROTOBUF="$DIR/third_party/protobuf/*.gyp \
$DIR/src/third_party/protobuf/*.gypi" $DIR/third_party/protobuf/*.gypi"
RE2=$DIR/src/third_party/re2/re2.gyp RE2=$DIR/third_party/re2/re2.gyp
fi fi
# It's tarball time! # It's tarball time!
# Note that this is highly-version specific, and requires tweaks for every # Note that this is highly-version specific, and requires tweaks for every
# release as its dependencies change. Always run the version of this # release as its dependencies change. Always run the version of this
# script that's on the branch you're making a tarball for. # script that's on the branch you're making a tarball for.
cd "$TEMP_DIR"
tar cj --dereference --exclude='.git' --exclude='.svn' --exclude='.hg' -f $TARBALL \ tar cj --dereference --exclude='.git' --exclude='.svn' --exclude='.hg' -f $TARBALL \
--exclude='*.mk' --exclude='*.pyc' \ --exclude='*.mk' --exclude='*.pyc' \
--exclude=$DIR/src/net/instaweb/genfiles/*/*.cc \ --exclude=$DIR/net/instaweb/genfiles/*/*.cc \
$DIR/generate.sh \ $WRAPPER_DIR/generate.sh \
$DIR/src/LASTCHANGE.in \ $DIR/LASTCHANGE.in \
$DIR/src/base \ $DIR/base \
$DIR/src/build \ $DIR/build \
--exclude $DIR/src/build/android/arm-linux-androideabi-gold \ --exclude $DIR/build/android/arm-linux-androideabi-gold \
$DIR/src/install \ $DIR/install \
$DIR/src/net/instaweb \ $DIR/net/instaweb \
$DIR/src/pagespeed \ $DIR/pagespeed \
$DIR/src/strings \ $DIR/strings \
$GTEST \ $GTEST \
$DIR/src/third_party/apr/apr.gyp \ $DIR/third_party/apr/apr.gyp \
$DIR/src/third_party/aprutil/aprutil.gyp \ $DIR/third_party/aprutil/aprutil.gyp \
$DIR/src/third_party/aprutil/apr_memcache2.h \ $DIR/third_party/aprutil/apr_memcache2.h \
$DIR/src/third_party/aprutil/apr_memcache2.c \ $DIR/third_party/aprutil/apr_memcache2.c \
$DIR/src/third_party/httpd/httpd.gyp \ $DIR/third_party/httpd/httpd.gyp \
$DIR/src/third_party/httpd24/httpd24.gyp \ $DIR/third_party/httpd24/httpd24.gyp \
$DIR/src/third_party/base64 \ $DIR/third_party/base64 \
$DIR/src/third_party/brotli \ $DIR/third_party/brotli \
$DIR/src/third_party/chromium/src/base \ $DIR/third_party/chromium/src/base \
--exclude src/third_party/chromium/src/base/third_party/xdg_mime \ --exclude src/third_party/chromium/src/base/third_party/xdg_mime \
--exclude src/third_party/chromium/src/base/third_party/xdg_user_dirs \ --exclude src/third_party/chromium/src/base/third_party/xdg_user_dirs \
$DIR/src/third_party/chromium/src/build \ $DIR/third_party/chromium/src/build \
--exclude $DIR/src/third_party/chromium/src/build/android \ --exclude $DIR/third_party/chromium/src/build/android \
$DIR/src/third_party/chromium/src/googleurl \ $DIR/third_party/chromium/src/googleurl \
$DIR/src/third_party/chromium/src/net/tools \ $DIR/third_party/chromium/src/net/tools \
$DIR/src/third_party/closure/ \ $DIR/third_party/closure/ \
$DIR/src/third_party/closure_library/ \ $DIR/third_party/closure_library/ \
$DIR/src/third_party/css_parser \ $DIR/third_party/css_parser \
$DIR/src/third_party/domain_registry_provider \ $DIR/third_party/domain_registry_provider \
$GFLAGS \ $GFLAGS \
$GIFLIB \ $GIFLIB \
$DIR/src/third_party/google-sparsehash \ $DIR/third_party/google-sparsehash \
$DIR/src/third_party/grpc \ $DIR/third_party/grpc \
$DIR/src/third_party/hiredis \ $DIR/third_party/hiredis \
$ICU \ $ICU \
$JSONCPP \ $JSONCPP \
$DIR/src/third_party/libjpeg_turbo/libjpeg_turbo.gyp \ $DIR/third_party/libjpeg_turbo/libjpeg_turbo.gyp \
$DIR/src/third_party/libpng/libpng.gyp \ $DIR/third_party/libpng/libpng.gyp \
$LIBWEBP \ $LIBWEBP \
$DIR/src/third_party/modp_b64 \ $DIR/third_party/modp_b64 \
$DIR/src/third_party/optipng \ $DIR/third_party/optipng \
$PROTOBUF \ $PROTOBUF \
$DIR/src/third_party/rdestl \ $DIR/third_party/rdestl \
$RE2 \ $RE2 \
$DIR/src/third_party/redis-crc \ $DIR/third_party/redis-crc \
$DIR/src/third_party/serf \ $DIR/third_party/serf \
$DIR/src/third_party/zlib/zlib.gyp \ $DIR/third_party/zlib/zlib.gyp \
$DIR/src/tools/gyp \ $DIR/tools/gyp \
$DIR/src/url $DIR/url
if [ x != x$CHECKOUT_DIR ]; then cd "$MPS_CHECKOUT"
echo "You may want to rm -rf $CHECKOUT_DIR to clean up" rm -r "$TEMP_DIR"
fi