build release tarball: open source it (#1474)

* build release tarball: open source it * change it from being given a branch to working on the current checkout * change it to work with create_distro_tarball, which was already opensourced * make it build openssl 1.0.2 if needed * doc ubuntu 14 dep
2017-01-11 15:19:44 -05:00
parent 58c175ca32
commit 6a15917500
2 changed files with 183 additions and 119 deletions
@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# Note: you might need to type your password a few times, once early, and once
+# at the end.
+#
+# This should be run on a release branch to make sure we can make a tarball and
+# at least build it on our workstations.  It will also copy the tarball into
+# ~/release (where the binaries usually go).
+#
+# Like most of our dev tools this assumes Ubuntu 14 LTS.  If that isn't what you
+# have, it's probably easiest to run this in a VM.
+#
+# Note that if this fails you may need to tweak the file list inside
+# devel/create_distro_tarball.sh
+
+set -e  # exit script if any command returns an error
+set -u  # exit the script if any variable is uninitialized
+
+function usage {
+  echo "Usage: devel/build_release_tarball.sh <beta|stable>"
+  exit 1
+}
+
+if [ $# -ne 1 ]; then
+  usage
+fi
+
+if [ ! -d net/instaweb ]; then
+  echo "This script must be run from the root of the mps checkout."
+  exit 1
+fi
+
+source net/instaweb/public/VERSION
+RELEASE="$MAJOR.$MINOR.$BUILD.$PATCH"
+CHANNEL="$1"
+
+deps="libpng12-dev libicu-dev libssl-dev libjpeg-dev realpath build-essential
+      pkg-config gperf unzip libapr1-dev libaprutil1-dev apache2-dev"
+if dpkg-query -Wf '${Status}\n' $deps 2>&1 | \
+     grep -v "install ok installed"; then
+  # Only run apt-get install if one of the deps is not already installed.
+  # See: http://stackoverflow.com/questions/1298066
+  sudo apt-get install $deps
+fi
+
+RELEASE_DIR="$HOME/release/$RELEASE"
+mkdir -p "$RELEASE_DIR"
+REVISION="$(build/lastchange.sh "$PWD" | sed 's/LASTCHANGE=//')"
+TARBALL="$RELEASE_DIR/mod-pagespeed-$CHANNEL-$RELEASE-r$REVISION.tar.bz2"
+devel/create_distro_tarball.sh "$TARBALL"
+
+echo "Tarball should now be at $TARBALL"
+
+# Try to build it
+BUILD_DIR="$(mktemp -d)"
+echo "Doing a test build inside $BUILD_DIR"
+cd "$BUILD_DIR"
+
+if openssl version | grep "^OpenSSL 1[.]0[.][01]\|^OpenSSL 0[.]"; then
+  echo "Your openssl version is too old to build the tarball; we need 1.0.2+"
+  echo "Building 1.0.2 from source..."
+  OPENSSL_VERSION="1.0.2j"
+  wget "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz"
+  tar -xzvf "openssl-${OPENSSL_VERSION}.tar.gz"
+  cd openssl-"${OPENSSL_VERSION}"
+  ./config --prefix="$BUILD_DIR" shared
+  make
+  make install
+  export SSL_CERT_DIR=/etc/ssl/certs
+  export PKG_CONFIG_PATH="$BUILD_DIR/lib/pkgconfig"
+  export LD_LIBRARY_PATH="$BUILD_DIR/lib"
+  cd "$BUILD_DIR"
+fi
+
+tar xjf "$TARBALL"
+cd modpagespeed*
+./generate.sh -Dsystem_include_path_apr=/usr/include/apr-1.0/ \
+              -Dsystem_include_path_httpd=/usr/include/apache2
+cd src
+make -j6
+out/Debug/mod_pagespeed_test
+# These tests fail because they are golded against a specific version of
+# compression libraries.
+# TODO(sligocki): Could we change the tests to be less fragile or test in a
+# different way in this case?
+BROKEN_TESTS=\
+ImageConverterTest.OptimizePngOrConvertToJpeg:\
+ImageConverterTest.ConvertOpaqueGifToJpeg:\
+JpegOptimizerTest.ValidJpegsLossy:\
+JpegOptimizerTest.ValidJpegLossyAndColorSampling:\
+JpegOptimizerTest.ValidJpegsProgressiveAndLossy
+out/Debug/pagespeed_automatic_test --gtest_filter=-$BROKEN_TESTS
+
+echo "Cleaning up"
+rm -rf "$BUILD_DIR"
+
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Copyright 2011 Google Inc.
 #
@@ -28,14 +28,15 @@
 # the .gyp[i] files and doesn't work out of the box. The pruning was also done
 # as of branch 33, so further tweaks might be required for this mode in
 # 34 or newer.
+#
+# This is expected to be run from build_release_tarball.sh, on the branch you
+# want a tarball for.

 set -e  # exit script if any command returns an error
 set -u  # exit the script if any variable is uninitialized

 function usage {
-  echo -n "create_distro_tarball_debian.sh [ --minimal ] [ --checkout_dir dir ] tarball"
-  echo " [ --checkout commit_ish | source_tree ]"
-  echo "examples of commit_ish would be 'master', '33', '1.11.33.4' or 04e3237"
+  echo "create_distro_tarball_debian.sh [ --minimal ] tarball"
  exit 1
 }

@@ -65,163 +66,130 @@ src/build/gyp_chromium -D use_system_libs=1 \$*
 SCRIPT_END
 }

-if [ $# -lt 2 ]; then
+if [ $# -lt 1 ]; then
  usage
  exit
 fi

-CHECKOUT_DIR=
-if [ "$1" = "--checkout_dir" ]; then
-  if [ -z $2 ]; then
-    usage
-  fi
-  CHECKOUT_DIR=$2
-  shift 2
-else
-  CHECKOUT_DIR=$(mktemp -d)
-  echo $CHECKOUT_DIR
-fi
-
 MINIMAL=0
 if [ "$1" == "--minimal" ]; then
  MINIMAL=1
  shift 1
 fi

-TARBALL=$1
-if [ -z $TARBALL ]; then
+TARBALL="$1"
+if [ -z "$TARBALL" ]; then
  usage
 fi
-touch $TARBALL
-TARBALL=$(realpath $TARBALL)
+touch "$TARBALL"
+TARBALL="$(realpath $TARBALL)"
+MPS_CHECKOUT="$PWD"

-if [ "$2" == --checkout ]; then
-  if [ -z $3 ]; then
-    usage
-  else
-    GIT_TAG=$3
-    if [ -z $CHECKOUT_DIR ]; then
-      CHECKOUT_DIR=$(mktemp -d)
-    fi
-    SRC_DIR=$CHECKOUT_DIR
-    cd $CHECKOUT_DIR
-    git clone https://github.com/pagespeed/mod_pagespeed.git src
-    cd src
-    git checkout "$GIT_TAG"
-    git submodule update --init --recursive
-  fi
-else
-  SRC_DIR=$2
-fi
+git submodule update --init --recursive

-cd $SRC_DIR
-
-# Pick up our version info, and wrap src inside a modpagespeed-version dir,
-# so unpackers don't get a src/ surprise
-source src/net/instaweb/public/VERSION
-VER_STRING=$MAJOR.$MINOR.$BUILD.$PATCH
-DIR=modpagespeed-$VER_STRING
-rm -rf $DIR
-mkdir $DIR
-echo ln -sf $SRC_DIR/src $DIR/src/
-ln -sf $SRC_DIR/src $DIR/src
+# Pick up our version info, and wrap src inside a modpagespeed-version dir.
+source net/instaweb/public/VERSION
+VER_STRING="$MAJOR.$MINOR.$BUILD.$PATCH"
+TEMP_DIR="$(mktemp -d)"
+WRAPPER_DIR="modpagespeed-$VER_STRING"
+mkdir "$TEMP_DIR/$WRAPPER_DIR"
+DIR="$WRAPPER_DIR/src"
+ln -s "$MPS_CHECKOUT" "$TEMP_DIR/$DIR"

 # Also create a little helper script that shows how to run gyp
-config > $DIR/generate.sh
-chmod +x $DIR/generate.sh
+config > "$TEMP_DIR/$WRAPPER_DIR/generate.sh"
+chmod +x "$TEMP_DIR/$WRAPPER_DIR/generate.sh"

 # Normally, the build system runs build/lastchange.sh to figure out what
 # to put into the last portion of the version number. We are, however, going to
 # be getting rid of the .git dirs, so that will not work (nor would it without
 # network access). Luckily, we can provide the number via LASTCHANGE.in,
 # so we just compute it now, and save it there.
-cd src/
-./build/lastchange.sh $PWD > LASTCHANGE.in
-cd ..
+./build/lastchange.sh "$MPS_CHECKOUT" > LASTCHANGE.in

 # Things that depends on minimal or not.
 if [ $MINIMAL -eq 0 ]; then
-  GTEST=$DIR/src/testing
-  GFLAGS=$DIR/src/third_party/gflags
-  GIFLIB=$DIR/src/third_party/giflib
-  ICU="$DIR/src/third_party/icu/icu.gyp \
-      $DIR/src/third_party/icu/source/common/unicode/"
-  JSONCPP=$DIR/src/third_party/jsoncpp
-  LIBWEBP=$DIR/src/third_party/libwebp
-  PROTOBUF=$DIR/src/third_party/protobuf
-  RE2=$DIR/src/third_party/re2
+  GTEST=$DIR/testing
+  GFLAGS=$DIR/third_party/gflags
+  GIFLIB=$DIR/third_party/giflib
+  ICU="$DIR/third_party/icu/icu.gyp \
+      $DIR/third_party/icu/source/common/unicode/"
+  JSONCPP=$DIR/third_party/jsoncpp
+  LIBWEBP=$DIR/third_party/libwebp
+  PROTOBUF=$DIR/third_party/protobuf
+  RE2=$DIR/third_party/re2
 else
-  GTEST="$DIR/src/testing \
-      --exclude $DIR/src/testing/gmock \
-      --exclude $DIR/src/testing/gtest"
-  GFLAGS=$DIR/src/third_party/gflags/gflags.gyp
-  GIFLIB=$DIR/src/third_party/giflib/giflib.gyp
-  ICU=$DIR/src/third_party/icu/icu.gyp
-  JSONCPP=$DIR/src/third_party/jsoncpp/jsoncpp.gyp
-  LIBWEBP="$DIR/src/third_party/libwebp/COPYING \
-      $DIR/src/third_party/libwebp/examples/gif2webp_util.*"
-  PROTOBUF="$DIR/src/third_party/protobuf/*.gyp \
-      $DIR/src/third_party/protobuf/*.gypi"
-  RE2=$DIR/src/third_party/re2/re2.gyp
+  GTEST="$DIR/testing \
+      --exclude $DIR/testing/gmock \
+      --exclude $DIR/testing/gtest"
+  GFLAGS=$DIR/third_party/gflags/gflags.gyp
+  GIFLIB=$DIR/third_party/giflib/giflib.gyp
+  ICU=$DIR/third_party/icu/icu.gyp
+  JSONCPP=$DIR/third_party/jsoncpp/jsoncpp.gyp
+  LIBWEBP="$DIR/third_party/libwebp/COPYING \
+      $DIR/third_party/libwebp/examples/gif2webp_util.*"
+  PROTOBUF="$DIR/third_party/protobuf/*.gyp \
+      $DIR/third_party/protobuf/*.gypi"
+  RE2=$DIR/third_party/re2/re2.gyp
 fi

 # It's tarball time!
 # Note that this is highly-version specific, and requires tweaks for every
 # release as its dependencies change.  Always run the version of this
 # script that's on the branch you're making a tarball for.
+cd "$TEMP_DIR"
 tar cj --dereference --exclude='.git' --exclude='.svn' --exclude='.hg' -f $TARBALL \
    --exclude='*.mk' --exclude='*.pyc' \
-    --exclude=$DIR/src/net/instaweb/genfiles/*/*.cc \
-    $DIR/generate.sh \
-    $DIR/src/LASTCHANGE.in \
-    $DIR/src/base \
-    $DIR/src/build \
-    --exclude $DIR/src/build/android/arm-linux-androideabi-gold \
-    $DIR/src/install \
-    $DIR/src/net/instaweb \
-    $DIR/src/pagespeed \
-    $DIR/src/strings \
+    --exclude=$DIR/net/instaweb/genfiles/*/*.cc \
+    $WRAPPER_DIR/generate.sh \
+    $DIR/LASTCHANGE.in \
+    $DIR/base \
+    $DIR/build \
+    --exclude $DIR/build/android/arm-linux-androideabi-gold \
+    $DIR/install \
+    $DIR/net/instaweb \
+    $DIR/pagespeed \
+    $DIR/strings \
    $GTEST \
-    $DIR/src/third_party/apr/apr.gyp \
-    $DIR/src/third_party/aprutil/aprutil.gyp \
-    $DIR/src/third_party/aprutil/apr_memcache2.h \
-    $DIR/src/third_party/aprutil/apr_memcache2.c \
-    $DIR/src/third_party/httpd/httpd.gyp \
-    $DIR/src/third_party/httpd24/httpd24.gyp \
-    $DIR/src/third_party/base64 \
-    $DIR/src/third_party/brotli \
-    $DIR/src/third_party/chromium/src/base \
+    $DIR/third_party/apr/apr.gyp \
+    $DIR/third_party/aprutil/aprutil.gyp \
+    $DIR/third_party/aprutil/apr_memcache2.h \
+    $DIR/third_party/aprutil/apr_memcache2.c \
+    $DIR/third_party/httpd/httpd.gyp \
+    $DIR/third_party/httpd24/httpd24.gyp \
+    $DIR/third_party/base64 \
+    $DIR/third_party/brotli \
+    $DIR/third_party/chromium/src/base \
    --exclude src/third_party/chromium/src/base/third_party/xdg_mime \
    --exclude src/third_party/chromium/src/base/third_party/xdg_user_dirs \
-    $DIR/src/third_party/chromium/src/build \
-    --exclude $DIR/src/third_party/chromium/src/build/android \
-    $DIR/src/third_party/chromium/src/googleurl \
-    $DIR/src/third_party/chromium/src/net/tools \
-    $DIR/src/third_party/closure/ \
-    $DIR/src/third_party/closure_library/ \
-    $DIR/src/third_party/css_parser \
-    $DIR/src/third_party/domain_registry_provider \
+    $DIR/third_party/chromium/src/build \
+    --exclude $DIR/third_party/chromium/src/build/android \
+    $DIR/third_party/chromium/src/googleurl \
+    $DIR/third_party/chromium/src/net/tools \
+    $DIR/third_party/closure/ \
+    $DIR/third_party/closure_library/ \
+    $DIR/third_party/css_parser \
+    $DIR/third_party/domain_registry_provider \
    $GFLAGS \
    $GIFLIB \
-    $DIR/src/third_party/google-sparsehash \
-    $DIR/src/third_party/grpc \
-    $DIR/src/third_party/hiredis \
+    $DIR/third_party/google-sparsehash \
+    $DIR/third_party/grpc \
+    $DIR/third_party/hiredis \
    $ICU \
    $JSONCPP \
-    $DIR/src/third_party/libjpeg_turbo/libjpeg_turbo.gyp \
-    $DIR/src/third_party/libpng/libpng.gyp \
+    $DIR/third_party/libjpeg_turbo/libjpeg_turbo.gyp \
+    $DIR/third_party/libpng/libpng.gyp \
    $LIBWEBP \
-    $DIR/src/third_party/modp_b64 \
-    $DIR/src/third_party/optipng \
+    $DIR/third_party/modp_b64 \
+    $DIR/third_party/optipng \
    $PROTOBUF  \
-    $DIR/src/third_party/rdestl \
+    $DIR/third_party/rdestl \
    $RE2 \
-    $DIR/src/third_party/redis-crc \
-    $DIR/src/third_party/serf \
-    $DIR/src/third_party/zlib/zlib.gyp \
-    $DIR/src/tools/gyp \
-    $DIR/src/url
+    $DIR/third_party/redis-crc \
+    $DIR/third_party/serf \
+    $DIR/third_party/zlib/zlib.gyp \
+    $DIR/tools/gyp \
+    $DIR/url

-if [ x != x$CHECKOUT_DIR ]; then
-  echo "You may want to rm -rf $CHECKOUT_DIR to clean up"
-fi
+cd "$MPS_CHECKOUT"
+rm -r "$TEMP_DIR"