From 498f00a2447a3c873c239313beeb5dc263bf5c74 Mon Sep 17 00:00:00 2001 From: Chris Rebert Date: Sat, 14 Dec 2013 21:02:15 -0800 Subject: use S3-cached Jekyll install --- .travis.yml | 12 +++-- test-infra/node_modules_cache.py | 107 --------------------------------------- test-infra/s3_cache.py | 107 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 111 deletions(-) delete mode 100755 test-infra/node_modules_cache.py create mode 100755 test-infra/s3_cache.py diff --git a/.travis.yml b/.travis.yml index 47e28bf23..be6353802 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,19 @@ language: node_js node_js: - 0.10 before_install: - - time sudo pip install --use-mirrors -r ./test-infra/requirements.txt + - time sudo pip install --use-mirrors -r test-infra/requirements.txt + - rvm use 1.9.3 --fuzzy + - if [ "$TWBS_TEST" = validate-html ]; then echo "ruby=$(basename $(rvm gemdir)) jekyll=$JEKYLL_VERSION" > pseudo_Gemfile.lock; fi install: - - if [ "$TWBS_TEST" = validate-html ]; then time gem install jekyll; fi - time npm install -g grunt-cli - - time ./test-infra/node_modules_cache.py download package.json ./node_modules || time npm install + - time ./test-infra/s3_cache.py download 'node.js packages' package.json ./node_modules || time npm install + - if [ "$TWBS_TEST" = validate-html ]; then time ./test-infra/s3_cache.py download rubygems pseudo_Gemfile.lock $(rvm gemdir) || gem install -N jekyll -v $JEKYLL_VERSION; fi after_script: - - if [ "$TWBS_TEST" = core ]; then time ./test-infra/node_modules_cache.py upload package.json ./node_modules; fi + - if [ "$TWBS_TEST" = core ]; then time ./test-infra/s3_cache.py upload 'node.js packages' package.json ./node_modules; fi + - if [ "$TWBS_TEST" = validate-html ]; then time ./test-infra/s3_cache.py upload rubygems pseudo_Gemfile.lock $(rvm gemdir); fi env: global: + - JEKYLL_VERSION: 1.4.1 - SAUCE_USERNAME: bootstrap - secure: "pJkBwnuae9dKU5tEcCqccfS1QQw7/meEcfz63fM7ba7QJNjoA6BaXj08L5Z3Vb5vBmVPwBawxo5Hp0jC0r/Z/O0hGnAmz/Cz09L+cy7dSAZ9x4hvZePSja/UAusaB5ogMoO8l2b773MzgQeSmrLbExr9BWLeqEfjC2hFgdgHLaQ=" - secure: "gqjqISbxBJK6byFbsmr1AyP1qoWH+rap06A2gI7v72+Tn2PU2nYkIMUkCvhZw6K889jv+LhQ/ybcBxDOXHpNCExCnSgB4dcnmYp+9oeNZb37jSP0rQ+Ib4OTLjzc3/FawE/fUq5kukZTC7porzc/k0qJNLAZRx3YLALmK1GIdUY=" diff --git a/test-infra/node_modules_cache.py b/test-infra/node_modules_cache.py deleted file mode 100755 index 6acddb38a..000000000 --- a/test-infra/node_modules_cache.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python2.7 -from __future__ import absolute_import, unicode_literals, print_function, division - -from sys import argv -from os import environ, stat, remove as _delete_file -from os.path import isfile, dirname, basename, abspath -from hashlib import sha256 -from subprocess import check_call as run - -from boto.s3.connection import S3Connection -from boto.s3.key import Key -from boto.exception import S3ResponseError - - -NEED_TO_UPLOAD_MARKER = '.need-to-upload' -BYTES_PER_MB = 1024 * 1024 -try: - BUCKET_NAME = environ['TWBS_S3_BUCKET'] -except KeyError: - raise SystemExit("TWBS_S3_BUCKET environment variable not set!") - - -def _sha256_of_file(filename): - hasher = sha256() - with open(filename, 'rb') as input_file: - hasher.update(input_file.read()) - file_hash = hasher.hexdigest() - print('sha256({}) = {}'.format(filename, file_hash)) - return file_hash - - -def _delete_file_quietly(filename): - try: - _delete_file(filename) - except (OSError, IOError): - pass - - -def _tarball_size(directory): - kib = stat(_tarball_filename_for(directory)).st_size // BYTES_PER_MB - return "{} MiB".format(kib) - - -def _tarball_filename_for(directory): - return abspath('./{}.tar.gz'.format(basename(directory))) - - -def _create_tarball(directory): - print("Creating tarball of {}...".format(directory)) - run(['tar', '-czf', _tarball_filename_for(directory), '-C', dirname(directory), basename(directory)]) - - -def _extract_tarball(directory): - print("Extracting tarball of {}...".format(directory)) - run(['tar', '-xzf', _tarball_filename_for(directory), '-C', dirname(directory)]) - - -def download(directory): - _delete_file_quietly(NEED_TO_UPLOAD_MARKER) - try: - print("Downloading {} tarball from S3...".format(basename(directory))) - key.get_contents_to_filename(_tarball_filename_for(directory)) - except S3ResponseError as err: - open(NEED_TO_UPLOAD_MARKER, 'a').close() - print(err) - raise SystemExit("Cached {} download failed!".format(basename(directory))) - print("Downloaded {}.".format(_tarball_size(directory))) - _extract_tarball(directory) - print("{} successfully installed from cache.".format(directory)) - - -def upload(directory): - _create_tarball(directory) - print("Uploading {} tarball to S3... ({})".format(basename(directory), _tarball_size(directory))) - key.set_contents_from_filename(_tarball_filename_for(directory)) - print("{} cache successfully updated.".format(directory)) - _delete_file_quietly(NEED_TO_UPLOAD_MARKER) - - -if __name__ == '__main__': - # Uses environment variables: - # AWS_ACCESS_KEY_ID - AWS Access Key ID - # AWS_SECRET_ACCESS_KEY - AWS Secret Access Key - argv.pop(0) - if len(argv) != 3: - raise SystemExit("USAGE: node_modules_cache.py ") - mode, dependencies_file, directory = argv - - conn = S3Connection() - bucket = conn.lookup(BUCKET_NAME) - if bucket is None: - raise SystemExit("Could not access bucket!") - - dependencies_file_hash = _sha256_of_file(dependencies_file) - - key = Key(bucket, dependencies_file_hash) - key.storage_class = 'REDUCED_REDUNDANCY' - - if mode == 'download': - download(directory) - elif mode == 'upload': - if isfile(NEED_TO_UPLOAD_MARKER): # FIXME - upload(directory) - else: - print("No need to upload anything.") - else: - raise SystemExit("Unrecognized mode {!r}".format(mode)) diff --git a/test-infra/s3_cache.py b/test-infra/s3_cache.py new file mode 100755 index 000000000..6f099a19f --- /dev/null +++ b/test-infra/s3_cache.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python2.7 +from __future__ import absolute_import, unicode_literals, print_function, division + +from sys import argv +from os import environ, stat, remove as _delete_file +from os.path import isfile, dirname, basename, abspath +from hashlib import sha256 +from subprocess import check_call as run + +from boto.s3.connection import S3Connection +from boto.s3.key import Key +from boto.exception import S3ResponseError + + +NEED_TO_UPLOAD_MARKER = '.need-to-upload' +BYTES_PER_MB = 1024 * 1024 +try: + BUCKET_NAME = environ['TWBS_S3_BUCKET'] +except KeyError: + raise SystemExit("TWBS_S3_BUCKET environment variable not set!") + + +def _sha256_of_file(filename): + hasher = sha256() + with open(filename, 'rb') as input_file: + hasher.update(input_file.read()) + file_hash = hasher.hexdigest() + print('sha256({}) = {}'.format(filename, file_hash)) + return file_hash + + +def _delete_file_quietly(filename): + try: + _delete_file(filename) + except (OSError, IOError): + pass + + +def _tarball_size(directory): + kib = stat(_tarball_filename_for(directory)).st_size // BYTES_PER_MB + return "{} MiB".format(kib) + + +def _tarball_filename_for(directory): + return abspath('./{}.tar.gz'.format(basename(directory))) + + +def _create_tarball(directory): + print("Creating tarball of {}...".format(directory)) + run(['tar', '-czf', _tarball_filename_for(directory), '-C', dirname(directory), basename(directory)]) + + +def _extract_tarball(directory): + print("Extracting tarball of {}...".format(directory)) + run(['tar', '-xzf', _tarball_filename_for(directory), '-C', dirname(directory)]) + + +def download(directory): + _delete_file_quietly(NEED_TO_UPLOAD_MARKER) + try: + print("Downloading {} tarball from S3...".format(friendly_name)) + key.get_contents_to_filename(_tarball_filename_for(directory)) + except S3ResponseError as err: + open(NEED_TO_UPLOAD_MARKER, 'a').close() + print(err) + raise SystemExit("Cached {} download failed!".format(friendly_name)) + print("Downloaded {}.".format(_tarball_size(directory))) + _extract_tarball(directory) + print("{} successfully installed from cache.".format(friendly_name)) + + +def upload(directory): + _create_tarball(directory) + print("Uploading {} tarball to S3... ({})".format(friendly_name, _tarball_size(directory))) + key.set_contents_from_filename(_tarball_filename_for(directory)) + print("{} cache successfully updated.".format(friendly_name)) + _delete_file_quietly(NEED_TO_UPLOAD_MARKER) + + +if __name__ == '__main__': + # Uses environment variables: + # AWS_ACCESS_KEY_ID - AWS Access Key ID + # AWS_SECRET_ACCESS_KEY - AWS Secret Access Key + argv.pop(0) + if len(argv) != 4: + raise SystemExit("USAGE: node_modules_cache.py ") + mode, friendly_name, dependencies_file, directory = argv + + conn = S3Connection() + bucket = conn.lookup(BUCKET_NAME) + if bucket is None: + raise SystemExit("Could not access bucket!") + + dependencies_file_hash = _sha256_of_file(dependencies_file) + + key = Key(bucket, dependencies_file_hash) + key.storage_class = 'REDUCED_REDUNDANCY' + + if mode == 'download': + download(directory) + elif mode == 'upload': + if isfile(NEED_TO_UPLOAD_MARKER): # FIXME + upload(directory) + else: + print("No need to upload anything.") + else: + raise SystemExit("Unrecognized mode {!r}".format(mode)) -- cgit v1.2.3