Merge pull request #200 from ascribe/fix/s3deployment

Fix S3 deployment

.travis.yml

@@ -9,17 +9,12 @@ cache:
 
 before_install:
     - nvm install 9
+    - pip install --user awscli
+    - export PATH=$PATH:$HOME/.local/bin
 
 before_script: "_ci/setup.sh"
 script: "_ci/build.sh"
 after_success: "_ci/deploy.sh"
 
-# deploy:
-#   skip_cleanup: true
-#   provider: script
-#   script: "_ci/deploy.sh"
-#   on:
-#     all_branches: true
-
 notifications:
     email: false

_ci/deploy.sh

@@ -1,4 +1,11 @@
 #!/usr/bin/env bash
+#
+# Required global environment variables:
+#
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+# AWS_DEFAULT_REGION
+#
 
 set -e;
 
@@ -9,7 +16,6 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ "$TRAVIS_BRANCH" == "master" ]; th
 
     gulp deploy --beta;
 
-
 ##
 ## check for master push which is no pull request
 ##
@@ -19,18 +25,14 @@ elif [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ];
 
 else
 
-    echo "$(tput setaf 64)" # green
+    echo "$(tput setaf 64)---------------------------------------------"
-    echo "---------------------------------------------"
     echo "      ✓ nothing to deploy "
-    echo "---------------------------------------------"
+    echo "---------------------------------------------$(tput sgr0)"
-    echo "$(tput sgr0)" # reset
 
 fi;
 
-echo "$(tput setaf 64)" # green
+echo "$(tput setaf 64)---------------------------------------------"
-echo "---------------------------------------------"
 echo "         ✓ done deployment "
-echo "---------------------------------------------"
+echo "---------------------------------------------$(tput sgr0)"
-echo "$(tput sgr0)" # reset
 
 exit;

gulpfile.babel.js

@@ -17,6 +17,7 @@ import request      from 'request'
 import uglifyjs     from 'uglify-es'
 import composer     from 'gulp-uglify/composer'
 const minify = composer(uglifyjs, console)
+const cp = require('child_process')
 
 // get all the configs: `pkg` and `site`
 import pkg from './package.json'
@@ -56,16 +57,15 @@ console.log("")
 const PORT = 1337
 
 // paths
-const SRC      = site.source + '/',
+const SRC      = site.source,
-      DIST     = site.destination + '/'
+      DIST     = site.destination
 
 // deployment
-const S3BUCKET         = 'www.bigchaindb.com',
+const S3_BUCKET_LIVE     = 'www.bigchaindb.com',
-      S3REGION         = 'eu-central-1',
+      S3_BUCKET_BETA     = 'beta.bigchaindb.com',
-      S3BUCKET_BETA    = 'beta.bigchaindb.com',
+      S3_BUCKET_GAMMA    = 'gamma.bigchaindb.com',
-      S3REGION_BETA    = 'eu-central-1',
+      S3_OPTIONS_DEFAULT = '--delete --acl public-read',
-      S3BUCKET_GAMMA   = 'gamma.bigchaindb.com',
+      S3_OPTIONS_CACHING = '--cache-control max-age=2592000,public'
-      S3REGION_GAMMA   = 'eu-central-1'
 
 // SVG sprite
 const SPRITECONFIG = {
@@ -100,8 +100,8 @@ const BANNER = [
 //
 export const clean = () =>
     del([
-        DIST + '**/*',
+        DIST + '/**/*',
-        DIST + '.*' // delete all hidden files
+        DIST + '/.*' // delete all hidden files
     ])
 
 
@@ -123,8 +123,7 @@ export const jekyll = (done) => {
         var jekyll_options = 'jekyll build --incremental --drafts --future'
     }
 
-    let spawn  = require('child_process').spawn,
+    const jekyll = cp.execFile('bundle', ['exec', jekyll_options], { stdio: 'inherit' })
-        jekyll = spawn('bundle', ['exec', jekyll_options], { stdio: 'inherit' })
 
     jekyll.on('error', (error) => onError() ).on('close', done)
 }
@@ -133,7 +132,7 @@ export const jekyll = (done) => {
 //
 // HTML
 //
-export const html = () => src(DIST + '**/*.html')
+export const html = () => src(DIST + '/**/*.html')
     .pipe($.if(isProduction || isStaging, $.htmlmin({
         collapseWhitespace: true,
         conservativeCollapse: true,
@@ -151,7 +150,7 @@ export const html = () => src(DIST + '**/*.html')
 //
 // Styles
 //
-export const css = () => src(SRC + '_assets/styles/bigchain.scss')
+export const css = () => src(SRC + '/_assets/styles/bigchain.scss')
     .pipe($.if(!(isProduction || isStaging), $.sourcemaps.init()))
     .pipe($.sass({
         includePaths: ['node_modules']
@@ -161,7 +160,7 @@ export const css = () => src(SRC + '_assets/styles/bigchain.scss')
     .pipe($.if(!(isProduction || isStaging), $.sourcemaps.write()))
     .pipe($.if(isProduction || isStaging, $.header(BANNER, { pkg: pkg })))
     .pipe($.rename({ suffix: '.min' }))
-    .pipe(dest(DIST + 'assets/css/'))
+    .pipe(dest(DIST + '/assets/css/'))
     .pipe(browser.stream())
 
 // inline critical-path CSS
@@ -194,55 +193,55 @@ export const criticalCss = (done) => {
 //
 export const js = () =>
     src([
-        SRC + '_assets/javascripts/bigchain.js',
+        SRC + '/_assets/javascripts/bigchain.js',
-        SRC + '_assets/javascripts/page-*.js'
+        SRC + '/_assets/javascripts/page-*.js'
     ])
     .pipe($.if(!(isProduction || isStaging), $.sourcemaps.init()))
     .pipe($.include({
-        includePaths: ['node_modules', SRC + '_assets/javascripts']
+        includePaths: ['node_modules', SRC + '/_assets/javascripts']
     })).on('error', onError)
     .pipe($.if(isProduction || isStaging, minify())).on('error', onError)
     .pipe($.if(!(isProduction || isStaging), $.sourcemaps.write()))
     .pipe($.if(isProduction || isStaging, $.header(BANNER, { pkg: pkg })))
     .pipe($.rename({suffix: '.min'}))
-    .pipe(dest(DIST + 'assets/js/'))
+    .pipe(dest(DIST + '/assets/js/'))
 
 
 //
 // SVG sprite
 //
-export const svg = () => src(SRC + '_assets/images/*.svg')
+export const svg = () => src(SRC + '/_assets/images/*.svg')
     .pipe($.if(isProduction || isStaging, $.imagemin({
         svgoPlugins: [{ removeRasterImages: true }]
     })))
     .pipe($.svgSprite(SPRITECONFIG))
-    .pipe(dest(DIST + 'assets/img/'))
+    .pipe(dest(DIST + '/assets/img/'))
 
 
 //
 // Copy Images
 //
-export const images = () => src(SRC + '_assets/images/**/*')
+export const images = () => src(SRC + '/_assets/images/**/*')
     .pipe($.if(isProduction || isStaging, $.imagemin([
     	$.imagemin.gifsicle({ interlaced: true }),
     	$.imagemin.jpegtran({ progressive: true }),
     	$.imagemin.optipng({ optimizationLevel: 5 }),
     	$.imagemin.svgo({plugins: [{ removeViewBox: true }]})
     ])))
-    .pipe(dest(DIST + 'assets/img/'))
+    .pipe(dest(DIST + '/assets/img/'))
 
 
 //
 // Copy Fonts
 //
-export const fonts = () => src(SRC + '_assets/fonts/**/*')
+export const fonts = () => src(SRC + '/_assets/fonts/**/*')
-    .pipe(dest(DIST + 'assets/fonts/'))
+    .pipe(dest(DIST + '/assets/fonts/'))
 
 
 //
 // Zip up media kit
 //
-export const mediakit = () => src(SRC + 'mediakit/**/*', { base: SRC })
+export const mediakit = () => src(SRC + '/mediakit/**/*', { base: SRC })
     .pipe($.zip('mediakit.zip'))
     .pipe(dest(DIST))
 
@@ -253,12 +252,12 @@ export const mediakit = () => src(SRC + 'mediakit/**/*', { base: SRC })
 export const rev = (done) => {
     // globbing is slow so do everything conditionally for faster dev build
     if (isProduction || isStaging) {
-        return src(DIST + 'assets/**/*.{css,js,png,jpg,jpeg,svg,eot,ttf,woff,woff2}')
+        return src(DIST + '/assets/**/*.{css,js,png,jpg,jpeg,svg,eot,ttf,woff,woff2}')
             .pipe($.rev())
-            .pipe(dest(DIST + 'assets/'))
+            .pipe(dest(DIST + '/assets/'))
             // output rev manifest for next replace task
             .pipe($.rev.manifest())
-            .pipe(dest(DIST + 'assets/'))
+            .pipe(dest(DIST + '/assets/'))
     }
     done()
 }
@@ -271,9 +270,9 @@ export const rev = (done) => {
 export const revReplace = (done) => {
     // globbing is slow so do everything conditionally for faster dev build
     if (isProduction || isStaging) {
-        let manifest = src(DIST + 'assets/rev-manifest.json')
+        let manifest = src(DIST + '/assets/rev-manifest.json')
 
-        return src(DIST + '**/*.{html,css,js}')
+        return src(DIST + '/**/*.{html,css,js}')
             .pipe($.revReplace({ manifest: manifest }))
             .pipe(dest(DIST))
     }
@@ -298,11 +297,11 @@ export const server = (done) => {
 // Watch for file changes
 //
 export const watchSrc = () => {
-    watch(SRC + '_assets/styles/**/*.scss').on('all', series(css))
+    watch(SRC + '/_assets/styles/**/*.scss').on('all', series(css))
-    watch(SRC + '_assets/javascripts/**/*.js').on('all', series(js, browser.reload))
+    watch(SRC + '/_assets/javascripts/**/*.js').on('all', series(js, browser.reload))
-    watch(SRC + '_assets/images/**/*.{png,jpg,jpeg,gif,webp}').on('all', series(images, browser.reload))
+    watch(SRC + '/_assets/images/**/*.{png,jpg,jpeg,gif,webp}').on('all', series(images, browser.reload))
-    watch(SRC + '_assets/images/**/*.{svg}').on('all', series(svg, browser.reload))
+    watch(SRC + '/_assets/images/**/*.{svg}').on('all', series(svg, browser.reload))
-    watch([SRC + '**/*.{html,xml,json,txt,md,yml}', './*.yml', SRC + '_includes/svg/*']).on('all', series('build', browser.reload))
+    watch([SRC + '/**/*.{html,xml,json,txt,md,yml}', './*.yml', SRC + '/_includes/svg/*']).on('all', series('build', browser.reload))
 }
 
 
@@ -365,91 +364,19 @@ export default dev
 // gulp deploy --beta
 // gulp deploy --gamma
 //
-export const s3 = () => {
+export const s3 = (cb) => {
+    let S3_BUCKET_TARGET
 
-    // create publisher, define config
     if ($.util.env.live === true) {
-        var publisher = $.awspublish.create({
+        S3_BUCKET_TARGET = S3_BUCKET_LIVE
-            params: { 'Bucket': S3BUCKET },
-            'accessKeyId': process.env.AWS_ACCESS_KEY,
-            'secretAccessKey': process.env.AWS_SECRET_KEY,
-            'region': S3REGION
-        })
     } else if ($.util.env.beta === true) {
-        var publisher = $.awspublish.create({
+        S3_BUCKET_TARGET = S3_BUCKET_BETA
-            params: { 'Bucket': S3BUCKET_BETA },
-            'accessKeyId': process.env.AWS_BETA_ACCESS_KEY,
-            'secretAccessKey': process.env.AWS_BETA_SECRET_KEY,
-            'region': S3REGION_BETA
-        })
     } else if ($.util.env.gamma === true) {
-        var publisher = $.awspublish.create({
+        S3_BUCKET_TARGET = S3_BUCKET_GAMMA
-            params: { 'Bucket': S3BUCKET_GAMMA },
-            'accessKeyId': process.env.AWS_GAMMA_ACCESS_KEY,
-            'secretAccessKey': process.env.AWS_GAMMA_SECRET_KEY,
-            'region': S3REGION_GAMMA
-        })
-    } else {
-        return
     }
 
-    return src(DIST + '**/*')
+    cp.exec(`aws s3 sync ${DIST} s3://${S3_BUCKET_TARGET} --exclude "assets/*" ${S3_OPTIONS_DEFAULT}`, (err) => cb(err))
-        .pipe($.awspublishRouter({
+    cp.exec(`aws s3 sync ${DIST} s3://${S3_BUCKET_TARGET} --exclude "*" --include "assets/*" ${S3_OPTIONS_DEFAULT} ${S3_OPTIONS_CACHING}`, (err) => cb(err))
-            cache: {
-                // cache for 5 minutes by default
-                cacheTime: 300
-            },
-            routes: {
-                // all static assets, cached & gzipped
-                '^assets/(?:.+)\\.(?:js|css|png|jpg|jpeg|gif|ico|svg|ttf|eot|woff|woff2)$': {
-                    cacheTime: 2592000, // cache for 1 month
-                    gzip: true
-                },
-
-                // every other asset, cached
-                '^assets/.+$': {
-                    cacheTime: 2592000  // cache for 1 month
-                },
-
-                // all html files, not cached & gzipped
-                '^.+\\.html': {
-                    cacheTime: 0,
-                    gzip: true
-                },
-
-                // all pdf files, not cached
-                '^.+\\.pdf': {
-                    cacheTime: 0
-                },
-
-                // all zip files, not cached
-                '^.+\\.zip': {
-                    cacheTime: 0
-                },
-
-                // font mime types
-                '\.ttf$': {
-                    key: '$&',
-                    headers: { 'Content-Type': 'application/x-font-ttf' }
-                },
-                '\.woff$': {
-                    key: '$&',
-                    headers: { 'Content-Type': 'application/x-font-woff' }
-                },
-                '\.woff2$': {
-                    key: '$&',
-                    headers: { 'Content-Type': 'application/x-font-woff2' }
-                },
-
-                // pass-through for anything that wasn't matched by routes above, to be uploaded with default options
-                "^.+$": "$&"
-            }
-        }))
-        .pipe(parallelize(publisher.publish(), 100))
-        .pipe(publisher.sync()) // delete files in bucket that are not in local folder
-        .pipe($.awspublish.reporter({
-            states: ['create', 'update', 'delete']
-        }))
 }
 
 

package.json

@@ -28,7 +28,7 @@
     "bigchaindb-driver": "^3.2.0",
     "gumshoe": "github:cferdinandi/gumshoe",
     "is-in-viewport": "^3.0.0",
-    "jquery": "^3.2.1",
+    "jquery": "^3.3.1",
     "normalize-css": "^2.3.1",
     "normalize-opentype.css": "^0.2.4",
     "parsleyjs": "^2.8.0",
@@ -40,22 +40,22 @@
     "whatwg-fetch": "^2.0.3"
   },
   "devDependencies": {
-    "acorn": "^5.3.0",
+    "acorn": "^5.4.1",
     "babel-core": "^6.26.0",
     "babel-preset-env": "^1.6.1",
-    "browser-sync": "^2.23.3",
+    "browser-sync": "^2.23.6",
     "concurrent-transform": "^1.0.0",
     "critical": "^1.1.0",
-    "cross-spawn": "^5.1.0",
+    "cross-spawn": "^6.0.4",
     "del": "^3.0.0",
     "gulp": "github:gulpjs/gulp#4.0",
     "gulp-autoprefixer": "^4.1.0",
-    "gulp-awspublish": "^3.3.0",
+    "gulp-awspublish": "^3.3.1",
     "gulp-awspublish-router": "^0.1.3",
     "gulp-clean-css": "^3.9.2",
-    "gulp-cli": "^2.0.0",
+    "gulp-cli": "^2.0.1",
     "gulp-concat": "^2.6.1",
-    "gulp-header": "^1.8.9",
+    "gulp-header": "^2.0.1",
     "gulp-htmlmin": "^4.0.0",
     "gulp-if": "^2.0.2",
     "gulp-imagemin": "^4.1.0",
@@ -66,7 +66,7 @@
     "gulp-rev": "^8.1.1",
     "gulp-rev-replace": "^0.4.3",
     "gulp-sass": "^3.1.0",
-    "gulp-sourcemaps": "^2.6.3",
+    "gulp-sourcemaps": "^2.6.4",
     "gulp-svg-sprite": "^1.3.7",
     "gulp-uglify": "^3.0.0",
     "gulp-util": "^3.0.8",
@@ -76,7 +76,7 @@
     "stylelint": "^8.4.0",
     "stylelint-config-bigchaindb": "^1.2.0",
     "stylelint-config-standard": "^18.0.0",
-    "uglify-es": "^3.3.5"
+    "uglify-es": "^3.3.9"
   },
   "engines": {
     "node": ">=7.0.0"