tabbit

Thu Nov 16 2017
Copied to clipboard! Copy reply
 /**
  * Script to push pages to staging or production server.
  * At least 2 argument is required - an action(up or down) and pagename.
  * always pass the page names last
  * arguments & usage:
  * --action, -a: action to take. 'up' or 'down'. Defaults to up if omitted. I realise 'down' should've been 'delete'.
  *               * push sitewide to stg, all templates
  *               node swag -a up sitewide
  *               node swag sitewide
  *
  * --env, -e: environment to perform action. 'stg' or 'prod'. Defaults to 'stg' if omitted
  *            * push sitewide to prod, all templates
  *            node swag -e prod sitewide
  *
  * --mastheads, -m: pass to attempt to upload mastheads as well.
  *                  * push sitewide to stg, attempts to push its masthead
  *                  node swag -m sitewide
  *
  * --templates, -t: only upload pages for the templates specified. Defaults to all if omitted
  *                  * pushes sale to prod, only for ppc template
  *                  node swag -e prod -t ppc -p sale
  *
  * --pages, -p: which pages to push.
  *              * push sitewide, rmsitewide, bluemonday to stg, for every template
  *              node swag  -p sitewide rmsitewide
  *
  * Note that if you're specifying --templates right before the pages to upload,
  * you need to pass --pages (-p), because if you do something like
  * swag -a up -t sbs sitewide, it's going to think sitewide is a template, since -t
  * accepts multiple params. In the case above, the right way is
  * swag -a up -t sbs -p sitewide
  *
  * You will need to put your public ssh key on the server
  * if for some reason you can't or dont want to do that, use push.sh,
  * which works just like this script, but it's in bash and puts
  * files recursively instead of scp'ish command.
  * to put a put a public key on the server;
  * 1. ssh-keygen -t rsa
  * 2. ssh-copy-id lpstagingadmin@10.130.250.237 (or manually copy your key via ftp. do prod too)
  * 3. do steps above for prod as well.
  * only do step 1 if you don't already have a private key.
  * look in ~/.ssh/id_rsa to find out if you do.
  * Your private key will need to be stored in /home/{user}/.ssh/id_rsa
  *
  * I'm also using some ES6 syntax so you'll need to use harmony flags. Best and easiest way
  * to do this is to add the following alias to your terminal rc file:
  * alias node="node  $(node --v8-options | grep harm | awk '{print $1}' | xargs)"
  *
  * PS: swag down and swag -i aren't implemented yet.
  *
  * A NOTE on pushing mastheads... or passing -m in the command line
  * Not the most reliable thing. We determine which folders to check
  * for images based on folders that exist both in templates/ and img/mastheads/
  * So if both templates/sbs and img/mastheads/sbs exist, img/masthead/sbs/img.jpg
  * will be added to the list to push. This isn't reliable because we may have a template,
  * ie, templates/sbsa that pulls its masthead from img/masthead/sbs ... since we
  * don't have a 1-to-1 mapping on templates/ - mastheads/ dirs, this won't
  * always work.
  *
  */


 'use strict';


var fs = require('fs')
  , _ = require('lodash')
  , Connection = require('ssh2')
  , colors = require('colors')
  , path = require('path')
  , cli = require('command-line-args')
  , blessed = require('blessed')
  ;


/*
 * Accepted arguments.
 * action: 'up' to upload, 'down' to remove
 * env: environment to upload to. stg | prod
 * mastheads: attempt to upload mastheads too? t|f
 * pages: pages to push.
 */
var args = cli([
  // Action: 'up' or 'down' (down == delete)
  {name: 'action', alias: 'a', type: String, defaultValue: 'up'},

  // env: 'stg' or 'prod'
  {name: 'env', alias: 'e', type: String, defaultValue: 'stg', required: true},

  // images: attempt to push mastheads. true|false
  {name: 'mastheads', alias: 'm', type: Boolean},

  // images: attempt to push mastheads. true|false
  {name: 'templates', alias: 't', type: String, multiple: true, defaultValue: []},

  // files: 'colorbii', etc. Pass these as last arguments
  {name: 'pages', alias: 'p', type: String, multiple: true, defaultOption: true}
]).parse()


// Only argument required are the pages to upload.
if (typeof args.pages == 'undefined') throw '** Must specify pages to upload';


// Needs a private key
if (!fs.existsSync(process.env.HOME + '/.ssh/id_rsa')) throw '** Needs private key in ~/.ssh/id_rsa'.red


// todo: make this a module? this is huge.
class Terminal {

  constructor() {
    this.screen = blessed.screen({
      smartCSR: true,
      title: 'swag..?'
    });

    this.mainbox = blessed.box({
      left: 0,
      top: 0,
      width: this.screen.width,
      height: this.screen.height,
    })

    // Top part of the screen
    this.title = blessed.box({
      top: 0,
      left: 'center',
      width: '100%',
      height: 'shrink',
      tags: true,
      content: `Pushing to {bold}${args.env}!{/bold}`,
      style: {
        fg: 'white',
        bg: function() {return args.env == 'stg' ? 'blue' : 'magenta'}()
      }
    })

    // Shows log messages about success/error pushes.
    this.log = blessed.log({
      top: 2,
      height: '100%-5',
      mouse: true,
      tags: true,
      scrollbar: {
        track: {
          bg: 'black',
          fg: 'cyan'
        },
        style: {
          bg: 'red'
        }
      }
    })

    this.deleteForm = blessed.Form({
      keys: true,
      left: 'center',
      top: 'center',
      width: '60%',
      shrink: true,
      bg: 'white',
      fg: 'black',
      content: `Delete from ${args.env}?`,
      padding: {
        left: 2,
        right: 2,
        top: 1,
        bottom: 1
      }
    })

    this.deleteCancel = blessed.button({
      parent: this.deleteForm,
      mouse: true,
      keys: true,
      shrink: true,
      right: 2,
      width: '30%',
      top: 2,
      name: 'cancel',
      content: 'Cancel',
      style: {
        bg: 'yellow',
        fg: 'black'
      },
      padding: {
        left: 1,
        right: 1,
        top: 1,
        bottom: 1
      },

    })

    this.deleteOK = blessed.button({
      parent: this.deleteForm,
      mouse: true,
      keys: true,
      shrink: true,
      left: 2,
      top: 2,
      width: '30%',
      tags: true,
      name: 'ok',
      content: ' Yes',
      style: {
        bg: 'yellow',
        fg: 'black',
        hover: { bg: 'red' },
        focus: { bg: 'green'}
      },
      padding: {
        left: 1,
        right: 1,
        top: 1,
        bottom: 1
      }
    })

    // Progress bar on bottom
    this.progressbar = blessed.ProgressBar({
      bottom: 0,
      left: 0,
      height: 'shrink',
      width: '100%',
      filled: 1,
      content: '',
      border: {
        type: 'line'
      },
      style: {
        bg: 'black',
        bar: {
          bg: 'green',
          fg: 'black'
        },
      }
    })

    // A floating box displaying some status/info to user.
    this.statusBox = blessed.box({
      top: '50%-4',
      left: 'center',
      height: 'shrink',
      padding: 1,
      tags: true,
      content: '{bold}Creating directories...{/bold}',
      style: {
        fg: 'cyan',
        bg: 'black'
      },
      border: {
        type: 'line'
      }
    })


    this.screen.key(['escape', 'q', 'C-c'], function(ch, key) {
      return process.exit(0);
    });

  }


  /**
   * Build layout for when we are uploading files.
   * Typically a title bar on top, with log in the
   * middle, and progress bar on bottom
   */
  uploadLayout() {
    this.screen.append(this.mainbox)
    this.mainbox.append(this.title)
    this.mainbox.append(this.log)
    this.mainbox.append(this.progressbar)
    this.screen.render()
  }

  /**
   * Layout for delete form. Shown when
   * deleting pages
   */
  deleteLayout() {
    this.screen.append(this.mainbox)
    this.mainbox.append(this.log)
    this.mainbox.append(this.deleteForm)
    this.screen.render()
  }


  /**
   * Updates upload progress. So it updates BOTH
   * ProgressBar and Log!!. then rerenders the screen
   * @param alue {number} - number to increment progress bar by
   * @param msg {string} - message to append to log window
   * @innerValue {string} - msg to show inside bar (ie, 7/100)
  */
  updateProgress(innerValue, value, logmsg) {
    this.progressbar.progress(value)
    this.progressbar.setContent(innerValue)
    this.log.add(logmsg)
    this.screen.render()
  }

  /**
   * A floating box in the middle of the screen that displays a msg.
   * the scritp starts creating directories on remote server.\
   * If kill is true, kill the box.
   * @param msg {string}: message to display in the box
   * @param kill {bool}: if true, hide the box
   */
  toggleStatusBox(msg, kill) {
    if (kill) {
      this.statusBox.hide()
    } else {
      this.statusBox.show()
      this.statusBox.setContent(msg)
      this.mainbox.append(this.statusBox)
    }
    this.screen.render()
  }


}


/*
 * Class to hold our settings.
 * For now, only holds connection settings.
 */
class Config {
  constructor(env) {

    this.stg = {
      host: '10.130.250.237',
      user: 'lpstagingadmin',
      deployPath: '/mnt/landingpagestg/autolandingpages/us'
    }

    this.prod = {
      host: '10.130.250.237',
      user: 'lpprodadmin',
      deployPath: '/mnt/landingpageprod/autolandingpages/us'
      /*host: '10.130.250.237',
      user: 'lpstagingadmin',
      deployPath: '/mnt/landingpagestg/autolandingpages/us'*/
    }

    this.env = env;

    // Max number of SSH2 simultaneous connections
    this.max_connections = 9;

    // So we can do config.foo isntead of config[args.env].foo
    this.deployPath = this[env].deployPath

    // SSH2 connection settings
    this.ssh2 = {
      host: this[env].host,
      port: 22,
      username: this[env].user,
      password: this[env].password,
      readyTimeout: 15000,
      keepaliveInterval: 0,
      privateKey: fs.readFileSync(process.env.HOME + '/.ssh/id_rsa'),
    }
  }
}




// Error codes for SSH2 -- not even being used right now
var errors = {
  ENETUNREACH: 'Connection was refused. Check your internet connection.',
  ENOTFOUND: 'Remote address not found.',
  ECONNRESET: 'Connection dropped or reset.'
}


// Instantiate config
var config = new Config(args.env)

// Templates directories
var templates = fs.readdirSync('../templates/')

// Pages array - (sitewide, redbbi, redbdi, etc)
var pages = _.intersection(fs.readdirSync('../deploy/sbsr/'), args.pages);


// If a page in the args isn't found, abort... can't let this pass silently
var fourOhfours = _.difference(args.pages, pages);

if (args.action == 'up' && fourOhfours.length) throw `*** Below pages not found:\n${fourOhfours}`.red


// Let's filter our templates to the ones we passed in the cli
if (args.templates.length) {

  // Quit if a template passed doesn't exist.
  var diff = _.difference(args.templates, templates)
  if (diff.length)
    throw `** Following template(s) not found:\n${diff}`.red

  templates = _.without(args.templates, templates)
}




var toDeploy = {

  // If -i was passed in arguments, we'll store mastheads uris here
  mastheads: [],

  /*
   * Holds all our local paths. We will send every file in here to the
   * remote server. While doing so, we pass each string through the
   * remotify() fn to convert the path from local to remote, like so
   * ../deploy/sbs/sitewide/index.html -> ashcclp/prod/sbs/sitewide/index.html
   */
  files: []
}



/*
 * Upload mastheads
 * 1. Grab all masthead images name from the pages we passed in args.
 * 2. Get all our mastheads dirs from globals/img/mastheads/
 * 3. Create all masthead dirs  in remote globals/img/mastheads/
 * 4. Push img from local to remote.
 * PS: if image exists, it'll be overwritten
 * PSS: Image is assumed to exist in all masteahds/* folders. You'll
 * get a yellow warning in the log otherwise
 * PSSS: Mastheads aren't critical and as noted above, iffy. So we
 * don't count them as part of the  progressbar
 */
if (args.mastheads && args.action == 'up') {
  var json = JSON.parse(fs.readFileSync('../data/pages.json', {encoding: 'utf8'}))

  var mastheads = pages
    .filter(page => !page.match(/js|assets|css/))
    .map(page => json[page].header.mastheadImage)

  // We only wants dirs that are both in templates/ and img/mastheads/
  var mastheadDirs = _.intersection(
    templates,
    fs.readdirSync('../deploy/globals/img/mastheads/')
  );

   // Array with all mastheads uris
  toDeploy.mastheads = mastheads.map((masthead, i) => {
    return mastheadDirs.map(dir => {
      return  `../deploy/globals/img/mastheads/${dir}/${masthead}`
    })
  })[0]

  // Create remote/mastheads/<dir> if needed
  var connection = new Connection()
  connection.connect(config.ssh2)
  connection.on('ready', () =>  createRemoteDirs(connection, toDeploy.mastheads))

  // Using close signal as assumption dirs got created
  connection.on('close', () => {

    // Open a new connection and push images
    var conn = new Connection()
    conn.connect(config.ssh2)

    conn.on('ready', (err, sftp) => {
      conn.sftp((err, sftp) => {
         toDeploy.mastheads.forEach(function(img, i) {
          sftp.fastPut(img, remotify(img), (err) => {
            var logMessage = `{yellow-fg}✔ ${img}{/yellow-bg}`
            if (err) logMessage = `{red-fg}{bold}✕ ${img}{/bold}{/red-bg}\n${err}`
            terminal.updateProgress(0, 0, logMessage)
          })
        })
      })

    })
  })

}




// All combinations of ../deploy/template/page/file.html paths.
// TODO use list comprehension when harmony flag arrives
for (var template of templates) {
  for (var page of pages) {
    let uri = `../deploy/${template}/${page}`
    try {
      fs.readdirSync(uri).forEach(file => toDeploy.files.push(`${uri}/${file}`))
    } catch (e) {
      throw `
      Deploy dirs are not in sync. run rsifast and try again.
      This error means some deploy/<template> has folders
      that some other template(s) dont, when in theory, all
      folders inside deploy should have the same pages.
      ie, if deploy/sbs/foobar exists, but deploy/newlang/foobar does not,
      this error happens
      `.red
    }
  }
}


/*
 * Appends model paths to toDeploy.files array.
 * Models are stored in, ie,  ../rsi/sitewide/models.js,
 * however, we'll store them in the server as
 * globals/models/sitewide.js
 */
var models = _.intersection(fs.readdirSync('../rsi'), args.pages)
models.every(model => toDeploy.files.push(`../deploy/globals/models/${model}.js`))



/**
 * Handles uploading of files (excluding mastheads)
 * to remote server.
 */
function up() {

  /*
   * Creates necessary directories on the remote server,
   * so we can push files to them.
   */
  var connection = new Connection()

  connection.connect(config.ssh2)

  connection.on('ready', (sftp) => createRemoteDirs(connection, toDeploy.files))

  // We'll use 'close' signal as a callback that createRemoteDirs is done.
  connection.on('close', () => deploy())

  /**
   * The main part of this script.
   * Deploy our files. First get each connection ready, then push.
   * Each connection opened will go through the toDeploy.files
   * array, removing and the first file on the stack and pushing it.
   * Each connection will keep doing this until files array exhausts.
   * Savage mode:  [x]on  []off
   */
  function deploy() {

    // Holds SSH2 connection objects. TODO: list comprehension
    let connections = []

    // We need this counter to know when all connections are closed
    let openConnections = 0

    for (let i = 0; i < config.max_connections; i++) {
      connections.push(new Connection())
    }

    terminal.toggleStatusBox(`creating ${config.max_connections} connections..`)

    // the whole purpose of 'i' is to debug
    connections.forEach((connection, i) => {
      connection.on('ready', () => {
        ++openConnections
        connection.sftp((err, sftp) => upload(connection, sftp,i))

        // remove floating message box,
        if (i == config.max_connections-1) terminal.toggleStatusBox(null, true)
      }).connect(config.ssh2)
    })



    // Total number of files to push and total number of files pushed. Used in progressbar
    var totalFiles = toDeploy.files.length
    var totalPushed = 0

    function upload(conn, sftp, i) {

      // Progressbar values
      var barValue = `${totalPushed}/${totalFiles}`
      var barProgress = (100/totalFiles)

      if (toDeploy.files.length) {

        var file = toDeploy.files.shift()

        // fake an error, dev purposes.
        //if (i == 3) file = file+'z'

        sftp.fastPut(file, remotify(file), (err) => {

          // Rename the file for log purposes. Remove '../deploy'
          file = file.slice(10)

          var logMessage;
          if (err) {
            logMessage = `{red-fg}{bold}✕ ${file}{/bold}{/red-bg}\n${err}`
          } else {
            ++totalPushed
            var logMessage = `{green-fg}✔ ${file}{/green-bg}`
          }

          //var barValue = `${totalPushed}/${totalFiles}`
          terminal.updateProgress(barValue, barProgress, logMessage)
          upload(conn, sftp,i)
        })
      } else {

        conn.end()
        --openConnections

        /*
         * Done with all pushing operations. Insert into the log a message to let
         * the user know we're done, and notify if there were errors, ie, some
         * files didn't make it to their destination
         */
        if (!openConnections) {
          var completedMessage = '{bold}-- Completed.{/bold}'

          if (totalPushed !== totalFiles)
            completedMessage += '\n{red-fg}-- Some files didn\'t make it.{/red-fg}'

          terminal.updateProgress(barValue, barProgress, completedMessage)
        }
      }
    }
  }
}




// Instantiate & initiate our blessed terminal
var terminal = new Terminal()

if (args.action == 'up') {
  terminal.uploadLayout()
  up()
}

if (args.action == 'down') {
  terminal.deleteLayout()
  terminal.deleteOK.on('press', () =>  down())
  terminal.deleteCancel.on('press', () => process.exit(0))
  terminal.screen.render()
}



/**
 * This is where we remove things from the remote server.
 * We delete the whole folder, as opposed to just the index
 * file inside.
 */
function down() {
  terminal.deleteForm.destroy()
  terminal.screen.render()


  terminal.updateProgress(0, 0, '{red-fg}{bold}deleting....{/bold}{red-fg}')

  // Yes, _.unique is needed. don't remove it
  // if we have ../deploy/sbs/index.html and ../sbs/foo.json
  // in toDeploy.files, we would end up having ../sbs/ listed here twice.
  var cmd = _.unique(toDeploy.files.map(uri => `'${path.dirname(uri)}'`))

  // Show all uris about to be deleted
  cmd.forEach(uri => terminal.updateProgress(0, 0, `{yellow-fg}{bold}${remotify(uri)}{/bold}{/yellow-bg}`))

  cmd = remotify(`rm -rfv ${cmd.join(' ')}`)



  var connection = new Connection()

  connection.on('ready', function() {
    connection.exec(cmd, function(err, stream) {
      if (err) throw 'ERROR! could not execute delete command'.red

      stream.on('data', function(data) {
        terminal.updateProgress(0, 0, data)
      })

      stream.on('close', function(code, signal) {
        terminal.updateProgress(0, 0, '-- done --')
      })

      stream.stderr.on('data', function(data) {
         terminal.updateProgress(0, 0, `{red-fg}data{/red-fg}`)
      })
    })
  }).connect(config.ssh2)
}



/**
 * Before we start pushing all files, we have to create the directories
 * trees in the remote server first. So open a new connection,
 * and do that.
 * @param connection {connection object}: a SSH2 connection object
 * @param dirsArray {array}: array of local paths to check & create
 * TODO possible bug - if the paths in array are actual directories,
 * then dirname() may go back one dir
 */
function createRemoteDirs(connection, dirsArray) {
  terminal.toggleStatusBox('Creating dirs..')

  // For each of our files in dirsArray, get the dirname().
  // so ../deploy/sbs/index.html becomes ../deploy/sbs
  let dirs =  dirsArray.map(uri => `'${path.dirname(uri)}'`)

  // Make sure globals/models/ exists too
  dirs.push('../deploy/globals/models/')

  let cmd = remotify(`mkdir -p ${dirs.join(' ')}`)

  connection.exec(cmd, (e, stream) => {
    if (e) throw errors[e].red

    terminal.toggleStatusBox(null,true)
    connection.end()
  })

}



/**
 * Converts local uris into remote ones. basically replaces
 * ../deploy/ with server's deploy path
 */
function remotify(uri) {
  return uri.replace(/\.\.\/deploy/gi, config.deployPath)
}



//todo sounds & crescendos