scripts/ssh_tools/multicopy.sh

#!/bin/sh
#
# multicopy.sh, Copyright © 2013, 2019 Matteo Cypriani <mcy@lm7.fr>
# (Formerly named cluster-deploy.sh)
#
# This program is free software. It comes without any warranty, to
# the extent permitted by applicable law. You can redistribute it
# and/or modify it under the terms of the Do What The Fuck You Want
# To Public License, Version 2, as published by Sam Hocevar. See
# http://sam.zoy.org/wtfpl/COPYING for more details.
#
# Deploy files on a number of remote hosts using Parallel SCP.

#set -x
set -u

# Exit codes
readonly EXIT_SUCCESS=0
readonly EXIT_USAGE=127
readonly EXIT_DEPENDENCY=4
readonly EXIT_HOSTS_LIST=6
readonly EXIT_HOSTS_DEAD=7
readonly EXIT_CONNECTION=8

print_usage()
{
  cat <<EOF
Usage:
  $0 [-P] [-r|-R] [-l login] <hosts_list> <file1> [file2 [...]]

Options:
  -h  Print this help message.
  -P  Transfer files in parallel instead of one by one whenever possible
      (ignored when -r or -R is used).
  -r  Use prsync instead of pscp.
  -R  Use prsync's --delete option (delete remote files that are not in
      the local copy).
EOF
}

bad_usage()
{
  err "$@"
  echo >&2
  print_usage >&2
  exit $EXIT_USAGE
}

err()
{
  printf 'Error! %s\n' "$@" >&2
}

warn()
{
  printf 'Warning! %s\n' "$@" >&2
}

# Parse CLI options
LOGIN=
PARALLEL=
RSYNC=
DELETE=
while getopts hl:PrR option ; do
  case $option in
    h)  print_usage
        exit $EXIT_SUCCESS
        ;;
    l)  LOGIN="$OPTARG" ;;
    P)  PARALLEL=1 ;;
    r)  RSYNC=1 ;;
    R)  DELETE=1 ;;
    \?) bad_usage "Bad option."
  esac
done
shift $((OPTIND - 1))

# Do we still have at least a host list and a file?
if [ $# -lt 2 ] ; then
  bad_usage "Wrong number of arguments."
fi

# Check the usage of -r and -R
if [ "$DELETE" = "1" ] ; then
  if [ "$RSYNC" = "1" ] ; then
    bad_usage "Use either -r or -R, but not both."
  fi
  RSYNC=1
  DELETE="-X --delete"
fi

# Ignore -P if rsync is going to be used
if [ "$RSYNC" = "1" ] && [ "$PARALLEL" = "1" ] ; then
  warn "Cannot transfer in parallel when using rsync: ignoring -P."
  PARALLEL=
fi

# Check dependencies
PSCP=$(command -v parallel-scp || command -v pscp.pssh || command -v pscp)
PRSYNC=$(command -v parallel-rsync || command -v prsync)
MULTIPING=$(command -v multiping)
if [ -z "$RSYNC" ] && [ -z "$PSCP" ] ; then
  err "Parallel SSH (pssh) is required for this script to work."
  exit $EXIT_DEPENDENCY
fi
if [ -z "$PRSYNC" ] ; then
  err "Parallel rsync (prsync) is required for this script to work."
  exit $EXIT_DEPENDENCY
fi
if [ -z "$MULTIPING" ] ; then
  err "multiping (which should have been provided along with this" \
      "script) is required for this script to work."
  exit $EXIT_DEPENDENCY
fi

# Hosts' list file
HOSTS_LIST_NAME="$1"
if [ -z ${XDG_CONFIG_HOME+x} ] ; then
  XDG_CONFIG_HOME="$HOME/.config"
fi
HOSTS="$XDG_CONFIG_HOME/ssh_tools/${HOSTS_LIST_NAME}.lst"
shift
echo "Using file '$HOSTS' as hosts' list."
if [ ! -f "$HOSTS" ] ; then
  err "The hosts' list file doesn't exist or is not a regular file."
  exit $EXIT_HOSTS_LIST
fi

# Login
SSH_LOGIN=
if [ -n "$LOGIN" ] ; then
  echo "Login: $LOGIN"
  SSH_LOGIN="${LOGIN}@"
  LOGIN="-l $LOGIN"
fi

# Test the connection to the first alive host and get the destination
# directory (home directory of the remote user)
FIRST_HOST=$($MULTIPING "$HOSTS_LIST_NAME" 2>/dev/null \
           | sed -n "s/ is alive$//p" | head -n1)
if [ -z "$FIRST_HOST" ] ; then
  err "None of the remote hosts is alive."
  exit $EXIT_HOSTS_DEAD
fi
echo "Testing connection to $FIRST_HOST..."
DEST_DIR="$(ssh "${SSH_LOGIN}${FIRST_HOST}" 'echo $HOME' 2>/dev/null)"
if [ -z "$DEST_DIR" ] ; then
  err "Cannot connect to the first alive host. Aborting."
  exit $EXIT_CONNECTION
fi
echo "Destination directory: $DEST_DIR"

# Transfer the files in parallel...
if [ "$PARALLEL" = "1" ] ; then
  # shellcheck disable=SC2086
  exec $PSCP -r $LOGIN -h "$HOSTS" -- "$@" "$DEST_DIR"
fi

# ... or one by one
for FILE in "$@" ; do
  echo "Deploying '$FILE'..."
  if [ ! -e "$FILE" ] ; then
    warn "This file doesn't exist, skipping."
    continue
  fi
  if [ "$RSYNC" = "1" ] ; then
    # shellcheck disable=SC2086
    $PRSYNC -a $DELETE $LOGIN -h "$HOSTS" -- "$FILE" "$DEST_DIR"
  else
    # shellcheck disable=SC2086
    $PSCP -r $LOGIN -h "$HOSTS" -- "$FILE" "$DEST_DIR"
  fi
done