scripts/ssh_tools/multicopy.sh

#!/bin/sh
#
# multicopy.sh, Copyright © 2013, 2019 Matteo Cypriani <mcy@lm7.fr>
# (Formerly named cluster-deploy.sh)
#
# This program is free software. It comes without any warranty, to
# the extent permitted by applicable law. You can redistribute it
# and/or modify it under the terms of the Do What The Fuck You Want
# To Public License, Version 2, as published by Sam Hocevar. See
# http://sam.zoy.org/wtfpl/COPYING for more details.
#
# Deploy files on a number of remote hosts using Parallel SCP.

#set -x
set -u

# Exit codes
readonly EXIT_SUCCESS=0
readonly EXIT_USAGE=127
readonly EXIT_DEPENDENCY=4
readonly EXIT_HOSTS_LIST=6
readonly EXIT_HOSTS_DEAD=7
readonly EXIT_CONNECTION=8

print_usage()
{
  cat <<EOF
Usage:
  $0 -h
  $0 [-P] [-r|-R] [-l <login>] <hosts_list> <file1> [file2 [...]]

Options:
  -h  Print this help message.
  -l <login>
      Login remotely as <login>.
  -P  Transfer files in parallel instead of one by one whenever possible
      (ignored when -r or -R is used).
  -r  Use prsync instead of pscp.
  -R  Use prsync's --delete option (delete remote files that are not in
      the local copy).

<hosts_list> is the list of remote hosts on which the file(s) will be copied.
EOF
}

bad_usage()
{
  err "$@"
  echo >&2
  print_usage >&2
  exit $EXIT_USAGE
}

err()
{
  printf 'Error! ' >&2
  printf '%s\n' "$@" >&2
}

warn()
{
  printf 'Warning! ' >&2
  printf '%s\n' "$@" >&2
}

# Copy file $FILE to the remote hosts using plain old SCP
scp_copy()
{
  grep -v '^#' "$HOSTS" | while read -r host ; do
    [ -z "$host" ] && continue
    echo "Deploying '$FILE' to '$host'..."
    scp -r "$FILE" "${SSH_LOGIN}${host}:${DEST_DIR}"
  done
}

# Parse CLI options
LOGIN=
PARALLEL=
RSYNC=
DELETE=
while getopts hl:PrR option ; do
  case $option in
    h)  print_usage
        exit $EXIT_SUCCESS
        ;;
    l)  LOGIN="$OPTARG" ;;
    P)  PARALLEL=1 ;;
    r)  RSYNC=1 ;;
    R)  DELETE=1 ;;
    \?) bad_usage "Bad option."
  esac
done
shift $((OPTIND - 1))

# Do we still have at least a host list and a file?
[ $# -lt 2 ] && bad_usage "Wrong number of arguments."

# Check the usage of -r and -R
if [ "$DELETE" = 1 ] ; then
  [ "$RSYNC" = 1 ] && bad_usage "Use either -r or -R, but not both."
  RSYNC=1
  DELETE="-X --delete"
fi

# Ignore -P if rsync is going to be used
if [ "$RSYNC" = 1 ] && [ "$PARALLEL" = 1 ] ; then
  warn "Cannot transfer in parallel when using rsync: ignoring -P."
  PARALLEL=
fi

# Check dependencies
if [ "$RSYNC" = 1 ] ; then
  PRSYNC=$(command -v parallel-rsync || command -v prsync)
  if [ -z "$PRSYNC" ] ; then
    err "Parallel rsync (prsync) is required for this script to work with -r or -R."
    exit $EXIT_DEPENDENCY
  fi

else # $RSYNC != 1
  PSCP=$(command -v parallel-scp || command -v pscp.pssh || command -v pscp)
  if [ -z "$PSCP" ] ; then
    warn "Parallel SSH (pssh) is not installed. Install it for best results."
    if [ "$PARALLEL" = 1 ] ; then
      warn "Parallel copy will not work without Parallel SSH. Ignoring -P."
      PARALLEL=
    fi
  fi
fi

MULTIPING=$(command -v multiping)
if [ -z "$MULTIPING" ] ; then
  warn "multiping (which should have been provided along with this" \
       "script) is not available. For best results, make sure it is" \
       "reachable from your PATH."
fi

# Hosts list file
HOSTS_LIST_NAME="$1"
shift
[ -z ${XDG_CONFIG_HOME+x} ] && XDG_CONFIG_HOME="$HOME/.config"
HOSTS="$XDG_CONFIG_HOME/ssh_tools/${HOSTS_LIST_NAME}.lst"
echo "Using file '$HOSTS' as hosts list."
if [ ! -f "$HOSTS" ] ; then
  err "The hosts list file doesn't exist or is not a regular file."
  exit $EXIT_HOSTS_LIST
fi

# Login
SSH_LOGIN=
if [ -n "$LOGIN" ] ; then
  echo "Login: $LOGIN"
  SSH_LOGIN="${LOGIN}@"
  LOGIN="-l $LOGIN"
fi

# Test the connection to the first alive host
if [ -n "$MULTIPING" ] ; then
  FIRST_HOST=$($MULTIPING "$HOSTS_LIST_NAME" 2>/dev/null \
             | sed -n "s/ is alive$//p" | head -n1)
  if [ -z "$FIRST_HOST" ] ; then
    err "None of the remote hosts is alive."
    exit $EXIT_HOSTS_DEAD
  fi
else # multiping is not available
  warn "multiping is not available, using the first host in the hosts list."
  FIRST_HOST="$(grep -v '^#' "$HOSTS" | head -n 1)"
fi

# Get the destination directory (home directory of the remote user)
echo "Testing connection to $FIRST_HOST..."
DEST_DIR="$(ssh "${SSH_LOGIN}${FIRST_HOST}" 'echo $HOME' 2>/dev/null)"
if [ -z "$DEST_DIR" ] ; then
  err "Cannot connect to the first alive host. Aborting."
  exit $EXIT_CONNECTION
fi
echo "Destination directory: $DEST_DIR"

# Transfer the files in parallel...
if [ "$PARALLEL" = 1 ] ; then
  # shellcheck disable=SC2086
  exec $PSCP -r $LOGIN -h "$HOSTS" -- "$@" "$DEST_DIR"
fi

# ... or one by one
for FILE in "$@" ; do
  echo "Deploying '$FILE'..."
  if [ ! -e "$FILE" ] ; then
    warn "This file doesn't exist, skipping."
    continue
  fi
  if [ "$RSYNC" = 1 ] ; then
    # shellcheck disable=SC2086
    $PRSYNC -a $DELETE $LOGIN -h "$HOSTS" -- "$FILE" "$DEST_DIR"
    continue
  fi
  if [ -n "$PSCP" ] ; then
    # shellcheck disable=SC2086
    $PSCP -r $LOGIN -h "$HOSTS" -- "$FILE" "$DEST_DIR"
    continue
  fi
  # If we're still here, that means we'll have to copy with plain SCP
  scp_copy
done