#!/bin/bash

# Author: Alan J. Pippin
# Description: This script calls zfs-replicate for each filesystem needing
#              to be backed up, or replicated, to another ZFS pool.

# source our configuration 
config="${0%/*}/zfs-scripts.conf"
[ -e "${config}.dist" ] && . ${config}.dist
[ -e "${config}" ] && . ${config}

# Setup some default values
logfile="$logdir/zfs-replicate.log"
mylogfile="$logdir/zfs-replicate-all.log"
date=`date`
starttime=`date +%s`
tmpfile=`tempfile`

# Make sure we aren't already running
SCRIPT_NAME=${0##*/}
PROCESS_LIST=`tempfile`
ps -ef | grep -e "$SCRIPT_NAME" | grep -v grep | grep -v $$ | grep -v $PPID >> $PROCESS_LIST
if [[ $? == 0 ]]; then
   echo "$date Another $SCRIPT_NAME process is already running" >> $mylogfile
   cat $PROCESS_LIST >> $mylogfile
   exit 1
fi
[[ -e "$PROCESS_LIST" ]] && rm -f $PROCESS_LIST

# This function checks to see if our runtime has exceeded our stoptime
timeexceeded() { 
  if [[ $maxruntime == 0 ]]; then
    return 0
  fi
  currenttime=`date +%s`  
  elapsedtime=$(($currenttime - $starttime))
  stoptime=$(($maxruntime*60))
  if [[ $elapsedtime -gt $stoptime ]]; then
    #echo "$elapsedtime > $stoptime"
    return 1
  fi
  #echo "$elapsedtime < $stoptime"
  return 0
}

# This function cleanup and exit trap
cleanup_and_exit() { 
  #echo "cleanup and exit"
  rm -rf "$lockdir"
  exit 0
}
trap cleanup_and_exit INT

fatal_and_exit() {
  # echo message to terminal
  echo -e 2>&1 "$1"
  # send email notification
  echo -e "$1" | $mailx -s "zfs-replicate-all on $hostname failed" "$2"
  # exit with bad return code unless 3rd argument was defined that says not to
  if [[ -z "$3" ]]; then
    exit 1
  fi
}

# This function executes the replicate command and checks the stoptime
replicate() { 
  zfs-replicate $*  >> $logfile 2>&1
  timeexceeded
  if [ $? == 1 ]; then
    cleanup_and_exit
  fi
}

# This function obtains the date a given snapshot was created in epoch seconds
snapshot_age() {
  snapshot=${backup_pool}/${1}${previous_backup_marker}
  $zfs list -t snapshot ${snapshot} > /dev/null 2>&1
  if [ $? == 0 ]; then
    $zfs get creation ${snapshot} > /dev/null 2>&1
    if [ $? == 0 ]; then
      snap_creation=`$zfs get creation ${snapshot} | grep $1 | awk '{ print $3" "$4" "$5" "$6" "$7 }'`
      snap_age=`date -d "$snap_creation" +%s` 
      echo "$snap_age"
    else
      echo "0"
    fi
  else
    echo "0"
  fi
}

# Replicate every zfs filesystem specified in the config file
echo "$date ZFS replicate started" >> $logfile
echo "$date ZFS replicate started" | tee -a $mylogfile

# Loop over each backup pool
backup_pools=$backup_pool
for backup_pool in $backup_pools; do

# Import the local backup pool if needed and the option is given to do so, else error out
zpool list -H $backup_pool > $tmpfile 2>&1
if [ $? != 0 ]; then
  if [[ $import_export_backup_pool == 1 ]]; then
    zpool import $backup_pool >> $tmpfile 2>&1
    if [ $? != 0 ]; then
      msgs=`cat $tmpfile`
      fatal_and_exit "-E- unable to import the backup pool '$backup_pool' on $hostname: $msgs" "$mailto"
    fi
  else 
    msgs=`cat $tmpfile`
    fatal_and_exit "-E- The local backup pool on $hostname, '$backup_pool' doesn't seem to exist: $msgs" "$mailto"
  fi
fi
rm -f $tmpfile

# Obtain the zpool guid for the local backup pool
backup_pool_guid=`zpool get guid $backup_pool 2>&1 | grep $backup_pool | awk '{ print $3 }'`
zpool get guid $backup_pool > /dev/null 2>&1
if [ $? != 0 ]; then
  fatal_and_exit "-E- Unable to extract the guid for the local backup pool on $hostname: $backup_pool" "$mailto"
fi

# Setup our backup marker names
current_backup_marker=@current-backup-${backup_pool_guid}
previous_backup_marker=@previous-backup-${backup_pool_guid}

# Sort the filesystems to replicate by the oldest backup first
for filesystem in $filesystems_to_replicate; do
  if [[ $filesystem =~ ':' ]]; then
    dst_pool=${filesystem%%:*}
    filesystem=${filesystem#*:} # remove src_pool from string
  else
    dst_pool=$backup_pool
  fi
  # Only backup filesystems that are specified to go this backup_pool
  if [[ $backup_pool == $dst_pool ]]; then
    age=`snapshot_age $filesystem`
    echo $filesystem $age >> $tmpfile
  fi
done
sorted_filesystems=`cat $tmpfile | sort -n -k 2 | awk '{ print $1 }'`
rm -f $tmpfile

# Replicate the sorted filesystems
for filesystem in $sorted_filesystems; do
  echo "-> Replicating $remote:$filesystem to ${backup_pool}/${filesystem}" | tee -a $mylogfile
  replicate $remote $filesystem $backup_pool
done

# Export the local pool if told to do so
if [[ $import_export_backup_pool == 1 ]]; then
  # Don't export the pool if there is a currently running zfs-scrub operation
  ps -ef | grep "zfs-scrub" | grep -q "${backup_pool}" | grep -v grep
  if [ $? != 0 ]; then
    zpool export $backup_pool >> $tmpfile 2>&1
    if [ $? != 0 ]; then
      sleep 300
      zpool export $backup_pool >> $tmpfile 2>&1
      if [ $? != 0 ]; then
        lsof /$backup_pool/* >> $tmpfile 2>&1
	msgs=`cat $tmpfile`
        fatal_and_exit "-E- unable to export the local backup pool $backup_pool on $hostname: $msgs" "$mailto" 0
      fi
    fi
  fi
fi
rm -f $tmpfile

done

# All done
echo `date` ZFS replicate complete >> $logfile
echo `date` ZFS replicate complete | tee -a $mylogfile

# Parse the log file and extract our backup stats
zfs-log-parser "$logfile" "$date" >> $logfile
zfs-log-parser "$logfile" "$date" | tee -a $mylogfile