Created the zfs-replicate-wrapper script that factors in duration to

stop replicating once duration exceeds a certain amount. Updated
zfs-log-parser to parse the stats from the replicate jobs properly.

diff --git a/zfs-log-parser b/zfs-log-parser
index 4ecca3c37e59db5a4aaeecfa9dea6277b4dda928..94e4490b500a581b623dc6daa681eabb4e1735ff 100755 (executable)
--- a/zfs-log-parser
+++ b/zfs-log-parser
@@ -52,14 +52,14 @@ sub parse_replicate_logfile {
        if(($in_replicate == 0) && ("$startdate" ne "") && ($line !~ /$startdate/)) { next; }
        if($line =~ /(\S+)\s+(\S+)\s+(\d+)\s+(\d+):(\d+):(\d+)\s+(\S+)\s+(\S+)/) {
            $dayname=$1; $month=$2; $daynum=$3; $hour=$4; $minute=$5; $sec=$6; $year=$8;
-           if(($in_replicate == 0) && ($line =~ /lock obtained/)) {
+           if(($in_replicate == 0) && ($line =~ /replicate started/)) {
                $in_replicate = 1;
                $date="$dayname $month $daynum $hour:$minute:$sec $year";
                $totals{$date}{data} = 0;
                $totals{$date}{transfertime} = 0;
                $totals{$date}{duration} = time_to_seconds($hour,$minute,$sec);
            }
-           elsif(($in_replicate == 1) && ($line=~ /lock released/)) {
+           elsif(($in_replicate == 1) && ($line=~ /replicate complete/)) {
                $in_replicate = 0;
                $totals{$date}{duration} = time_to_seconds($hour,$minute,$sec) - $totals{$date}{duration};
            }

diff --git a/zfs-replicate b/zfs-replicate
index dc6dab215c592241685f6f90eb017ab4e6f85ded..09cb3babb7600f0526a14b4b2c2f945f41660ea4 100755 (executable)
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -10,7 +10,7 @@ remote_lockdir="/tmp/zfs-admin-lock"
 local_pool=backups
 
 # Set the email address to send notification to
-mailto=alan@pippins.net
+mailto=root@pippins.net
 
 # When this variable is set, local filesystems will be destroyed 
 # before receiving a full streams into them from the remote source.
@@ -19,6 +19,14 @@ destroy_local_filesystem_on_full_replicate=0
 # The ssh connection doesn't find zfs without this.
 zfs=/usr/sbin/zfs
 
+# Setup our cleanup and exit trap
+cleanup() {
+  ssh $remote rm -rf "$remote_lockdir"
+  exit
+}
+trap cleanup INT
+trap cleanup EXIT
+
 # Make sure we have valid arguments
 if [[ -z "$remote" ]] || [[ -z "$remote_fs" ]]; then
   echo "Usage: $0 <hostname> <zfs filesystem>"
@@ -78,10 +86,6 @@ while true; do
   fi
 done
 
-# Declare a cleanup() method to remove the remote lockdir
-cleanup() { ssh $remote rm -rf "$remote_lockdir"; }
-trap cleanup EXIT
-
 # Setup our backup marker names
 current_backup_marker=${remote_fs}@current-backup-${local_pool_guid}
 previous_backup_marker=${remote_fs}@previous-backup-${local_pool_guid}

diff --git a/zfs-replicate-wrapper b/zfs-replicate-wrapper
new file mode 100755 (executable)
index 0000000..d699c79
--- /dev/null
+++ b/zfs-replicate-wrapper
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+# Author: Alan J. Pippin
+# Description: This script calls zfs-replicate for each filesystem needing
+#              to be backed up, or replicated, to another ZFS pool.
+
+# Setup some default values
+zfsreplicate="/etc/bin/zfs-replicate"
+logfile_parser="/etc/bin/zfs-log-parser"
+logfile="/var/log/zfs/zfs-replicate.log"
+remote="tank.pippins.net"
+local_pool=backups
+mailto=root@pippins.net
+date=`date`
+starttime=`date +%s`
+zfs=/usr/sbin/zfs
+
+# Specify the list of filesystems to replicate
+filesystems_to_replicate='
+naspool/www
+naspool/git
+'
+
+# Specify the maximum run time in minutes that this script can run
+maxruntime=240
+
+# This function checks to see if our runtime has exceeded our stoptime
+timeexceeded() { 
+  currenttime=`date +%s`  
+  elapsedtime=$(($currenttime - $starttime))
+  stoptime=$(($maxruntime*60))
+  if [[ $elapsedtime -gt $stoptime ]]; then
+    #echo "$elapsedtime > $stoptime"
+    return 1
+  fi
+  #echo "$elapsedtime < $stoptime"
+  return 0
+}
+
+# This function cleanup and exit trap
+cleanup_and_exit() { 
+  #echo "cleanup and exit"
+  rm -rf "$lockdir"
+  exit 0
+}
+trap cleanup_and_exit INT
+
+# This function executes the replicate command and checks the stoptime
+replicate() { 
+  $zfsreplicate $*  >> $logfile 2>&1
+  timeexceeded
+  if [ $? == 1 ]; then
+    cleanup_and_exit
+  fi
+}
+
+# This function obtains the date a given snapshot was created in epoch seconds
+snapshot_age() {
+  snapshot=${local_pool}/${1}${previous_backup_marker}
+  $zfs list -t snapshot ${snapshot} > /dev/null 2>&1
+  if [ $? == 0 ]; then
+    $zfs get creation ${snapshot} > /dev/null 2>&1
+    if [ $? == 0 ]; then
+      snap_creation=`$zfs get creation ${snapshot} | grep $1 | awk '{ print $3" "$4" "$5" "$6" "$7 }'`
+      snap_age=`date -d "$snap_creation" +%s` 
+      echo "$snap_age"
+    else
+      echo "0"
+    fi
+  else
+    echo "0"
+  fi
+}
+
+# Obtain the zpool guid for the local pool
+local_pool_guid=`zpool get guid $local_pool 2>&1 | grep $local_pool | awk '{ print $3 }'`
+zpool get guid $local_pool > /dev/null 2>&1
+if [ $? != 0 ]; then
+  echo >&2 "-E- Unable to extract the guid for the local pool: $local_pool"
+  exit 1
+fi
+
+# Setup our backup marker names
+current_backup_marker=@current-backup-${local_pool_guid}
+previous_backup_marker=@previous-backup-${local_pool_guid}
+
+# Auto snapshot every zfs filesystem on the system specified below
+echo "$date ZFS replicate started" >> $logfile
+
+# Sort the filesystems to replicate by the oldest backup first
+tmpfile=`tempfile`
+for filesystem in $filesystems_to_replicate; do
+  age=`snapshot_age $filesystem`
+  echo $filesystem $age >> $tmpfile
+done
+sorted_filesystems=`cat $tmpfile | sort -n -k 2 | awk '{ print $1 }'`
+rm -f $tmpfile
+
+# Replicate the sorted filesystems
+for filesystem in $sorted_filesystems; do
+  echo "-> Replicating $remote:$filesystem to ${local_pool}/${filesystem}"
+  replicate $remote $filesystem
+done
+
+# All done
+echo `date` ZFS replicate complete >> $logfile
+
+# Parse the log file and extract our backup stats
+$logfile_parser "$logfile" "$date" >> $logfile
+