From f1a0eaf6cb15faf514577513f17fa9a06c34d129 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Wed, 13 Feb 2019 22:51:45 -0700
Subject: [PATCH 01/16] Added --compressed flag to zfs send cmds

Change-Id: I010c927fa250f739ccd6c62c0882be2066fd9562
---
 zfs-replicate | 6 +++---
 zfs-restore   | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/zfs-replicate b/zfs-replicate
index 9f72b06..b3f7cff 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -201,7 +201,7 @@ if [ $no_markers == 0 ]; then
     fatal_and_exit "-E- remote incremental $zfs rollback command failed on $hostname" $mailto
   fi
   # Now it should be safe to send the snaps
-  replicate "$zfs send -R -I${previous_backup_marker} ${current_backup_marker}"
+  replicate "$zfs send -Rc -I${previous_backup_marker} ${current_backup_marker}"
   if [ $? != 0 ]; then
     fatal_and_exit "-E- remote incremental $zfs send command failed on $hostname" $mailto
   fi
@@ -220,7 +220,7 @@ else
 
   if [[ -n "$common" ]]; then
     # We found a common snapshot, incrementally send the new snaps
-    replicate "$zfs send -R -I${common/*@/@} ${current_backup_marker}"
+    replicate "$zfs send -Rc -I${common/*@/@} ${current_backup_marker}"
     if [ $? != 0 ]; then
       fatal_and_exit "-E- remote incremental $zfs send command failed on $hostname" $mailto
     fi
@@ -242,7 +242,7 @@ else
       fi
     fi
     # Send the full filesystem
-    replicate "$zfs send -R ${current_backup_marker}"
+    replicate "$zfs send -Rc ${current_backup_marker}"
     if [ $? != 0 ]; then
       fatal_and_exit "-E- remote full $zfs send command failed on $hostname" $mailto
     fi
diff --git a/zfs-restore b/zfs-restore
index fab2b6e..4014ff7 100755
--- a/zfs-restore
+++ b/zfs-restore
@@ -80,12 +80,12 @@ fi
 
 # Now send the src filesystem
 if [[ -n "$SCRIPT_UNDER_TEST" ]]; then
-  echo "$zfs send -R $src_pool/$last_backup_marker | $ssh $dst_hostname $zfs recv -dv $dst_pool"
+  echo "$zfs send -Rc $src_pool/$last_backup_marker | $ssh $dst_hostname $zfs recv -dv $dst_pool"
 else 
    if [[ $throttle_enable == 1 && -e $throttle ]]; then
-     $zfs send -R "$src_pool/$last_backup_marker" | $throttle $throttle_opt | $ssh $dst_hostname $zfs recv -dv $dst_pool
+     $zfs send -Rc "$src_pool/$last_backup_marker" | $throttle $throttle_opt | $ssh $dst_hostname $zfs recv -dv $dst_pool
    else 
-     $zfs send -R "$src_pool/$last_backup_marker" | $ssh $dst_hostname $zfs recv -dv $dst_pool
+     $zfs send -Rc "$src_pool/$last_backup_marker" | $ssh $dst_hostname $zfs recv -dv $dst_pool
    fi 
 fi
 
-- 
2.34.1


From 17efdff10293f272ca326b4997f8352816872d53 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Thu, 14 Feb 2019 01:29:24 -0700
Subject: [PATCH 02/16] Fixed syntax error in latest fix

Change-Id: I0fd708aca9563982c3106ba7838364df5ffd8b4c
---
 zfs-replicate-all | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zfs-replicate-all b/zfs-replicate-all
index 6fc9bfc..456e835 100755
--- a/zfs-replicate-all
+++ b/zfs-replicate-all
@@ -140,7 +140,7 @@ if [[ $import_export_backup_pool == 1 ]]; then
     zpool export $backup_pool
     if [ $? != 0 ]; then
       lsof /$backup_pool/*
-      fatal("-E- unable to export the local backup pool $backup_pool on $hostname")
+      fatal "-E- unable to export the local backup pool $backup_pool on $hostname" "$mailto"
     fi
   fi
 fi
-- 
2.34.1


From 621be18ea5c913faa24d1e099120eddfb49c212b Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Fri, 15 Feb 2019 19:18:48 -0700
Subject: [PATCH 03/16] Updated replicate size to work with K M or G

Change-Id: I4e83ae8665f24561bd34c1b8085277826c402e3e
---
 zfs-log-parser | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/zfs-log-parser b/zfs-log-parser
index f13c2a0..560cba2 100755
--- a/zfs-log-parser
+++ b/zfs-log-parser
@@ -66,9 +66,9 @@ sub parse_replicate_logfile {
 	}   
 	if(($in_replicate == 1) && ($line =~ /received ([\d\.]+)(\w+)/)) {
 	    $data = $1; $size = $2;
-	    if($size =~ /Kb/i) { $data = $data * $kilo; }
-	    if($size =~ /Mb/i) { $data = $data * $mega; }
-	    if($size =~ /Gb/i) { $data = $data * $giga; }
+	    if($size =~ /(Kb|K)/i) { $data = $data * $kilo; }
+	    if($size =~ /(Mb|M)/i) { $data = $data * $mega; }
+	    if($size =~ /(Gb|G)/i) { $data = $data * $giga; }
 	    chomp($line);
 	    $totals{$date}{data} += $data;
 	}
-- 
2.34.1


From 7efa74b7c3ad633378498822fe408b3d2da2dcca Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Mon, 3 Feb 2020 13:32:38 -0700
Subject: [PATCH 04/16] Added support to replicate and restore from multiple
 backup pools

Change-Id: Ia3d0207cc0b5f551b5a464b73411c310c4fe3e0a
---
 .gitignore            |  2 ++
 zfs-replicate         |  7 +++++++
 zfs-replicate-all     | 29 ++++++++++++++++++++++-------
 zfs-restore-all       |  8 +++++++-
 zfs-scripts.conf.dist |  3 ++-
 zfs-scrub             |  6 +++---
 6 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index 60fa576..d4d52a6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
 zfs-scripts.conf
+*.swp
+*~
diff --git a/zfs-replicate b/zfs-replicate
index b3f7cff..9dfd9f8 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -18,6 +18,13 @@ remote=$1
 remote_fs=$2
 remote_pool=${2%%/*}
 
+# get the backup pool from the command line or the config file if not specified
+if [[ -n $3 ]]; then
+    backup_pool=$3
+else
+    backup_pool=${backup_pool%% *} # use the first backup pool if none specified
+fi
+
 # Setup our cleanup and exit trap
 cleanup() {
   if [[ -e "$local_list" ]]; then
diff --git a/zfs-replicate-all b/zfs-replicate-all
index 456e835..66949df 100755
--- a/zfs-replicate-all
+++ b/zfs-replicate-all
@@ -89,6 +89,14 @@ snapshot_age() {
   fi
 }
 
+# Replicate every zfs filesystem specified in the config file
+echo "$date ZFS replicate started" >> $logfile
+echo "$date ZFS replicate started" | tee -a $mylogfile
+
+# Loop over each backup pool
+backup_pools=$backup_pool
+for backup_pool in $backup_pools; do
+
 # Import the local backup pool if needed and the option is given to do so, else error out
 zpool list -H "$backup_pool" >/dev/null 2>&1
 if [ $? != 0 ]; then
@@ -113,15 +121,20 @@ fi
 current_backup_marker=@current-backup-${backup_pool_guid}
 previous_backup_marker=@previous-backup-${backup_pool_guid}
 
-# Auto snapshot every zfs filesystem on the system specified below
-echo "$date ZFS replicate started" >> $logfile
-echo "$date ZFS replicate started" | tee -a $mylogfile
-
 # Sort the filesystems to replicate by the oldest backup first
 tmpfile=`tempfile`
 for filesystem in $filesystems_to_replicate; do
-  age=`snapshot_age $filesystem`
-  echo $filesystem $age >> $tmpfile
+  if [[ $filesystem =~ ':' ]]; then
+    dst_pool=${filesystem%%:*}
+    filesystem=${filesystem#*:} # remove src_pool from string
+  else
+    dst_pool=$backup_pool
+  fi
+  # Only backup filesystems that are specified to go this backup_pool
+  if [[ $backup_pool == $dst_pool ]]; then
+    age=`snapshot_age $filesystem`
+    echo $filesystem $age >> $tmpfile
+  fi
 done
 sorted_filesystems=`cat $tmpfile | sort -n -k 2 | awk '{ print $1 }'`
 rm -f $tmpfile
@@ -129,7 +142,7 @@ rm -f $tmpfile
 # Replicate the sorted filesystems
 for filesystem in $sorted_filesystems; do
   echo "-> Replicating $remote:$filesystem to ${backup_pool}/${filesystem}" | tee -a $mylogfile
-  replicate $remote $filesystem
+  replicate $remote $filesystem $backup_pool
 done
 
 # Export the local pool if told to do so
@@ -145,6 +158,8 @@ if [[ $import_export_backup_pool == 1 ]]; then
   fi
 fi
 
+done
+
 # All done
 echo `date` ZFS replicate complete >> $logfile
 echo `date` ZFS replicate complete | tee -a $mylogfile
diff --git a/zfs-restore-all b/zfs-restore-all
index 3931190..dbe09a1 100755
--- a/zfs-restore-all
+++ b/zfs-restore-all
@@ -30,6 +30,12 @@ echo `date` ZFS restore started
 
 # For each filesystem we are supposed to restore, do it
 for filesystem in $filesystems_to_replicate; do
+  if [[ $filesystem =~ ':' ]]; then
+      src_pool=${filesystem%%:*}
+      filesystem=${filesystem#*:} # remove src_pool from string
+  else
+      src_pool=${backup_pool%% *} # use the first backup pool if none specified
+  fi
   dst_pool=${filesystem%%/*}
   dst_fs=${filesystem#*/}
   # Check to make sure the dst filesystem does not exist
@@ -43,7 +49,7 @@ for filesystem in $filesystems_to_replicate; do
     if [ $? == 0 ]; then
       # This filesystem matches our restore pool pattern 
       echo `date` Restoring $filesystem to $remote
-      zfs-restore $backup_pool $filesystem $dst_pool $dst_fs $remote
+      zfs-restore $src_pool $filesystem $dst_pool $dst_fs $remote
     fi
   else
     echo "-I- Filesystem already exists on destination. Skipping: $filesystem"
diff --git a/zfs-scripts.conf.dist b/zfs-scripts.conf.dist
index 019ae87..e213658 100644
--- a/zfs-scripts.conf.dist
+++ b/zfs-scripts.conf.dist
@@ -32,6 +32,7 @@ ssh="ssh"
 remote=
 
 # specify the name of the local pool to store the $remote's snapshots into
+# multiple pools can be specified by separating with a space
 backup_pool=
 
 # import backup pool before replication and export backup pool after replication
@@ -55,7 +56,7 @@ throttle_enable=0
 throttle_opt="-v -M 10"
 
 # Specify the list of filesystems to replicate from the $remote to the $backup_pool (1 per line)
-# The format of each line should be: pool/filesystem
+# The format of each line should be: pool/filesystem OR dst_pool:src_pool/filesystem
 filesystems_to_replicate="
 "
 
diff --git a/zfs-scrub b/zfs-scrub
index fe94f81..50813fc 100755
--- a/zfs-scrub
+++ b/zfs-scrub
@@ -34,7 +34,7 @@ do
   # Import the local pool if needed and option was given to do so, else error out
   zpool list -H "$i" >/dev/null 2>&1
   if [ $? != 0 ]; then
-    if [[ $import_export_backup_pool == 1 ]] && [[ "$i" =~ "$backup_pool" ]]; then
+    if [[ $import_export_backup_pool == 1 ]] && [[ "$backup_pool" =~ "$i" ]]; then
       zpool import $i
       if [ $? != 0 ]; then
         fatal_and_exit "-E- unable to import the pool $i"
@@ -84,7 +84,7 @@ do
   # do this so we can check its status to see if it repaired any errors or not
   zpool list -H "$i" >/dev/null 2>&1
   if [ $? != 0 ]; then
-    if [[ $import_export_backup_pool == 1 ]] && [[ "$i" =~ "$backup_pool" ]]; then
+    if [[ $import_export_backup_pool == 1 ]] && [[ "$backup_pool" =~ "$i" ]]; then
       zpool import $i
       if [ $? != 0 ]; then
         fatal_and_exit "-E- unable to import the pool $i"
@@ -103,7 +103,7 @@ do
   fi 
 
   # Export the local pool if told to do so
-  if [[ $import_export_backup_pool == 1 ]] && [[ "$i" =~ "$backup_pool" ]]; then
+  if [[ $import_export_backup_pool == 1 ]] && [[ "$backup_pool" =~ "$i" ]]; then
     # Don't export the pool if there is a currently running zfs-replicate operation
     ps -ef | grep -q "zfs-replicate" | grep -v grep
     if [ $? != 0 ]; then
-- 
2.34.1


From 3cae66d2b4a3eba94cb0765db85ff5cdc0ac194a Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Mon, 3 Feb 2020 14:01:29 -0700
Subject: [PATCH 05/16] Fixed bug that caused empty error messages to be
 emailed back

Change-Id: I28685c64435343bc48c938cb422ad06373de01f6
---
 zfs-replicate-all | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/zfs-replicate-all b/zfs-replicate-all
index 66949df..f5e23e7 100755
--- a/zfs-replicate-all
+++ b/zfs-replicate-all
@@ -50,16 +50,15 @@ cleanup_and_exit() {
 }
 trap cleanup_and_exit INT
 
-fatal() {
+fatal_and_exit() {
   # echo message to terminal
   echo -e 2>&1 "$1"
   # send email notification
-  echo -e "$1" | $mailx -s "zfs-replicate-all on $hostname failed" "$mailto"
-}
-
-fatal_and_exit() {
-  fatal $*
-  exit 1
+  echo -e "$1" | $mailx -s "zfs-replicate-all on $hostname failed" "$2"
+  # exit with bad return code unless 3rd argument was defined that says not to
+  if [[ -z "$3" ]]; then
+    exit 1
+  fi
 }
 
 # This function executes the replicate command and checks the stoptime
@@ -153,7 +152,7 @@ if [[ $import_export_backup_pool == 1 ]]; then
     zpool export $backup_pool
     if [ $? != 0 ]; then
       lsof /$backup_pool/*
-      fatal "-E- unable to export the local backup pool $backup_pool on $hostname" "$mailto"
+      fatal_and_exit "-E- unable to export the local backup pool $backup_pool on $hostname" "$mailto" 0
     fi
   fi
 fi
-- 
2.34.1


From 8a8252624f621fd3ebc13b45b2c456d535c339b8 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Mon, 3 Feb 2020 14:05:51 -0700
Subject: [PATCH 06/16] Updated comment to describe actual behavior

Change-Id: Id8138151c76c3211809cb2ff27960dfca0a247e2
---
 zfs-scripts.conf.dist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/zfs-scripts.conf.dist b/zfs-scripts.conf.dist
index e213658..894f783 100644
--- a/zfs-scripts.conf.dist
+++ b/zfs-scripts.conf.dist
@@ -57,6 +57,7 @@ throttle_opt="-v -M 10"
 
 # Specify the list of filesystems to replicate from the $remote to the $backup_pool (1 per line)
 # The format of each line should be: pool/filesystem OR dst_pool:src_pool/filesystem
+# NOTE: If dst_pool is not specified, the given pool/filesystem will be replicated to all pools in backup_pool
 filesystems_to_replicate="
 "
 
-- 
2.34.1


From 16f8b14e3150ee6efec728b86c9a07cdd2dc3773 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Sun, 16 Feb 2020 21:35:48 -0700
Subject: [PATCH 07/16] Made detecting if zpool is already mounted or unmounted
 better

Change-Id: I39bf8bb70f57ff6d1c7c260207beaadbd855b10d
---
 zfs-replicate-all | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/zfs-replicate-all b/zfs-replicate-all
index f5e23e7..5d606bf 100755
--- a/zfs-replicate-all
+++ b/zfs-replicate-all
@@ -97,12 +97,12 @@ backup_pools=$backup_pool
 for backup_pool in $backup_pools; do
 
 # Import the local backup pool if needed and the option is given to do so, else error out
-zpool list -H "$backup_pool" >/dev/null 2>&1
+zpool list -H $backup_pool >/dev/null 2>&1
 if [ $? != 0 ]; then
   if [[ $import_export_backup_pool == 1 ]]; then
     zpool import $backup_pool
     if [ $? != 0 ]; then
-      fatal_and_exit "-E- unable to import the backup pool $backup_pool on $hostname" "$mailto"
+      fatal_and_exit "-E- unable to import the backup pool '$backup_pool' on $hostname" "$mailto"
     fi
   else 
     fatal_and_exit "-E- The local backup pool on $hostname, '$backup_pool' doesn't seem to exist." "$mailto"
@@ -151,8 +151,12 @@ if [[ $import_export_backup_pool == 1 ]]; then
   if [ $? != 0 ]; then
     zpool export $backup_pool
     if [ $? != 0 ]; then
-      lsof /$backup_pool/*
-      fatal_and_exit "-E- unable to export the local backup pool $backup_pool on $hostname" "$mailto" 0
+      sleep 300
+      zpool export $backup_pool
+      if [ $? != 0 ]; then
+        lsof /$backup_pool/*
+        fatal_and_exit "-E- unable to export the local backup pool $backup_pool on $hostname" "$mailto" 0
+      fi
     fi
   fi
 fi
-- 
2.34.1


From 65f2beeb586c52a7c7571f43de328b584fa687fd Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Tue, 18 Feb 2020 08:00:47 -0700
Subject: [PATCH 08/16] Added logging of messages when trying to import backup
 pool

Change-Id: I44bec9aa3593e5b80d05d22d2d76323cadef3bee
---
 zfs-replicate-all | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/zfs-replicate-all b/zfs-replicate-all
index 5d606bf..999c95e 100755
--- a/zfs-replicate-all
+++ b/zfs-replicate-all
@@ -97,17 +97,21 @@ backup_pools=$backup_pool
 for backup_pool in $backup_pools; do
 
 # Import the local backup pool if needed and the option is given to do so, else error out
-zpool list -H $backup_pool >/dev/null 2>&1
+tmpfile=`tempfile`
+zpool list -H $backup_pool > $tmpfile 2>&1
 if [ $? != 0 ]; then
   if [[ $import_export_backup_pool == 1 ]]; then
-    zpool import $backup_pool
+    zpool import $backup_pool >> $tmpfile 2>&1
     if [ $? != 0 ]; then
-      fatal_and_exit "-E- unable to import the backup pool '$backup_pool' on $hostname" "$mailto"
+      msgs=`cat $tmpfile`
+      fatal_and_exit "-E- unable to import the backup pool '$backup_pool' on $hostname: $msgs" "$mailto"
     fi
   else 
-    fatal_and_exit "-E- The local backup pool on $hostname, '$backup_pool' doesn't seem to exist." "$mailto"
+    msgs=`cat $tmpfile`
+    fatal_and_exit "-E- The local backup pool on $hostname, '$backup_pool' doesn't seem to exist: $msgs" "$mailto"
   fi
 fi
+rm -f $tmpfile
 
 # Obtain the zpool guid for the local backup pool
 backup_pool_guid=`zpool get guid $backup_pool 2>&1 | grep $backup_pool | awk '{ print $3 }'`
-- 
2.34.1


From 802dba950b6593de90330d45ecfc26643cc7bb78 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Sat, 29 Feb 2020 11:13:35 -0700
Subject: [PATCH 09/16] Collect messages from zpool export cmds and email those
 if it fails to export

Change-Id: I7bb12c4fbab23b2c5bf6b63c76a0fb4abf97da33
---
 zfs-replicate-all | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/zfs-replicate-all b/zfs-replicate-all
index 999c95e..01f1bec 100755
--- a/zfs-replicate-all
+++ b/zfs-replicate-all
@@ -14,6 +14,7 @@ logfile="$logdir/zfs-replicate.log"
 mylogfile="$logdir/zfs-replicate-all.log"
 date=`date`
 starttime=`date +%s`
+tmpfile=`tempfile`
 
 # Make sure we aren't already running
 SCRIPT_NAME=${0##*/}
@@ -97,7 +98,6 @@ backup_pools=$backup_pool
 for backup_pool in $backup_pools; do
 
 # Import the local backup pool if needed and the option is given to do so, else error out
-tmpfile=`tempfile`
 zpool list -H $backup_pool > $tmpfile 2>&1
 if [ $? != 0 ]; then
   if [[ $import_export_backup_pool == 1 ]]; then
@@ -125,7 +125,6 @@ current_backup_marker=@current-backup-${backup_pool_guid}
 previous_backup_marker=@previous-backup-${backup_pool_guid}
 
 # Sort the filesystems to replicate by the oldest backup first
-tmpfile=`tempfile`
 for filesystem in $filesystems_to_replicate; do
   if [[ $filesystem =~ ':' ]]; then
     dst_pool=${filesystem%%:*}
@@ -153,17 +152,19 @@ if [[ $import_export_backup_pool == 1 ]]; then
   # Don't export the pool if there is a currently running zfs-scrub operation
   ps -ef | grep "zfs-scrub" | grep -q "${backup_pool}" | grep -v grep
   if [ $? != 0 ]; then
-    zpool export $backup_pool
+    zpool export $backup_pool >> $tmpfile 2>&1
     if [ $? != 0 ]; then
       sleep 300
-      zpool export $backup_pool
+      zpool export $backup_pool >> $tmpfile 2>&1
       if [ $? != 0 ]; then
-        lsof /$backup_pool/*
-        fatal_and_exit "-E- unable to export the local backup pool $backup_pool on $hostname" "$mailto" 0
+        lsof /$backup_pool/* >> $tmpfile 2>&1
+	msgs=`cat $tmpfile`
+        fatal_and_exit "-E- unable to export the local backup pool $backup_pool on $hostname: $msgs" "$mailto" 0
       fi
     fi
   fi
 fi
+rm -f $tmpfile
 
 done
 
-- 
2.34.1


From 1b47ee7ab9a7902d5a97ff550fabf9f635abef30 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Sun, 5 Apr 2020 10:36:50 -0600
Subject: [PATCH 10/16] Add feature to try rollback twice if it fails

Change-Id: I7b502fbfecaa3269e1dea64b110e95a84ce9b1b7
---
 zfs-replicate | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/zfs-replicate b/zfs-replicate
index 9dfd9f8..02f9924 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -203,9 +203,13 @@ if [ $no_markers == 0 ]; then
   # backup was interrupted for some reason. If we don't do this, the zfs send -R command
   # below may complain about snaps already existing as it tries to resend from the 
   # previous backup marker again from a previously interrupted replicate.
-  $zfs rollback -r ${backup_pool}/${previous_backup_marker} 
+  $zfs rollback -rf ${backup_pool}/${previous_backup_marker} 
   if [ $? != 0 ]; then
-    fatal_and_exit "-E- remote incremental $zfs rollback command failed on $hostname" $mailto
+    sleep 120
+    $zfs rollback -rf ${backup_pool}/${previous_backup_marker}
+    if [ $? != 0 ]; then
+      fatal_and_exit "-E- remote incremental $zfs rollback command failed on $hostname" $mailto
+    fi
   fi
   # Now it should be safe to send the snaps
   replicate "$zfs send -Rc -I${previous_backup_marker} ${current_backup_marker}"
-- 
2.34.1


From a44c3f17def4addd4be08f31b934afa9c4ce0a8d Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Sun, 23 Aug 2020 18:21:42 -0600
Subject: [PATCH 11/16] Return non-zero exit code if any command in pipe fails

Change-Id: I463e7afc7ee01a352558194eabbac67c9446b106
---
 zfs-replicate | 4 ++++
 zfs-restore   | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/zfs-replicate b/zfs-replicate
index 02f9924..7f205e2 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -18,6 +18,9 @@ remote=$1
 remote_fs=$2
 remote_pool=${2%%/*}
 
+# return non-zero exit code if any command in the pipe fails
+set -o pipefail
+
 # get the backup pool from the command line or the config file if not specified
 if [[ -n $3 ]]; then
     backup_pool=$3
@@ -264,6 +267,7 @@ fi
 # don't check the return codes here because these may not exist, and that is ok
 $zfs destroy ${backup_pool}/${previous_backup_marker} > /dev/null 2>&1
 $ssh $remote $zfs destroy ${previous_backup_marker} > /dev/null 2>&1
+sleep 1
 
 # Rename the current backup marker to be the previous backup marker
 $zfs rename ${backup_pool}/${current_backup_marker} ${backup_pool}/${previous_backup_marker}
diff --git a/zfs-restore b/zfs-restore
index 4014ff7..d5ca2e6 100755
--- a/zfs-restore
+++ b/zfs-restore
@@ -16,6 +16,9 @@ dst_pool=$3
 dst_fs=$4
 dst_hostname=$5
 
+# return non-zero exit code if any command in the pipe fails
+set -o pipefail
+
 if [[ -z "$SCRIPT_UNDER_TEST" ]]; then
     exec >> $logdir/zfs-restore.log 2>&1
 fi
-- 
2.34.1


From 4f58891850b6285301001bf64cb31659ee87af67 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Fri, 2 Oct 2020 12:47:50 -0600
Subject: [PATCH 12/16] Don't report still in progress zfs scrubs as errors

Change-Id: I7476a6f0f2f010b5a24bbcdfd07e19418eb6688b
---
 zfs-scrub | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zfs-scrub b/zfs-scrub
index 50813fc..5388f54 100755
--- a/zfs-scrub
+++ b/zfs-scrub
@@ -96,7 +96,7 @@ do
 
   # Check for any scrub errors
   zpool status $i | grep scan: 
-  zpool status $i | grep scan: | grep "with 0 errors" > /dev/null 2>&1
+  zpool status $i | grep scan: | grep -e "with 0 errors" -e "in progress" > /dev/null 2>&1
   if [ $? != 0 ]; then
     # The scrub found errors
     zpool status $i | $mailx -s "zpool scrub on $hostname $i found errors" "$mailto" 
-- 
2.34.1


From 042746138ba0f15394db01ce99dd5b5e81002451 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Sun, 31 Jan 2021 18:44:04 -0700
Subject: [PATCH 13/16] Added new option to expect empty remote lists when
 backing up to a new disk

Change-Id: I77f3f0323f876adbb3e46640dc3acee9e3b43146
---
 zfs-replicate         | 2 +-
 zfs-scripts.conf.dist | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/zfs-replicate b/zfs-replicate
index 7f205e2..a5a3ad9 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -170,7 +170,7 @@ $zfs list -H -t snapshot |
     grep ^${backup_pool}/${remote_fs}@ |
     grep -v ^${backup_pool}/${current_backup_marker} | 
     awk "{gsub(/^$backup_pool./,\"\",\$1); print\$1}" > $local_list
-if [ $? != 0 ]; then
+if [[ $? != 0 ]] && [[ $expect_empty_remote_list == 0 ]]; then
   fatal_and_exit "-E- local $zfs list on $hostname command failed" $mailto
 fi
 
diff --git a/zfs-scripts.conf.dist b/zfs-scripts.conf.dist
index 894f783..e4f3651 100644
--- a/zfs-scripts.conf.dist
+++ b/zfs-scripts.conf.dist
@@ -38,6 +38,11 @@ backup_pool=
 # import backup pool before replication and export backup pool after replication
 import_export_backup_pool=0
 
+# expect empty remote list 
+# - set to 1 if backing up any new filesystems for the first time
+# - set to 0 if not backing up new filesystems for the first time (to prevent dropping the entire filesystem if the remote list command fails)
+expect_empty_remote_list=0
+
 # when this variable is set, local filesystems will be destroyed
 # before receiving a full streams into them from the remote source.
 # if it needs to do this, and this option is set to 0, it aborts.
-- 
2.34.1


From 9eb1c325c28f5608ae26eeccc161f1fc5f70b083 Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Sun, 20 Jun 2021 09:37:28 -0600
Subject: [PATCH 14/16] Fixed case when local filesystem is missing

Change-Id: Ifcf8773d148b058825519b4676a4a456e95e933f
---
 zfs-replicate | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/zfs-replicate b/zfs-replicate
index a5a3ad9..e6e6dd5 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -170,9 +170,9 @@ $zfs list -H -t snapshot |
     grep ^${backup_pool}/${remote_fs}@ |
     grep -v ^${backup_pool}/${current_backup_marker} | 
     awk "{gsub(/^$backup_pool./,\"\",\$1); print\$1}" > $local_list
-if [[ $? != 0 ]] && [[ $expect_empty_remote_list == 0 ]]; then
-  fatal_and_exit "-E- local $zfs list on $hostname command failed" $mailto
-fi
+# If no local snapshots exist, we may need to send the entire filesystem, which we'll do later
+# So, no error check here as a non-zero return code means the local filesystem or snaps are missing
+# We'll catch this later on as a case where we need to send the entire filesystem
 
 # Destroy the current backup marker snapshot on the remote system if it exists
 grep -q ${current_backup_marker} $remote_list
-- 
2.34.1


From d343c96fd4190924d3407cc9de1a12f66491670c Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Fri, 20 Aug 2021 16:03:56 -0600
Subject: [PATCH 15/16] Need to be root to destroy snapshots

Change-Id: I59d7cc59785b99219af3bdf1163461e738ee1d9c
---
 zfs-rm-all-snaps | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zfs-rm-all-snaps b/zfs-rm-all-snaps
index 868bba5..5bc0609 100755
--- a/zfs-rm-all-snaps
+++ b/zfs-rm-all-snaps
@@ -29,7 +29,7 @@ read
 
 for snap in `zfs list -t snapshot 2>/dev/null | grep "^$zfs_pool" | grep "$snap_pattern" | awk '{print $1}'`; do
   echo "   destroying snapshot: $snap"
-  zfs destroy "$snap" 2>/dev/null
+  sudo zfs destroy "$snap" 2>/dev/null
 done
 
 echo "-> Snapshots removed"
-- 
2.34.1


From 0b70bfc0d86add94bad09cd7f5ef78e3fed92daf Mon Sep 17 00:00:00 2001
From: "Alan J. Pippin" <alan@pippins.net>
Date: Wed, 22 Sep 2021 17:31:24 -0600
Subject: [PATCH 16/16] Use expect_empty_remote_list flag when checking if
 remote list should be empty or not

Change-Id: I3d7820c19f4549163d946668f2c3c99a3d5f79fe
---
 zfs-replicate | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/zfs-replicate b/zfs-replicate
index e6e6dd5..60a2345 100755
--- a/zfs-replicate
+++ b/zfs-replicate
@@ -158,8 +158,10 @@ $ssh $remote \
     $zfs list -H -t snapshot |
     grep ^${remote_fs}@ |
     awk '{print$1}' > $remote_list
-if [ $? != 0 ]; then
-  fatal_and_exit "-E- remote $zfs list on $hostname command failed" $mailto
+if [[ $? != 0 ]] && [[ $expect_empty_remote_list == 0 ]]; then
+  echo "-W- Unable to find $remote_fs on the remote host $hostname. Unable to proceed since the"
+  echo "    expect_empty_remote_list option has not been set to allow this, skipping replicate operation."
+  fatal_and_exit "-E- remote $zfs list on $hostname for $remote_fs command failed" $mailto
 fi
 
 # List the snapshots on the local machine.
-- 
2.34.1