From e5bbf1952d51ab406d73ddb49496422c2d7ebb61 Mon Sep 17 00:00:00 2001 From: "Alan J. Pippin" Date: Sun, 11 Jan 2009 23:19:35 -0700 Subject: [PATCH] Moved old zfs-replicate scripts out of the way. Copied Carl's new replicate script into place. --- zfs-replicate | 134 ++++-------------- zfs-replicate-local | 116 +++++++++++++++ ...ate-wrapper => zfs-replicate-wrapper-local | 0 3 files changed, 143 insertions(+), 107 deletions(-) create mode 100755 zfs-replicate-local rename zfs-replicate-wrapper => zfs-replicate-wrapper-local (100%) diff --git a/zfs-replicate b/zfs-replicate index da3df12..ef90b54 100755 --- a/zfs-replicate +++ b/zfs-replicate @@ -1,116 +1,36 @@ -#!/bin/bash +#/bin/bash -# Author: Carl Baldwin & Alan Pippin -# Description: This script replicates a given zfs filesystem to a given zfs pool. -# This script will keep all snapshots in sync, removing the ones -# that have been deleted since the last replicate was performed. -# This script will only send the new, or missing, snapshots since -# the last replicate was performed. +set -x -# In test mode (test=1) commands are echoed, not executed -test=0 +# Usage: replicate +remote=$1 +remote_fs=$2 -[ $test == 0 ] && exec >> /var/log/zfs/zfs-replicate.log 2>&1 +# The ssh connection doesn't find zfs without this. +zfs=/usr/sbin/zfs -# Usage: zfs-backup [filesystem] [destination_pool] -# This script has a limitation with children under a given filesystem. -# You must initially backup the parent filesystems first using this script -# before backing up any of the children filesystems. +# List the snapshots on the remote machine. +remote_list=$(mktemp /tmp/replicate.XXXXXX) +ssh $remote \ + $zfs list -H -t snapshot | + grep ^${remote_fs}@ | + awk '{print$1}' > $remote_list -fs=$1 -fs=${fs%/} -fsname=${1#*/} -srcpool=${1%%/*} -srcfs="${srcpool}/$fsname" -srcfs=${srcfs%/} -dstpool=$2 -dstfs="${dstpool}/$srcfs" -dstfs=${dstfs%/} -nodstsnaps=0 -common="" +# List the snapshots on the local machine. +local_list=$(mktemp /tmp/replicate.XXXXXX) +$zfs list -H -t snapshot | + grep ^backups/${remote_fs}@ | + awk '{gsub(/^backups./,"",$1); print$1}' > $local_list -if [ $test == 1 ]; then - echo "fs: $fs" - echo "fsname: $fsname" - echo "srcpool: $srcpool" - echo "srcfs: $srcfs" - echo "dstpool: $dstpool" - echo "dstfs: $dstfs" -fi +# See what the most recent snapshot on the remote end is. +latest=$(tail -n 1 $remote_list) -if ! zpool list -H "$srcpool" >/dev/null 2>&1; then - echo >&2 "-E- The source pool, '$srcpool' doesn't seem to exist." - exit 1 -fi +# I did this to make sure that diff would always display the most recent common +echo bogus.remote >> $remote_list +echo bogus.local >> $local_list +common=$(diff -u $remote_list $local_list | grep '^ ' | tail -n 1) -if ! zpool list -H "$dstpool" >/dev/null 2>&1; then - echo >&2 "-E- The destination pool, '$dstpool' doesn't seem to exist." - exit 1 -fi +ssh $remote $zfs send -R -I${common/*@/@} $latest | + $zfs receive -vF -d backups/${remote_fs%/*} -if ! zfs list -rH -t snapshot "$dstfs" 2>&1 | grep "$dstfs@" > /dev/null 2>&1; then - echo >&2 "-W- No snapshots detected on the destination drive for this filesystem: $dstfs" - if zfs list -t filesystem | grep "$dstfs"; then - echo >&2 "-I- Found zfs filesystem $dstfs on the destination pool $dstpool without any snapshots" - echo >&2 "-I- Removing the zfs filesystem: $dstfs" - zfs destroy "$dstfs" - fi - nodstsnaps=1 -fi - -if [ $nodstsnaps == 0 ]; then - zfs list -rH -t snapshot $srcfs | grep "$srcfs@" | awk '{print $1}' > /tmp/source-list - zfs list -rH -t snapshot $dstfs | grep "$dstfs@" | sed "s,$dstpool/,," | awk '{print $1}' > /tmp/destination-list - diff -u /tmp/source-list /tmp/destination-list | grep -v '^+++' | awk '/^\+/ {print}' | sed "s,^\+,$dstpool/," > /tmp/obsolete-snapshots - rm -f /tmp/source-list /tmp/destination-list - - echo >&2 "Removing obsolete backups from the destination pool" - for snapshot in $(cat /tmp/obsolete-snapshots); do - echo >&2 "Removing '$snapshot' from destination." - [ $test == 0 ] && zfs destroy "$snapshot" - done - - echo >&2 "Rolling back to the most recent snapshot on the destination." - [ $test == 0 ] && zfs rollback $(zfs list -rH -t snapshot $dstfs | grep "$dstfs@" | awk '{snap=$1} END {print snap}') - - echo >&2 "Calculating the most recent common snapshot between the two filesystems." - if zfs list -H "$dstfs" > /dev/null 2>&1; then - for snap in $(zfs list -rH -t snapshot "$dstfs" | grep "$dstfs@" | - sed 's,.*@,,' | awk '{print$1}'); do - if zfs list -rH -t snapshot "$fs" | grep "$fs@" | sed 's,.*@,,' | awk '{print$1}' | grep "^${snap}$" >/dev/null 2>&1; then - common=$snap - fi - done - fi -fi - -base=$common -foundcommon=false -if [ -z "$common" ]; then - foundcommon=true -fi - -for snap in $(zfs list -rH -t snapshot "$fs" | grep "$fs@" | - sed 's,.*@,,' | awk '{print$1}'); do - if [ "$snap" = "$common" ]; then - foundcommon=true - continue - fi - - if $foundcommon; then - if [ -z "$base" ]; then - echo >&2 "Sending '$fs@$snap'" - [ $test == 0 ] && zfs set readonly=on "$dstpool" - [ $test == 0 ] && zfs set atime=off "$dstpool" - [ $test == 0 ] && zfs set sharenfs=off "$dstpool" - [ $test == 0 ] && zfs set mountpoint=legacy "$dstpool" - [ $test == 0 ] && zfs send "$fs@$snap" | zfs recv -v "$dstfs" - else - echo >&2 "Sending '$fs@$base' -> '$fs@$snap'" - [ $test == 0 ] && zfs send -i "$fs@$base" "$fs@$snap" | zfs recv -v "$dstfs" - fi - base=$snap - fi -done - -true +rm -f $local_list $remote_list diff --git a/zfs-replicate-local b/zfs-replicate-local new file mode 100755 index 0000000..da3df12 --- /dev/null +++ b/zfs-replicate-local @@ -0,0 +1,116 @@ +#!/bin/bash + +# Author: Carl Baldwin & Alan Pippin +# Description: This script replicates a given zfs filesystem to a given zfs pool. +# This script will keep all snapshots in sync, removing the ones +# that have been deleted since the last replicate was performed. +# This script will only send the new, or missing, snapshots since +# the last replicate was performed. + +# In test mode (test=1) commands are echoed, not executed +test=0 + +[ $test == 0 ] && exec >> /var/log/zfs/zfs-replicate.log 2>&1 + +# Usage: zfs-backup [filesystem] [destination_pool] +# This script has a limitation with children under a given filesystem. +# You must initially backup the parent filesystems first using this script +# before backing up any of the children filesystems. + +fs=$1 +fs=${fs%/} +fsname=${1#*/} +srcpool=${1%%/*} +srcfs="${srcpool}/$fsname" +srcfs=${srcfs%/} +dstpool=$2 +dstfs="${dstpool}/$srcfs" +dstfs=${dstfs%/} +nodstsnaps=0 +common="" + +if [ $test == 1 ]; then + echo "fs: $fs" + echo "fsname: $fsname" + echo "srcpool: $srcpool" + echo "srcfs: $srcfs" + echo "dstpool: $dstpool" + echo "dstfs: $dstfs" +fi + +if ! zpool list -H "$srcpool" >/dev/null 2>&1; then + echo >&2 "-E- The source pool, '$srcpool' doesn't seem to exist." + exit 1 +fi + +if ! zpool list -H "$dstpool" >/dev/null 2>&1; then + echo >&2 "-E- The destination pool, '$dstpool' doesn't seem to exist." + exit 1 +fi + +if ! zfs list -rH -t snapshot "$dstfs" 2>&1 | grep "$dstfs@" > /dev/null 2>&1; then + echo >&2 "-W- No snapshots detected on the destination drive for this filesystem: $dstfs" + if zfs list -t filesystem | grep "$dstfs"; then + echo >&2 "-I- Found zfs filesystem $dstfs on the destination pool $dstpool without any snapshots" + echo >&2 "-I- Removing the zfs filesystem: $dstfs" + zfs destroy "$dstfs" + fi + nodstsnaps=1 +fi + +if [ $nodstsnaps == 0 ]; then + zfs list -rH -t snapshot $srcfs | grep "$srcfs@" | awk '{print $1}' > /tmp/source-list + zfs list -rH -t snapshot $dstfs | grep "$dstfs@" | sed "s,$dstpool/,," | awk '{print $1}' > /tmp/destination-list + diff -u /tmp/source-list /tmp/destination-list | grep -v '^+++' | awk '/^\+/ {print}' | sed "s,^\+,$dstpool/," > /tmp/obsolete-snapshots + rm -f /tmp/source-list /tmp/destination-list + + echo >&2 "Removing obsolete backups from the destination pool" + for snapshot in $(cat /tmp/obsolete-snapshots); do + echo >&2 "Removing '$snapshot' from destination." + [ $test == 0 ] && zfs destroy "$snapshot" + done + + echo >&2 "Rolling back to the most recent snapshot on the destination." + [ $test == 0 ] && zfs rollback $(zfs list -rH -t snapshot $dstfs | grep "$dstfs@" | awk '{snap=$1} END {print snap}') + + echo >&2 "Calculating the most recent common snapshot between the two filesystems." + if zfs list -H "$dstfs" > /dev/null 2>&1; then + for snap in $(zfs list -rH -t snapshot "$dstfs" | grep "$dstfs@" | + sed 's,.*@,,' | awk '{print$1}'); do + if zfs list -rH -t snapshot "$fs" | grep "$fs@" | sed 's,.*@,,' | awk '{print$1}' | grep "^${snap}$" >/dev/null 2>&1; then + common=$snap + fi + done + fi +fi + +base=$common +foundcommon=false +if [ -z "$common" ]; then + foundcommon=true +fi + +for snap in $(zfs list -rH -t snapshot "$fs" | grep "$fs@" | + sed 's,.*@,,' | awk '{print$1}'); do + if [ "$snap" = "$common" ]; then + foundcommon=true + continue + fi + + if $foundcommon; then + if [ -z "$base" ]; then + echo >&2 "Sending '$fs@$snap'" + [ $test == 0 ] && zfs set readonly=on "$dstpool" + [ $test == 0 ] && zfs set atime=off "$dstpool" + [ $test == 0 ] && zfs set sharenfs=off "$dstpool" + [ $test == 0 ] && zfs set mountpoint=legacy "$dstpool" + [ $test == 0 ] && zfs send "$fs@$snap" | zfs recv -v "$dstfs" + else + echo >&2 "Sending '$fs@$base' -> '$fs@$snap'" + [ $test == 0 ] && zfs send -i "$fs@$base" "$fs@$snap" | zfs recv -v "$dstfs" + fi + base=$snap + fi +done + +true diff --git a/zfs-replicate-wrapper b/zfs-replicate-wrapper-local similarity index 100% rename from zfs-replicate-wrapper rename to zfs-replicate-wrapper-local -- 2.34.1