From 984d48e66a916fd9ba533ae19b43aafd449dad77 Mon Sep 17 00:00:00 2001 From: "Alan J. Pippin" Date: Thu, 15 Jan 2009 23:57:18 -0700 Subject: [PATCH] Created the zfs-replicate-wrapper script that factors in duration to stop replicating once duration exceeds a certain amount. Updated zfs-log-parser to parse the stats from the replicate jobs properly. --- zfs-log-parser | 4 +- zfs-replicate | 14 ++++-- zfs-replicate-wrapper | 110 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 7 deletions(-) create mode 100755 zfs-replicate-wrapper diff --git a/zfs-log-parser b/zfs-log-parser index 4ecca3c..94e4490 100755 --- a/zfs-log-parser +++ b/zfs-log-parser @@ -52,14 +52,14 @@ sub parse_replicate_logfile { if(($in_replicate == 0) && ("$startdate" ne "") && ($line !~ /$startdate/)) { next; } if($line =~ /(\S+)\s+(\S+)\s+(\d+)\s+(\d+):(\d+):(\d+)\s+(\S+)\s+(\S+)/) { $dayname=$1; $month=$2; $daynum=$3; $hour=$4; $minute=$5; $sec=$6; $year=$8; - if(($in_replicate == 0) && ($line =~ /lock obtained/)) { + if(($in_replicate == 0) && ($line =~ /replicate started/)) { $in_replicate = 1; $date="$dayname $month $daynum $hour:$minute:$sec $year"; $totals{$date}{data} = 0; $totals{$date}{transfertime} = 0; $totals{$date}{duration} = time_to_seconds($hour,$minute,$sec); } - elsif(($in_replicate == 1) && ($line=~ /lock released/)) { + elsif(($in_replicate == 1) && ($line=~ /replicate complete/)) { $in_replicate = 0; $totals{$date}{duration} = time_to_seconds($hour,$minute,$sec) - $totals{$date}{duration}; } diff --git a/zfs-replicate b/zfs-replicate index dc6dab2..09cb3ba 100755 --- a/zfs-replicate +++ b/zfs-replicate @@ -10,7 +10,7 @@ remote_lockdir="/tmp/zfs-admin-lock" local_pool=backups # Set the email address to send notification to -mailto=alan@pippins.net +mailto=root@pippins.net # When this variable is set, local filesystems will be destroyed # before receiving a full streams into them from the remote source. @@ -19,6 +19,14 @@ destroy_local_filesystem_on_full_replicate=0 # The ssh connection doesn't find zfs without this. zfs=/usr/sbin/zfs +# Setup our cleanup and exit trap +cleanup() { + ssh $remote rm -rf "$remote_lockdir" + exit +} +trap cleanup INT +trap cleanup EXIT + # Make sure we have valid arguments if [[ -z "$remote" ]] || [[ -z "$remote_fs" ]]; then echo "Usage: $0 " @@ -78,10 +86,6 @@ while true; do fi done -# Declare a cleanup() method to remove the remote lockdir -cleanup() { ssh $remote rm -rf "$remote_lockdir"; } -trap cleanup EXIT - # Setup our backup marker names current_backup_marker=${remote_fs}@current-backup-${local_pool_guid} previous_backup_marker=${remote_fs}@previous-backup-${local_pool_guid} diff --git a/zfs-replicate-wrapper b/zfs-replicate-wrapper new file mode 100755 index 0000000..d699c79 --- /dev/null +++ b/zfs-replicate-wrapper @@ -0,0 +1,110 @@ +#!/bin/bash + +# Author: Alan J. Pippin +# Description: This script calls zfs-replicate for each filesystem needing +# to be backed up, or replicated, to another ZFS pool. + +# Setup some default values +zfsreplicate="/etc/bin/zfs-replicate" +logfile_parser="/etc/bin/zfs-log-parser" +logfile="/var/log/zfs/zfs-replicate.log" +remote="tank.pippins.net" +local_pool=backups +mailto=root@pippins.net +date=`date` +starttime=`date +%s` +zfs=/usr/sbin/zfs + +# Specify the list of filesystems to replicate +filesystems_to_replicate=' +naspool/www +naspool/git +' + +# Specify the maximum run time in minutes that this script can run +maxruntime=240 + +# This function checks to see if our runtime has exceeded our stoptime +timeexceeded() { + currenttime=`date +%s` + elapsedtime=$(($currenttime - $starttime)) + stoptime=$(($maxruntime*60)) + if [[ $elapsedtime -gt $stoptime ]]; then + #echo "$elapsedtime > $stoptime" + return 1 + fi + #echo "$elapsedtime < $stoptime" + return 0 +} + +# This function cleanup and exit trap +cleanup_and_exit() { + #echo "cleanup and exit" + rm -rf "$lockdir" + exit 0 +} +trap cleanup_and_exit INT + +# This function executes the replicate command and checks the stoptime +replicate() { + $zfsreplicate $* >> $logfile 2>&1 + timeexceeded + if [ $? == 1 ]; then + cleanup_and_exit + fi +} + +# This function obtains the date a given snapshot was created in epoch seconds +snapshot_age() { + snapshot=${local_pool}/${1}${previous_backup_marker} + $zfs list -t snapshot ${snapshot} > /dev/null 2>&1 + if [ $? == 0 ]; then + $zfs get creation ${snapshot} > /dev/null 2>&1 + if [ $? == 0 ]; then + snap_creation=`$zfs get creation ${snapshot} | grep $1 | awk '{ print $3" "$4" "$5" "$6" "$7 }'` + snap_age=`date -d "$snap_creation" +%s` + echo "$snap_age" + else + echo "0" + fi + else + echo "0" + fi +} + +# Obtain the zpool guid for the local pool +local_pool_guid=`zpool get guid $local_pool 2>&1 | grep $local_pool | awk '{ print $3 }'` +zpool get guid $local_pool > /dev/null 2>&1 +if [ $? != 0 ]; then + echo >&2 "-E- Unable to extract the guid for the local pool: $local_pool" + exit 1 +fi + +# Setup our backup marker names +current_backup_marker=@current-backup-${local_pool_guid} +previous_backup_marker=@previous-backup-${local_pool_guid} + +# Auto snapshot every zfs filesystem on the system specified below +echo "$date ZFS replicate started" >> $logfile + +# Sort the filesystems to replicate by the oldest backup first +tmpfile=`tempfile` +for filesystem in $filesystems_to_replicate; do + age=`snapshot_age $filesystem` + echo $filesystem $age >> $tmpfile +done +sorted_filesystems=`cat $tmpfile | sort -n -k 2 | awk '{ print $1 }'` +rm -f $tmpfile + +# Replicate the sorted filesystems +for filesystem in $sorted_filesystems; do + echo "-> Replicating $remote:$filesystem to ${local_pool}/${filesystem}" + replicate $remote $filesystem +done + +# All done +echo `date` ZFS replicate complete >> $logfile + +# Parse the log file and extract our backup stats +$logfile_parser "$logfile" "$date" >> $logfile + -- 2.34.1