Use bookmarks created after the latest snapshot

This commit changes syncoid's behavior so it is always looking for
a matching snapshot and a matching bookmark. If the bookmark was created
after the snapshot it is used instead. This allows replication when the
latest snapshot replicated was deleted on the source, a common snapshot
was found but rollback on the target is not allowed. The matching bookmark
is used instead for replication.

This fixes jimsalterjrs#602

syncoid

@@ -600,26 +600,30 @@ sub syncdataset {
 		my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used');
 
 		my $bookmark = 0;
-		my $bookmarkcreation = 0;
+		my $bookmarkguid = 0;
+		my $bookmarksnap = 0;
+		my $usebookmark = 0;
 
 		my $matchingsnap = getmatchingsnapshot($sourcefs, $targetfs, \%snaps);
-		if (! $matchingsnap) {
-			# no matching snapshots, check for bookmarks as fallback
-			my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
 
-			# check for matching guid of source bookmark and target snapshot (oldest first)
-			foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) {
-				my $guid = $snaps{'target'}{$snap}{'guid'};
+		# find most recent matching bookmark
+		my %bookmarks = getbookmarks($sourcehost,$sourcefs,$sourceisroot);
 
-				if (defined $bookmarks{$guid}) {
-					# found a match
-					$bookmark = $bookmarks{$guid}{'name'};
-					$bookmarkcreation = $bookmarks{$guid}{'creation'};
-					$matchingsnap = $snap;
-					last;
-				}
+		# check for matching guid of source bookmark and target snapshot (oldest first)
+		foreach my $snap ( sort { $snaps{'target'}{$b}{'creation'}<=>$snaps{'target'}{$a}{'creation'} } keys %{ $snaps{'target'} }) {
+			my $guid = $snaps{'target'}{$snap}{'guid'};
+
+			if (defined $bookmarks{$guid}) {
+				# found a match
+				$bookmark = $bookmarks{$guid}{'name'};
+				$bookmarkguid = $guid;
+				$bookmarksnap = $snap;
+				last;
 			}
+		}
 
+		if (! $matchingsnap) {
+			# no matching snapshots, check for bookmarks as fallback
 			if (! $bookmark) {
 				if ($args{'force-delete'}) {
 					if (!$quiet) { print "Removing $targetfs because no matching snapshots were found\n"; }
@@ -665,9 +669,18 @@ sub syncdataset {
 
 				# return false now in case more child datasets need replication.
 				return 0;
+			} else {
+				# use bookmark as fallback
+				$matchingsnap = $bookmarksnap;
+				$usebookmark = 1;
 			}
+		} elsif ($bookmark && $bookmarkguid ne $snaps{'source'}{$matchingsnap}{'guid'} && $bookmarks{$bookmarkguid}{'creation'} > substr($snaps{'source'}{$matchingsnap}{'creation'}, 0, -3)) {
+			if ($debug) { print "DEBUG: using bookmark $bookmark because it was created after latest matching snapshot $matchingsnap\n"; }
+			$matchingsnap = $bookmarksnap;
+			$usebookmark = 1;
 		}
 
+
 		# make sure target is (still) not currently in receive.
 		if (iszfsbusy($targethost,$targetfs,$targetisroot)) {
 			warn "Cannot sync now: $targetfs is already target of a zfs receive process.\n";
@@ -699,7 +712,7 @@ sub syncdataset {
 
 			my $nextsnapshot = 0;
 
-			if ($bookmark) {
+			if ($usebookmark) {
 				my $bookmarkescaped = escapeshellparam($bookmark);
 
 				if (!defined $args{'no-stream'}) {
@@ -782,7 +795,7 @@ sub syncdataset {
 
 			# do a normal replication if bookmarks aren't used or if previous
 			# bookmark replication was only done to the next oldest snapshot
-			if (!$bookmark || $nextsnapshot) {
+			if (!$usebookmark || $nextsnapshot) {
 				if ($matchingsnap eq $newsyncsnap) {
 					# edge case: bookmark replication used the latest snapshot
 					return 0;