From 3ebfcee8326938e775dd63eed8075de0717dcd84 Mon Sep 17 00:00:00 2001 From: Sander van der Burg Date: Tue, 13 Feb 2018 20:31:03 +0100 Subject: [PATCH] Add delete and list generation operations --- NEWS | 2 + doc/manual/basicusage.xml | 42 ++++++++++++++ scripts/disnix-env.in | 113 ++++++++++++++++++++++++++++++-------- tests/deployment.nix | 53 ++++++++++++++++-- 4 files changed, 182 insertions(+), 28 deletions(-) diff --git a/NEWS b/NEWS index 10f459f0..a1334c97 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,8 @@ Version 0.8 - The copy closure command now partitions the requisites to prevent hitting the "arguments too long" error +- The disnix-env command now also takes the --list-generations option to display coordinator profile generations and --delete-generations, --delete-all-generations to delete them + Version 0.7 =========== - When moving state it will no longer be deleted by default. Instead, the user must provide --delete-state diff --git a/doc/manual/basicusage.xml b/doc/manual/basicusage.xml index 84f9719d..54912351 100644 --- a/doc/manual/basicusage.xml +++ b/doc/manual/basicusage.xml @@ -1092,6 +1092,15 @@ $ disnix-env -s services.nix -i infrastructure.nix -d distribution.nix on the coordinator machine, then the services and their intra-dependencies are transferred to the target machines in the network. Finally, the services are activated. + + + When running the following command-line instruction: + +$ disnix-env --list-generations + 1 2018-02-13 13:22:35 (current) + + We can see an overview of deployment generations. Because this is our initial deployment, we only have one generation. +
@@ -1118,6 +1127,17 @@ $ disnix-env -s services.nix -i infrastructure.nix -d distribution.nix from the coordinator machine to test2 and redeploys the web application front-end to connect to the new location of the ZipcodeService. All the other parts of the system will remain untouched. + + + When running the following command-line instruction: + + 1 2018-02-13 13:22:35 + 2 2018-02-13 13:30:24 (current) + + We will see that we have two configurations deployed, the current after the upgrade (generation 2) and the previous configuration (generation 1). + Because Disnix builts on top of Nix, older configurations are never removed by default. We can use the older + configuration to easily roll back to a previous configuration, when desired. +
@@ -1172,6 +1192,28 @@ $ disnix-collect-garbage infrastructure.nix option can be used: $ disnix-collect-garbage -d infrastructure.nix + + + + In addition to the target machines, the coordinator machine also keeps packages of all configurations + unless they are declared obsolete and garbage collected. We can use the following command + to delete all older generations on the coordinator machine: + +$ disnix-env --delete-generations old + + + + When running the garbage collector on the coordinator machine, all obsolete packages of the obsolete + configurations will be removed: + +$ nix-collect-garbage -d + + + + It also possible to discard all profile generations. This is useful, for example, when the + target machines are no longer available: + +$ disnix-env --delete-all-generations
diff --git a/scripts/disnix-env.in b/scripts/disnix-env.in index 7570c884..503a3394 100644 --- a/scripts/disnix-env.in +++ b/scripts/disnix-env.in @@ -28,6 +28,9 @@ showUsage() Usage: $0 -s services_nix -i infrastructure_nix -d distribution_nix [OPTION] or: $0 --rollback [OPTION] or: $0 --switch-generation NUM [OPTION] + or: $0 --list-generations [OPTION] + or: $0 --delete-generations NUM [OPTION] + or: $0 --delete-all-generations NUM [OPTION] The command \`disnix-env' is used to install, upgrade, or roll back a service-oriented system in a given environment. @@ -75,6 +78,13 @@ Options: --switch-to-generation=NUM Switches to a specific profile generation --rollback Switches back to the previously deployed configuration + --list-generations Lists all profile generations of the current + deployment + --delete-generations=NUM Deletes the specified generations. The number + can correspond to generation numbers, days + (d postfix) or 'old'. + --delete-all-generations Deletes all profile generations. This is + useful when a deployment has been discarded --target-property=PROP The target property of an infrastructure model, that specifies how to connect to the remote Disnix interface. (Defaults to @@ -151,7 +161,7 @@ EOF # Parse valid argument options -PARAMS=`@getopt@ -n $0 -o s:i:d:p:m:hv -l services:,infrastructure:,distribution:,rollback,switch-to-generation:,interface:,target-property:,deploy-state,profile:,max-concurrent-transfers:,build-on-targets,coordinator-profile-path:,no-upgrade,no-lock,no-coordinator-profile,no-target-profiles,no-migration,delete-state,depth-first,keep:,show-trace,help,version -- "$@"` +PARAMS=`@getopt@ -n $0 -o s:i:d:p:m:hv -l services:,infrastructure:,distribution:,rollback,switch-to-generation:,list-generations,delete-generations:,delete-all-generations,interface:,target-property:,deploy-state,profile:,max-concurrent-transfers:,build-on-targets,coordinator-profile-path:,no-upgrade,no-lock,no-coordinator-profile,no-target-profiles,no-migration,delete-state,depth-first,keep:,show-trace,help,version -- "$@"` if [ $? != 0 ] then @@ -176,12 +186,22 @@ do distributionFile=`readlink -f $2` ;; --rollback) - switchGeneration=1 + operation="switchGenerations" ;; --switch-to-generation) - switchGeneration=1 + operation="switchGenerations" generationId=$2 ;; + --list-generations) + operation="listGenerations" + ;; + --delete-generations) + operation="deleteGenerations" + generations="$2" + ;; + --delete-all-generations) + operation="deleteAllGenerations" + ;; -m|--max-concurrent-transfers) maxConcurrentTransfersArg="-m $2" ;; @@ -241,7 +261,7 @@ do exit 0 ;; esac - + shift done @@ -255,13 +275,6 @@ source @datadir@/@PACKAGE@/checks # Validate the given options -if [ "$switchGeneration" != "1" ] -then - checkServicesFile - checkInfrastructureFile - checkDistributionFile -fi - checkClientInterface checkTargetProperty checkProfile @@ -279,10 +292,17 @@ else profileDir="$coordinatorProfilePath" fi -# Execute operations +# Operation functions -if [ "$switchGeneration" = "1" ] -then +checkModels() +{ + checkServicesFile + checkInfrastructureFile + checkDistributionFile +} + +switchGenerations() +{ # If no generation link has been requested, figure out what the previous one is if [ "$generationId" = "" ] then @@ -296,7 +316,7 @@ then else generation="$profile-$generationId-link" fi - + if [ -e "$profileDir/$generation" ] then # Retrieve the manifest of the requested profile generation @@ -305,7 +325,26 @@ then echo "[coordinator]: Cannot switch profile generations, because the requested profile does not exist!" >&2 exit 1 fi -else +} + +listGenerations() +{ + nix-env -p $profileDir/$profile --list-generations +} + +deleteGenerations() +{ + nix-env -p $profileDir/$profile --delete-generations "$generations" +} + +deleteAllGenerations() +{ + rm -f $profileDir/$profile + rm -f $profileDir/$profile-*-link +} + +produceManifest() +{ if [ "$buildOnTargets" = "1" ] then disnix-delegate -s $servicesFile -i $infrastructureFile -d $distributionFile --target-property $targetProperty --interface $interface $maxConcurrentTransfersArg $showTraceArg @@ -313,13 +352,39 @@ else echo "[coordinator]: Building manifest..." manifest=`disnix-manifest -s $servicesFile -i $infrastructureFile -d $distributionFile --target-property $targetProperty --no-out-link --interface $interface $deployStateArg $showTraceArg` -fi +} -# Determine the name of the previous manifest file -if [ -d "$profileDir" ] && [ "$(echo $profileDir/$profile-*-link)" != "" ] -then - oldManifestArg="-o $(ls $profileDir/$profile-*-link | sort -V | tail -1)" -fi +deploy() +{ + # Determine the name of the previous manifest file + if [ -d "$profileDir" ] && [ "$(echo $profileDir/$profile-*-link)" != "" ] + then + oldManifestArg="-o $(ls $profileDir/$profile-*-link | sort -V | tail -1)" + fi + + # Deploy the (pre)built Disnix configuration (implying a manifest file) + disnix-deploy $maxConcurrentTransfersArg $noLockArg $profileArg $noUpgradeArg $deleteStateArg $noCoordinatorProfileArg $coordinatorProfilePathArg $noTargetProfilesArg $noMigrationArg $oldManifestArg $depthFirstArg $keepArg $manifest +} + +# Execute operations -# Deploy the (pre)built Disnix configuration (implying a manifest file) -disnix-deploy $maxConcurrentTransfersArg $noLockArg $profileArg $noUpgradeArg $deleteStateArg $noCoordinatorProfileArg $coordinatorProfilePathArg $noTargetProfilesArg $noMigrationArg $oldManifestArg $depthFirstArg $keepArg $manifest +case "$operation" in + switchGenerations) + switchGenerations + deploy + ;; + listGenerations) + listGenerations + ;; + deleteGenerations) + deleteGenerations + ;; + deleteAllGenerations) + deleteAllGenerations + ;; + *) + checkModels + produceManifest + deploy + ;; +esac diff --git a/tests/deployment.nix b/tests/deployment.nix index 0296df25..a109718d 100644 --- a/tests/deployment.nix +++ b/tests/deployment.nix @@ -43,6 +43,15 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem # This test should succeed. $coordinator->mustSucceed("${env} disnix-env -s ${manifestTests}/services-complete.nix -i ${manifestTests}/infrastructure.nix -d ${manifestTests}/distribution-simple.nix"); + # Check if we have one profile generation. + my $result = $coordinator->mustSucceed("${env} disnix-env --list-generations | wc -l"); + + if($result == 1) { + print("We have 1 profile generation\n"); + } else { + die "We should have 1 profile generation, instead we have: $result"; + } + # Do another rollback. Since there is no previous deployment, it should fail. $coordinator->mustFail("${env} disnix-env --rollback"); @@ -89,7 +98,7 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem # folder and one generation link in the target profiles folder on each # machine. - my $result = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); + $result = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); if($result == 2) { print "We have only one generation symlink on the coordinator!\n"; @@ -119,6 +128,14 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem $coordinator->mustSucceed("${env} disnix-env -s ${manifestTests}/services-complete.nix -i ${manifestTests}/infrastructure.nix -d ${manifestTests}/distribution-simple.nix"); + $result = $coordinator->mustSucceed("${env} disnix-env --list-generations | wc -l"); + + if($result == 1) { + print "We have one generation symlink\n"; + } else { + die "We should have one generation symlink, instead we have: $result" + } + $result = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); if($result == 2) { @@ -147,6 +164,14 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem # This test should succeed. $coordinator->mustSucceed("${env} disnix-env -s ${manifestTests}/services-complete.nix -i ${manifestTests}/infrastructure.nix -d ${manifestTests}/distribution-reverse.nix"); + $result = $coordinator->mustSucceed("${env} disnix-env --list-generations | wc -l"); + + if($result == 2) { + print "We have two generation symlinks"; + } else { + die "We should have two generation symlinks, instead we have: $result" + } + @lines = split('\n', $coordinator->mustSucceed("${env} disnix-query ${manifestTests}/infrastructure.nix")); if($lines[3] =~ /\-testService1/) { @@ -452,6 +477,19 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem die "disnix-query output line 9 does not contain testService3!\n"; } + # Remove old generation test. We remove one profile generation and we + # check if it has been successfully removed + + my $oldNumOfGenerations = $coordinator->mustSucceed("${env} disnix-env --list-generations | wc -l"); + $coordinator->mustSucceed("${env} disnix-env --delete-generations 1"); + my $newNumOfGenerations = $coordinator->mustSucceed("${env} disnix-env --list-generations | wc -l"); + + if($newNumOfGenerations == $oldNumOfGenerations - 1) { + print "We have successfully removed one generation!\n"; + } else { + die "We seem to have removed more than one generation, new: $newNumOfGenerations, old: $oldNumOfGenerations"; + } + # Test disnix-capture-infra. Capture the container properties of all # machines and generate an infrastructure expression from it. It should # contain: "foo" = "bar"; twice. @@ -475,9 +513,9 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem # Test disnix-reconstruct. Because nothing has changed the coordinator # profile should remain identical. - my $oldNumOfGenerations = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); + $oldNumOfGenerations = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); $coordinator->mustSucceed("${env} disnix-reconstruct ${manifestTests}/infrastructure.nix"); - my $newNumOfGenerations = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); + $newNumOfGenerations = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l"); if($oldNumOfGenerations == $newNumOfGenerations) { print "The amount of manifest generations remained the same!\n"; @@ -488,7 +526,14 @@ with import "${nixpkgs}/nixos/lib/testing.nix" { system = builtins.currentSystem # Test disnix-reconstruct. First, we remove the old manifests. They # should have been reconstructed. - $coordinator->mustSucceed("rm /nix/var/nix/profiles/per-user/root/disnix-coordinator/*"); + $result = $coordinator->mustSucceed("${env} disnix-env --delete-all-generations | wc -l"); + + if($result == 0) { + print "We have no symlinks"; + } else { + die "We should have no symlinks, instead we have: $result"; + } + $coordinator->mustSucceed("${env} disnix-reconstruct ${manifestTests}/infrastructure.nix"); $result = $coordinator->mustSucceed("ls /nix/var/nix/profiles/per-user/root/disnix-coordinator | wc -l");