diff --git a/apps/generate.nix b/apps/generate.nix
index 7c9677a..6851cab 100644
--- a/apps/generate.nix
+++ b/apps/generate.nix
@@ -56,7 +56,9 @@
   # If the path already exists, this makes sure that the definition is the same.
   addGeneratedSecretChecked = host: set: secretName: let
     secret = nodes.${host}.config.age.secrets.${secretName};
-    sourceFile = relativeToFlake secret.rekeyFile;
+    sourceFile = assert assertMsg (secret.rekeyFile != null)
+    "Host ${host}: age.secrets.${secretName}: `rekeyFile` must be set when using a generator.";
+      relativeToFlake secret.rekeyFile;
     script = secret.generator._script {
       inherit secret pkgs;
       inherit (pkgs) lib;
@@ -141,6 +143,12 @@
       (map (x: relativeToFlake x.rekeyFile) contextSecret.secret.generator.dependencies));
   in
     stringsWithDeps.textClosureMap (x: x) stages (attrNames stages);
+
+  # It only makes sense to clean up directories of generated secrets
+  # for the nodes that have a dedicated generatedSecretsDir set.
+  nodesWithGeneratedSecretsDir =
+    filter (x: x.config.age.rekey.generatedSecretsDir != null)
+    (attrValues nodes);
 in
   pkgs.writeShellScriptBin "agenix-generate" ''
     set -euo pipefail
@@ -234,10 +242,9 @@ in
     (
       REMOVED_ORPHANS=0
       shopt -s nullglob
-      for f in ${pkgs.lib.concatMapStrings (
-      x:
-        escapeShellArg (relativeToFlake x.config.age.rekey.generatedSecretsDir) + "/* "
-    ) (attrValues nodes)}; do
+      for f in ${pkgs.lib.concatMapStringsSep " "
+      (x: escapeShellArg (relativeToFlake x.config.age.rekey.generatedSecretsDir) + "/*")
+      nodesWithGeneratedSecretsDir}; do
         if [[ "''${KNOWN_SECRETS_SET["$f"]-false}" == false ]]; then
           rm -- "$f" || true
           REMOVED_ORPHANS=$((REMOVED_ORPHANS + 1))
@@ -247,10 +254,9 @@ in
         echo "     Removed ''${REMOVED_ORPHANS} orphaned files in generation directories"
 
         if [[ "$ADD_TO_GIT" == true ]]; then
-          git add ${pkgs.lib.concatMapStrings (
-      x:
-        escapeShellArg (relativeToFlake x.config.age.rekey.generatedSecretsDir) + " "
-    ) (attrValues nodes)}
+          git add ${pkgs.lib.concatMapStringsSep " "
+      (x: escapeShellArg (relativeToFlake x.config.age.rekey.generatedSecretsDir))
+      nodesWithGeneratedSecretsDir}
         fi
       fi
     )