From 33f18b404eb8ebf1485256fa832cb59c63d72dfa Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Thu, 7 Nov 2024 12:06:10 -0600 Subject: [PATCH] Rework `restart-and-reload` to assert more strictly on the activation output I've reworked the test to assert on the entire output. This allows us to detect unexpected output without having to write weird "i expect this random string to *not* show up assertions", which aren't great at preventing regressions. I did have to change the code under test a little bit to make it behavior deterministically (by sorting the files it outputs). tl;dr: this demonstrates but does not fix it. I will fix it in a subsequent commit. --- pkgs/sops-install-secrets/main.go | 14 ++- pkgs/sops-install-secrets/nixos-test.nix | 113 ++++++++++++++++------- 2 files changed, 89 insertions(+), 38 deletions(-) diff --git a/pkgs/sops-install-secrets/main.go b/pkgs/sops-install-secrets/main.go index fcdb264..307d4d9 100644 --- a/pkgs/sops-install-secrets/main.go +++ b/pkgs/sops-install-secrets/main.go @@ -11,6 +11,7 @@ import ( "os/user" "path" "path/filepath" + "sort" "strconv" "strings" "syscall" @@ -984,12 +985,15 @@ func handleModifications(isDry bool, logcfg loggingConfig, symlinkPath string, s } else { fmt.Printf("%s secret%s: ", regularPrefix, s) } - comma := "" - for name := range changed { - fmt.Printf("%s%s", comma, name) - comma = ", " + + // Sort the output for deterministic behavior. + keys := make([]string, 0, len(changed)) + for key := range changed { + keys = append(keys, key) } - fmt.Println() + sort.Strings(keys) + + fmt.Println(strings.Join(keys, ", ")) } } outputChanged(newSecrets, "adding", "would add") diff --git a/pkgs/sops-install-secrets/nixos-test.nix b/pkgs/sops-install-secrets/nixos-test.nix index f5b9ed6..2a36ffa 100644 --- a/pkgs/sops-install-secrets/nixos-test.nix +++ b/pkgs/sops-install-secrets/nixos-test.nix @@ -326,7 +326,7 @@ in { restart-and-reload = testers.runNixOSTest { name = "sops-restart-and-reload"; - nodes.machine = { + nodes.machine = {config, ...}: { imports = [ ../../modules/sops ]; sops = { @@ -336,6 +336,11 @@ in { restartUnits = [ "restart-unit.service" "reload-unit.service" ]; reloadUnits = [ "reload-trigger.service" ]; }; + + templates.test_template.content = '' + this is a template with + a secret: ${config.sops.placeholder.test_key} + ''; }; system.switch.enable = true; @@ -374,6 +379,19 @@ in { }; testScript = '' + def assertOutput(output, *expected_lines): + expected_lines = list(expected_lines) + + # Remove unrelated fluff that shows up in the output of `switch-to-configuration`. + prefix = "setting up /etc...\n" + if output.startswith(prefix): + output = output.removeprefix(prefix) + + actual_lines = output.splitlines(keepends=False) + + if actual_lines != expected_lines: + raise Exception(f"{actual_lines} != {expected_lines}") + machine.wait_for_unit("multi-user.target") machine.fail("test -f /restarted") machine.fail("test -f /reloaded") @@ -397,46 +415,75 @@ in { machine.succeed("test -f /reloaded") with subtest("change detection"): - machine.succeed("rm /run/secrets/test_key") - out = machine.succeed("/run/current-system/bin/switch-to-configuration test") - if "adding secret" not in out: - raise Exception("Addition detection does not work") + machine.succeed("rm /run/secrets/test_key") + machine.succeed("rm /run/secrets/rendered/test_template") + out = machine.succeed("/run/current-system/bin/switch-to-configuration test") + assertOutput( + out, + "adding secret: test_key", + ) - machine.succeed(": > /run/secrets/test_key") - out = machine.succeed("/run/current-system/bin/switch-to-configuration test") - if "modifying secret" not in out: - raise Exception("Modification detection does not work") + machine.succeed(": > /run/secrets/test_key") + machine.succeed(": > /run/secrets/rendered/test_template") + out = machine.succeed("/run/current-system/bin/switch-to-configuration test") + assertOutput( + out, + "modifying secret: test_key", + # This is wrong. TODO: fix https://github.com/Mic92/sops-nix/issues/652 + "removing secret: rendered/test_template", + ) - machine.succeed(": > /run/secrets/another_key") - out = machine.succeed("/run/current-system/bin/switch-to-configuration test") - if "removing secret" not in out: - raise Exception("Removal detection does not work") + machine.succeed(": > /run/secrets/another_key") + machine.succeed(": > /run/secrets/rendered/another_template") + out = machine.succeed("/run/current-system/bin/switch-to-configuration test") + assertOutput( + out, + # This is wrong. TODO: fix https://github.com/Mic92/sops-nix/issues/652 + "removing secrets: another_key, rendered/another_template, rendered/test_template", + ) with subtest("dry activation"): - machine.succeed("rm /run/secrets/test_key") - machine.succeed(": > /run/secrets/another_key") - out = machine.succeed("/run/current-system/bin/switch-to-configuration dry-activate") - if "would add secret" not in out: - raise Exception("Dry addition detection does not work") - if "would remove secret" not in out: - raise Exception("Dry removal detection does not work") + machine.succeed("rm /run/secrets/test_key") + machine.succeed("rm /run/secrets/rendered/test_template") + machine.succeed(": > /run/secrets/another_key") + machine.succeed(": > /run/secrets/rendered/another_template") + out = machine.succeed("/run/current-system/bin/switch-to-configuration dry-activate") + assertOutput( + out, + "would add secret: test_key", + # This is wrong. TODO: fix https://github.com/Mic92/sops-nix/issues/652 + "would remove secrets: another_key, rendered/another_template", + ) - machine.fail("test -f /run/secrets/test_key") - machine.succeed("test -f /run/secrets/another_key") + # Verify that we did not actually activate the new configuration. + machine.fail("test -f /run/secrets/test_key") + machine.fail("test -f /run/secrets/rendered/test_template") + machine.succeed("test -f /run/secrets/another_key") + machine.succeed("test -f /run/secrets/rendered/another_template") - machine.succeed("/run/current-system/bin/switch-to-configuration test") - machine.succeed("test -f /run/secrets/test_key") - machine.succeed("rm /restarted /reloaded") - machine.fail("test -f /run/secrets/another_key") + # Now actually activate and sanity check the resulting secrets. + machine.succeed("/run/current-system/bin/switch-to-configuration test") + machine.succeed("test -f /run/secrets/test_key") + machine.succeed("test -f /run/secrets/rendered/test_template") + machine.fail("test -f /run/secrets/another_key") + machine.fail("test -f /run/secrets/rendered/another_template") - machine.succeed(": > /run/secrets/test_key") - out = machine.succeed("/run/current-system/bin/switch-to-configuration dry-activate") - if "would modify secret" not in out: - raise Exception("Dry modification detection does not work") - machine.succeed("[ $(cat /run/secrets/test_key | wc -c) = 0 ]") + # Remove the restarted/reloaded indicators so we can confirm a + # dry-activate doesn't trigger systemd units. + machine.succeed("rm /restarted /reloaded") - machine.fail("test -f /restarted") # not done in dry mode - machine.fail("test -f /reloaded") # not done in dry mode + machine.succeed(": > /run/secrets/test_key") + out = machine.succeed("/run/current-system/bin/switch-to-configuration dry-activate") + assertOutput( + out, + "would modify secret: test_key", + # This is wrong. TODO: fix https://github.com/Mic92/sops-nix/issues/652 + "would remove secret: rendered/test_template", + ) + machine.succeed("[ $(cat /run/secrets/test_key | wc -c) = 0 ]") + + machine.fail("test -f /restarted") # not done in dry mode + machine.fail("test -f /reloaded") # not done in dry mode ''; };