Merge pull request #113 from helsinki-systems/feat/restart-and-reload

Add actual support for restarting/reloading units, change detection, dry mode

README.md

@@ -555,6 +555,20 @@ the service needs a token and a ssh private key to function:
 }
 ```
 
+## Restarting/reloading systemd units
+
+**With NixOS 21.11**, it is possible to restart or reload units when a secret changes or is newly initialized.
+This behaviour can be configured per-secret:
+```nix
+{
+  sops.secrets."home-assistant-secrets.yaml" = {
+    restartUnits = [ "home-assistant.service" ];
+  };
+}
+```
+
+This logic respects units that prefer to be reloaded or not to be restarted at all.
+
 ## Symlinks to other directories
 
 Some services might expect files in certain locations.
@@ -813,11 +827,6 @@ You can also check out [nix-community infrastructure repository](https://github.
 
 ## Known limitations
 
-### Restarting systemd services
-
-Right now systemd services are not restarted automatically.
-We want to implement this in future.
-
 ### Initrd secrets
 
 sops-nix does not fully support initrd secrets.

modules/sops/default.nix

@@ -78,6 +78,15 @@ let
           Hash of the sops file, useful in <xref linkend="opt-systemd.services._name_.restartTriggers" />.
         '';
       };
+      restartUnits = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "sshd.service" ];
+        description = ''
+          Names of units that should be restarted when this secret changes.
+          This works the same way as <xref linkend="opt-systemd.services._name_.restartTriggers" />.
+        '';
+      };
     };
   });
   manifest = pkgs.writeText "manifest.json" (builtins.toJSON {
@@ -89,6 +98,10 @@ let
     sshKeyPaths = cfg.gnupg.sshKeyPaths;
     ageKeyFile = cfg.age.keyFile;
     ageSshKeyPaths = cfg.age.sshKeyPaths;
+    logging = {
+      keyImport = builtins.elem "keyImport" cfg.log;
+      secretChanges = builtins.elem "secretChanges" cfg.log;
+    };
   });
 
   checkedManifest = let
@@ -133,6 +146,12 @@ in {
       '';
     };
 
+    log = mkOption {
+      type = types.listOf (types.enum [ "keyImport" "secretChanges" ]);
+      default = [ "keyImport" "secretChanges" ];
+      description = "What to log";
+    };
+
     age = {
       keyFile = mkOption {
         type = types.nullOr types.path;
@@ -209,10 +228,12 @@ in {
 
     system.activationScripts.setup-secrets = let
       sops-install-secrets = (pkgs.callPackage ../.. {}).sops-install-secrets;
-    in stringAfter ([ "specialfs" "users" "groups" ] ++ optional cfg.age.generateKey "generate-age-key") ''
+    in (stringAfter ([ "specialfs" "users" "groups" ] ++ optional cfg.age.generateKey "generate-age-key") ''
-      echo setting up secrets...
+      [ -e /run/current-system ] || echo setting up secrets...
       ${optionalString (cfg.gnupg.home != null) "SOPS_GPG_EXEC=${pkgs.gnupg}/bin/gpg"} ${sops-install-secrets}/bin/sops-install-secrets ${checkedManifest}
-    '';
+    '') // lib.optionalAttrs (config.system ? dryActivationScript) {
+      supportsDryActivation = true;
+    };
 
     system.activationScripts.generate-age-key = (mkIf cfg.age.generateKey) (stringAfter [] ''
       if [[ ! -f '${cfg.age.keyFile}' ]]; then

pkgs/sops-install-secrets/main.go

@@ -3,6 +3,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/hex"
 	"encoding/json"
 	"flag"
@@ -26,30 +27,35 @@ import (
 )
 
 type secret struct {
-	Name            string     `json:"name"`
+	Name         string     `json:"name"`
-	Key             string     `json:"key"`
+	Key          string     `json:"key"`
-	Path            string     `json:"path"`
+	Path         string     `json:"path"`
-	Owner           string     `json:"owner"`
+	Owner        string     `json:"owner"`
-	Group           string     `json:"group"`
+	Group        string     `json:"group"`
-	SopsFile        string     `json:"sopsFile"`
+	SopsFile     string     `json:"sopsFile"`
-	Format          FormatType `json:"format"`
+	Format       FormatType `json:"format"`
-	Mode            string     `json:"mode"`
+	Mode         string     `json:"mode"`
-	RestartServices []string   `json:"restartServices"`
+	RestartUnits []string   `json:"restartUnits"`
-	ReloadServices  []string   `json:"reloadServices"`
+	value        []byte
-	value           []byte
+	mode         os.FileMode
-	mode            os.FileMode
+	owner        int
-	owner           int
+	group        int
-	group           int
+}
+
+type loggingConfig struct {
+	KeyImport     bool `json:"keyImport"`
+	SecretChanges bool `json:"secretChanges"`
 }
 
 type manifest struct {
-	Secrets           []secret `json:"secrets"`
+	Secrets           []secret      `json:"secrets"`
-	SecretsMountPoint string   `json:"secretsMountpoint"`
+	SecretsMountPoint string        `json:"secretsMountpoint"`
-	SymlinkPath       string   `json:"symlinkPath"`
+	SymlinkPath       string        `json:"symlinkPath"`
-	SSHKeyPaths       []string `json:"sshKeyPaths"`
+	SSHKeyPaths       []string      `json:"sshKeyPaths"`
-	GnupgHome         string   `json:"gnupgHome"`
+	GnupgHome         string        `json:"gnupgHome"`
-	AgeKeyFile        string   `json:"ageKeyFile"`
+	AgeKeyFile        string        `json:"ageKeyFile"`
-	AgeSshKeyPaths    []string `json:"ageSshKeyPaths"`
+	AgeSshKeyPaths    []string      `json:"ageSshKeyPaths"`
+	Logging           loggingConfig `json:"logging"`
 }
 
 type secretFile struct {
@@ -549,7 +555,7 @@ func atomicSymlink(oldname, newname string) error {
 	return os.RemoveAll(d)
 }
 
-func importSSHKeys(keyPaths []string, gpgHome string) error {
+func importSSHKeys(logcfg loggingConfig, keyPaths []string, gpgHome string) error {
 	secringPath := filepath.Join(gpgHome, "secring.gpg")
 
 	secring, err := os.OpenFile(secringPath, os.O_WRONLY|os.O_CREATE, 0600)
@@ -570,7 +576,9 @@ func importSSHKeys(keyPaths []string, gpgHome string) error {
 			return fmt.Errorf("Cannot write secring: %w", err)
 		}
 
-		fmt.Printf("%s: Imported %s with fingerprint %s\n", path.Base(os.Args[0]), p, hex.EncodeToString(gpgKey.PrimaryKey.Fingerprint[:]))
+		if logcfg.KeyImport {
+			fmt.Printf("%s: Imported %s with fingerprint %s\n", path.Base(os.Args[0]), p, hex.EncodeToString(gpgKey.PrimaryKey.Fingerprint[:]))
+		}
 	}
 
 	return nil
@@ -598,6 +606,157 @@ func importAgeSSHKeys(keyPaths []string, ageFile os.File) error {
 	return nil
 }
 
+// Like filepath.Walk but symlink-aware.
+// Inspired by https://github.com/facebookarchive/symwalk
+func symlinkWalk(filename string, linkDirname string, walkFn filepath.WalkFunc) error {
+	symWalkFunc := func(path string, info os.FileInfo, err error) error {
+
+		if fname, err := filepath.Rel(filename, path); err == nil {
+			path = filepath.Join(linkDirname, fname)
+		} else {
+			return err
+		}
+
+		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
+			finalPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				return err
+			}
+			info, err := os.Lstat(finalPath)
+			if err != nil {
+				return walkFn(path, info, err)
+			}
+			if info.IsDir() {
+				return symlinkWalk(finalPath, path, walkFn)
+			}
+		}
+
+		return walkFn(path, info, err)
+	}
+	return filepath.Walk(filename, symWalkFunc)
+}
+
+func handleModifications(isDry bool, logcfg loggingConfig, symlinkPath string, secretDir string, secrets []secret) error {
+	var restart []string
+
+	newSecrets := make(map[string]bool)
+	modifiedSecrets := make(map[string]bool)
+	removedSecrets := make(map[string]bool)
+
+	// When the symlink path does not exist yet, we are being run in stage-2-init.sh
+	// where switch-to-configuration is not run so the services would only be restarted
+	// the next time switch-to-configuration is run.
+	if _, err := os.Stat(symlinkPath); os.IsNotExist(err) {
+		return nil
+	}
+
+	// Find modified/new secrets
+	for _, secret := range secrets {
+		oldPath := filepath.Join(symlinkPath, secret.Name)
+		newPath := filepath.Join(secretDir, secret.Name)
+
+		// Read the old file
+		oldData, err := ioutil.ReadFile(oldPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				// File did not exist before
+				restart = append(restart, secret.RestartUnits...)
+				newSecrets[secret.Name] = true
+				continue
+			}
+			return err
+		}
+
+		// Read the new file
+		newData, err := ioutil.ReadFile(newPath)
+		if err != nil {
+			return err
+		}
+
+		if !bytes.Equal(oldData, newData) {
+			restart = append(restart, secret.RestartUnits...)
+			modifiedSecrets[secret.Name] = true
+		}
+	}
+
+	writeLines := func(list []string, file string) error {
+		if len(list) != 0 {
+			f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+			for _, unit := range list {
+				if _, err = f.WriteString(unit + "\n"); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+	var dryPrefix string
+	if isDry {
+		dryPrefix = "/run/nixos/dry-activation"
+	} else {
+		dryPrefix = "/run/nixos/activation"
+	}
+	if err := writeLines(restart, dryPrefix+"-restart-list"); err != nil {
+		return err
+	}
+
+	// Do not output changes if not requested
+	if !logcfg.SecretChanges {
+		return nil
+	}
+
+	// Find removed secrets
+	err := symlinkWalk(symlinkPath, symlinkPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		path = strings.TrimPrefix(path, symlinkPath+string(os.PathSeparator))
+		for _, secret := range secrets {
+			if secret.Name == path {
+				return nil
+			}
+		}
+		removedSecrets[path] = true
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Output new/modified/removed secrets
+	outputChanged := func(changed map[string]bool, regularPrefix, dryPrefix string) {
+		if len(changed) > 0 {
+			s := ""
+			if len(changed) != 1 {
+				s = "s"
+			}
+			if isDry {
+				fmt.Printf("%s secret%s: ", dryPrefix, s)
+			} else {
+				fmt.Printf("%s secret%s: ", regularPrefix, s)
+			}
+			comma := ""
+			for name := range changed {
+				fmt.Printf("%s%s", comma, name)
+				comma = ", "
+			}
+			fmt.Println()
+		}
+	}
+	outputChanged(newSecrets, "adding", "would add")
+	outputChanged(modifiedSecrets, "modifying", "would modify")
+	outputChanged(removedSecrets, "removing", "would remove")
+
+	return nil
+}
+
 type keyring struct {
 	path string
 }
@@ -607,14 +766,14 @@ func (k *keyring) Remove() {
 	os.Unsetenv("GNUPGHOME")
 }
 
-func setupGPGKeyring(sshKeys []string, parentDir string) (*keyring, error) {
+func setupGPGKeyring(logcfg loggingConfig, sshKeys []string, parentDir string) (*keyring, error) {
 	dir, err := ioutil.TempDir(parentDir, "gpg")
 	if err != nil {
 		return nil, fmt.Errorf("Cannot create gpg home in '%s': %s", parentDir, err)
 	}
 	k := keyring{dir}
 
-	if err := importSSHKeys(sshKeys, dir); err != nil {
+	if err := importSSHKeys(logcfg, sshKeys, dir); err != nil {
 		os.RemoveAll(dir)
 		return nil, err
 	}
@@ -681,12 +840,14 @@ func installSecrets(args []string) error {
 		return err
 	}
 
+	isDry := os.Getenv("NIXOS_ACTION") == "dry-activate"
+
 	if err := mountSecretFs(manifest.SecretsMountPoint, keysGid); err != nil {
 		return fmt.Errorf("Failed to mount filesystem for secrets: %w", err)
 	}
 
 	if len(manifest.SSHKeyPaths) != 0 {
-		keyring, err := setupGPGKeyring(manifest.SSHKeyPaths, manifest.SecretsMountPoint)
+		keyring, err := setupGPGKeyring(manifest.Logging, manifest.SSHKeyPaths, manifest.SecretsMountPoint)
 		if err != nil {
 			return fmt.Errorf("Error setting up gpg keyring: %w", err)
 		}
@@ -740,6 +901,13 @@ func installSecrets(args []string) error {
 	if err := writeSecrets(*secretDir, manifest.Secrets, keysGid); err != nil {
 		return fmt.Errorf("Cannot write secrets: %w", err)
 	}
+	if err := handleModifications(isDry, manifest.Logging, manifest.SymlinkPath, *secretDir, manifest.Secrets); err != nil {
+		return fmt.Errorf("Cannot request units to restart: %w", err)
+	}
+	// No need to perform the actual symlinking
+	if isDry {
+		return nil
+	}
 	if err := symlinkSecrets(manifest.SymlinkPath, manifest.Secrets); err != nil {
 		return fmt.Errorf("Failed to prepare symlinks to secret store: %w", err)
 	}

pkgs/sops-install-secrets/main_test.go

@@ -99,15 +99,14 @@ func testGPG(t *testing.T) {
 
 	// should create a symlink
 	yamlSecret := secret{
-		Name:            "test",
+		Name:         "test",
-		Key:             "test_key",
+		Key:          "test_key",
-		Owner:           "nobody",
+		Owner:        "nobody",
-		Group:           "nogroup",
+		Group:        "nogroup",
-		SopsFile:        path.Join(assets, "secrets.yaml"),
+		SopsFile:     path.Join(assets, "secrets.yaml"),
-		Path:            path.Join(testdir.path, "test-target"),
+		Path:         path.Join(testdir.path, "test-target"),
-		Mode:            "0400",
+		Mode:         "0400",
-		RestartServices: []string{"affected-service"},
+		RestartUnits: []string{"affected-service"},
-		ReloadServices:  make([]string, 0),
 	}
 
 	var jsonSecret, binarySecret secret
@@ -198,15 +197,14 @@ func testSSHKey(t *testing.T) {
 	file.Close()
 
 	s := secret{
-		Name:            "test",
+		Name:         "test",
-		Key:             "test_key",
+		Key:          "test_key",
-		Owner:           "nobody",
+		Owner:        "nobody",
-		Group:           "nogroup",
+		Group:        "nogroup",
-		SopsFile:        path.Join(assets, "secrets.yaml"),
+		SopsFile:     path.Join(assets, "secrets.yaml"),
-		Path:            target,
+		Path:         target,
-		Mode:            "0400",
+		Mode:         "0400",
-		RestartServices: []string{"affected-service"},
+		RestartUnits: []string{"affected-service"},
-		ReloadServices:  make([]string, 0),
 	}
 
 	m := manifest{
@@ -231,15 +229,14 @@ func TestAge(t *testing.T) {
 	file.Close()
 
 	s := secret{
-		Name:            "test",
+		Name:         "test",
-		Key:             "test_key",
+		Key:          "test_key",
-		Owner:           "nobody",
+		Owner:        "nobody",
-		Group:           "nogroup",
+		Group:        "nogroup",
-		SopsFile:        path.Join(assets, "secrets.yaml"),
+		SopsFile:     path.Join(assets, "secrets.yaml"),
-		Path:            target,
+		Path:         target,
-		Mode:            "0400",
+		Mode:         "0400",
-		RestartServices: []string{"affected-service"},
+		RestartUnits: []string{"affected-service"},
-		ReloadServices:  make([]string, 0),
 	}
 
 	m := manifest{
@@ -264,15 +261,14 @@ func TestAgeWithSSH(t *testing.T) {
 	file.Close()
 
 	s := secret{
-		Name:            "test",
+		Name:         "test",
-		Key:             "test_key",
+		Key:          "test_key",
-		Owner:           "nobody",
+		Owner:        "nobody",
-		Group:           "nogroup",
+		Group:        "nogroup",
-		SopsFile:        path.Join(assets, "secrets.yaml"),
+		SopsFile:     path.Join(assets, "secrets.yaml"),
-		Path:            target,
+		Path:         target,
-		Mode:            "0400",
+		Mode:         "0400",
-		RestartServices: []string{"affected-service"},
+		RestartUnits: []string{"affected-service"},
-		ReloadServices:  make([]string, 0),
 	}
 
 	m := manifest{
@@ -298,15 +294,14 @@ func TestValidateManifest(t *testing.T) {
 	defer testdir.Remove()
 
 	s := secret{
-		Name:            "test",
+		Name:         "test",
-		Key:             "test_key",
+		Key:          "test_key",
-		Owner:           "nobody",
+		Owner:        "nobody",
-		Group:           "nogroup",
+		Group:        "nogroup",
-		SopsFile:        path.Join(assets, "secrets.yaml"),
+		SopsFile:     path.Join(assets, "secrets.yaml"),
-		Path:            path.Join(testdir.path, "test-target"),
+		Path:         path.Join(testdir.path, "test-target"),
-		Mode:            "0400",
+		Mode:         "0400",
-		RestartServices: []string{},
+		RestartUnits: []string{},
-		ReloadServices:  make([]string, 0),
 	}
 
 	m := manifest{

pkgs/sops-install-secrets/nixos-test.nix

@@ -1,4 +1,4 @@
-{ makeTest ? import <nixpkgs/nixos/tests/make-test-python.nix>, pkgs ? import <nixpkgs> }:
+{ makeTest ? import <nixpkgs/nixos/tests/make-test-python.nix>, pkgs ? (import <nixpkgs> {}) }:
 {
  ssh-keys = makeTest {
    name = "sops-ssh-keys";
@@ -129,4 +129,110 @@
    inherit pkgs;
    inherit (pkgs) system;
  };
+
+} // pkgs.lib.optionalAttrs (pkgs.lib.versionAtLeast (pkgs.lib.versions.majorMinor pkgs.lib.version) "21.11") {
+
+  restart-and-reload = makeTest {
+    name = "sops-restart-and-reload";
+    machine = { pkgs, lib, config, ... }: {
+      imports = [
+        ../../modules/sops
+      ];
+
+      sops = {
+        age.keyFile = ./test-assets/age-keys.txt;
+        defaultSopsFile = ./test-assets/secrets.yaml;
+        secrets.test_key = {
+          restartUnits = [ "restart-unit.service" "reload-unit.service" ];
+        };
+      };
+
+      systemd.services."restart-unit" = {
+        description = "Restart unit";
+        # not started on boot
+        serviceConfig = {
+          ExecStart = "/bin/sh -c 'echo ok > /restarted'";
+        };
+      };
+      systemd.services."reload-unit" = {
+        description = "Restart unit";
+        wantedBy = [ "multi-user.target" ];
+        reloadIfChanged = true;
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          ExecStart = "/bin/sh -c true";
+          ExecReload = "/bin/sh -c 'echo ok > /reloaded'";
+        };
+      };
+   };
+   testScript = ''
+     machine.wait_for_unit("multi-user.target")
+     machine.fail("test -f /restarted")
+     machine.fail("test -f /reloaded")
+
+     # Nothing is to be restarted after boot
+     machine.fail("ls /run/nixos/*-list")
+
+     # Nothing happens when the secret is not changed
+     machine.succeed("/run/current-system/bin/switch-to-configuration test")
+     machine.fail("test -f /restarted")
+     machine.fail("test -f /reloaded")
+
+     # Ensure the secret is changed
+     machine.succeed(": > /run/secrets/test_key")
+
+     # The secret is changed, now something should happen
+     machine.succeed("/run/current-system/bin/switch-to-configuration test")
+
+     # Ensure something happened
+     machine.succeed("test -f /restarted")
+     machine.succeed("test -f /reloaded")
+
+     with subtest("change detection"):
+        machine.succeed("rm /run/secrets/test_key")
+        out = machine.succeed("/run/current-system/bin/switch-to-configuration test")
+        if "adding secret" not in out:
+            raise Exception("Addition detection does not work")
+
+        machine.succeed(": > /run/secrets/test_key")
+        out = machine.succeed("/run/current-system/bin/switch-to-configuration test")
+        if "modifying secret" not in out:
+            raise Exception("Modification detection does not work")
+
+        machine.succeed(": > /run/secrets/another_key")
+        out = machine.succeed("/run/current-system/bin/switch-to-configuration test")
+        if "removing secret" not in out:
+            raise Exception("Removal detection does not work")
+
+     with subtest("dry activation"):
+         machine.succeed("rm /run/secrets/test_key")
+         machine.succeed(": > /run/secrets/another_key")
+         out = machine.succeed("/run/current-system/bin/switch-to-configuration dry-activate")
+         if "would add secret" not in out:
+             raise Exception("Dry addition detection does not work")
+         if "would remove secret" not in out:
+             raise Exception("Dry removal detection does not work")
+
+         machine.fail("test -f /run/secrets/test_key")
+         machine.succeed("test -f /run/secrets/another_key")
+
+         machine.succeed("/run/current-system/bin/switch-to-configuration test")
+         machine.succeed("test -f /run/secrets/test_key")
+         machine.succeed("rm /restarted /reloaded")
+         machine.fail("test -f /run/secrets/another_key")
+
+         machine.succeed(": > /run/secrets/test_key")
+         out = machine.succeed("/run/current-system/bin/switch-to-configuration dry-activate")
+         if "would modify secret" not in out:
+             raise Exception("Dry modification detection does not work")
+         machine.succeed("[ $(cat /run/secrets/test_key | wc -c) = 0 ]")
+
+         machine.fail("test -f /restarted")  # not done in dry mode
+         machine.fail("test -f /reloaded")  # not done in dry mode
+   '';
+  } {
+    inherit pkgs;
+    inherit (pkgs) system;
+  };
 }