Merge pull request #10 from Mic92/validate

README.md

@@ -17,11 +17,12 @@ key management APIs such as AWS KMS, GCP KMS, Azure Key Vault or Hashicorp's vau
 
 - Compatible with all NixOS deployment frameworks: [NixOps](https://github.com/NixOS/nixops), nixos-rebuild, [krops](https://github.com/krebs/krops/), [morph](https://github.com/DBCDK/morph), [nixus](https://github.com/Infinisil/nixus)
 - Version-control friendly: Since all files are encrypted they can directly committed to version control. The format is readable in diffs and there are also ways of showing [git diffs in cleartext](https://github.com/mozilla/sops#showing-diffs-in-cleartext-in-git)
-- CI friendly: Since nixops files can be added to the nix store as well without leaking secrets, machine definition can be build as a whole.
+- CI friendly: Since sops files can be added to the nix store as well without leaking secrets, machine definition can be build as a whole.
 - Atomic upgrades: New secrets are written to a new directory which replaces the old directory in an atomic step.
 - Rollback support: If sops files are added to Nix store, old secrets can be rolled back. This is optional.
 - Fast: Unlike solutions implemented by NixOps, krops and morph there is no extra step required to upload secrets
 - Different storage formats: Secrets can be stored in Yaml, JSON or binary.
+- Minimize configuration errors: sops files are checked against the configuration at evluation time.
 
 ## Usage example
 

modules/sops/default.nix

@@ -80,6 +80,13 @@ let
     symlinkPath = "/run/secrets";
     inherit (cfg) gnupgHome sshKeyPaths;
   });
+
+  checkedManifest = pkgs.runCommandNoCC "checked-manifest.json" {
+    nativeBuildInputs = [ sops-install-secrets ];
+  } ''
+    sops-install-secrets -check-mode=${if cfg.validateSopsFiles then "sopsfile" else "manifest"} ${manifest}
+    cp ${manifest} $out
+  '';
 in {
   options.sops = {
     secrets = mkOption {
@@ -97,6 +104,15 @@ in {
       '';
     };
 
+    validateSopsFiles = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Check all sops files at evaluation time.
+        This requires sops files to be added to the nix store.
+      '';
+    };
+
     gnupgHome = mkOption {
       type = types.nullOr types.str;
       default = null;
@@ -118,18 +134,22 @@ in {
     };
   };
   config = mkIf (cfg.secrets != {}) {
-
     assertions = [{
       assertion = cfg.gnupgHome != null -> cfg.sshKeyPaths == [];
-      message = "config.sops.gnupgHome and config.sops.sshKeyPaths are mutual exclusive";
+      message = "Configuration options sops.gnupgHome and sops.sshKeyPaths cannot be set both at the same time";
     } {
       assertion = cfg.gnupgHome == null -> cfg.sshKeyPaths != [];
-      message = "Either config.sops.sshKeyPaths and config.sops.gnupgHome must be set";
-    }];
+      message = "Either sops.sshKeyPaths and sops.gnupgHome must be set";
+    }] ++ map (name: let
+      inherit (cfg.secrets.${name}) sopsFile;
+    in {
+      assertion = cfg.validateSopsFiles -> builtins.isPath sopsFile;
+      message = "${sopsFile} is not in the nix store. Either add it to the nix store or set `sops.validateSopsFiles` to false";
+    }) (builtins.attrNames cfg.secrets);
 
     system.activationScripts.setup-secrets = stringAfter [ "users" "groups" ] ''
       echo setting up secrets...
-      ${optionalString (cfg.gnupgHome != null) "SOPS_GPG_EXEC=${pkgs.gnupg}/bin/gpg"} ${sops-install-secrets}/bin/sops-install-secrets ${manifest}
+      ${optionalString (cfg.gnupgHome != null) "SOPS_GPG_EXEC=${pkgs.gnupg}/bin/gpg"} ${sops-install-secrets}/bin/sops-install-secrets ${checkedManifest}
     '';
   };
 }

pkgs/sops-install-secrets/main.go

@@ -21,16 +21,16 @@ import (
 )
 
 type secret struct {
-	Name            string   `json:"name"`
-	Key             string   `json:"key"`
-	Path            string   `json:"path"`
-	Owner           string   `json:"owner"`
-	Group           string   `json:"group"`
-	SopsFile        string   `json:"sopsFile"`
-	Format          string   `json:"format"`
-	Mode            string   `json:"mode"`
-	RestartServices []string `json:"restartServices"`
-	ReloadServices  []string `json:"reloadServices"`
+	Name            string     `json:"name"`
+	Key             string     `json:"key"`
+	Path            string     `json:"path"`
+	Owner           string     `json:"owner"`
+	Group           string     `json:"group"`
+	SopsFile        string     `json:"sopsFile"`
+	Format          FormatType `json:"format"`
+	Mode            string     `json:"mode"`
+	RestartServices []string   `json:"restartServices"`
+	ReloadServices  []string   `json:"reloadServices"`
 	value           []byte
 	mode            os.FileMode
 	owner           int
@@ -45,6 +45,60 @@ type manifest struct {
 	GnupgHome         string   `json:"gnupgHome"`
 }
 
+type secretFile struct {							  
+	cipherText []byte
+	keys       map[string]interface{}
+	/// First secret that defined this secretFile, used for error messages
+	firstSecret *secret
+}
+
+type FormatType string
+
+const (
+	Yaml   FormatType = "yaml"
+	Json   FormatType = "json"
+	Binary FormatType = "binary"
+)
+
+func (f *FormatType) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	var t = FormatType(s)
+	switch t {
+	case "":
+		*f = Yaml
+	case Yaml, Json, Binary:
+		*f = t
+	}
+
+	return nil
+}
+
+func (f FormatType) MarshalJSON() ([]byte, error) {
+	return json.Marshal(string(f))
+}
+
+type CheckMode string
+
+const (
+	Manifest CheckMode = "manifest"
+	SopsFile CheckMode = "sopsfile"
+	Off      CheckMode = "off"
+)
+
+type options struct {
+	checkMode CheckMode
+	manifest  string
+}
+
+type appContext struct {
+	manifest    manifest
+	secretFiles map[string]secretFile
+	checkMode   CheckMode
+}
+
 func readManifest(path string) (*manifest, error) {
 	file, err := os.Open(path)
 	if err != nil {
@@ -103,14 +157,14 @@ type plainData struct {
 func decryptSecret(s *secret, sourceFiles map[string]plainData) error {
 	sourceFile := sourceFiles[s.SopsFile]
 	if sourceFile.data == nil || sourceFile.binary == nil {
-		plain, err := decrypt.File(s.SopsFile, s.Format)
+		plain, err := decrypt.File(s.SopsFile, string(s.Format))
 		if err != nil {
 			return fmt.Errorf("Failed to decrypt '%s': %s", s.SopsFile, err)
 		}
-		if s.Format == "binary" {
+		if s.Format == Binary {
 			sourceFile.binary = plain
 		} else {
-			if s.Format == "yaml" {
+			if s.Format == Yaml {
 				if err := yaml.Unmarshal(plain, &sourceFile.data); err != nil {
 					return fmt.Errorf("Cannot parse yaml of '%s': %s", s.SopsFile, err)
 				}
@@ -121,7 +175,7 @@ func decryptSecret(s *secret, sourceFiles map[string]plainData) error {
 			}
 		}
 	}
-	if s.Format == "binary" {
+	if s.Format == Binary {
 		s.value = sourceFile.binary
 	} else {
 		val, ok := sourceFile.data[s.Key]
@@ -215,7 +269,55 @@ func lookupKeysGroup() (int, error) {
 	return int(gid), nil
 }
 
-func validateSecret(secret *secret) error {
+func (app *appContext) loadSopsFile(s *secret) (*secretFile, error) {
+	if app.checkMode == Manifest {
+		return &secretFile{firstSecret: s}, nil
+	}
+
+	cipherText, err := ioutil.ReadFile(s.SopsFile)
+	if err != nil {
+		return nil, fmt.Errorf("Failed reading %s: %s", s.SopsFile, err)
+	}
+
+	var keys map[string]interface{}
+	if s.Format == Binary {
+		if err := json.Unmarshal(cipherText, &keys); err != nil {
+			return nil, fmt.Errorf("Cannot parse json of '%s': %s", s.SopsFile, err)
+		}
+		return &secretFile{cipherText: cipherText, firstSecret: s}, nil
+	}
+
+	if s.Format == Yaml {
+		if err := yaml.Unmarshal(cipherText, &keys); err != nil {
+			return nil, fmt.Errorf("Cannot parse yaml of '%s': %s", s.SopsFile, err)
+		}
+	} else if err := json.Unmarshal(cipherText, &keys); err != nil {
+		return nil, fmt.Errorf("Cannot parse json of '%s': %s", s.SopsFile, err)
+	}
+
+	return &secretFile{
+		cipherText:  cipherText,
+		keys:        keys,
+		firstSecret: s,
+	}, nil
+
+}
+
+func (app *appContext) validateSopsFile(s *secret, file *secretFile) error {
+	if file.firstSecret.Format != s.Format {
+		return fmt.Errorf("secret %s defined the format of %s as %s, but it was specified as %s in %s before",
+			s.Name, s.SopsFile, s.Format,
+			file.firstSecret.Format, file.firstSecret.Name)
+	}
+	if app.checkMode != Manifest && s.Format != Binary {
+		if _, ok := file.keys[s.Key]; !ok {
+			return fmt.Errorf("secret %s with the key %s not found in %s", s.Name, s.Key, s.SopsFile)
+		}
+	}
+	return nil
+}
+
+func (app *appContext) validateSecret(secret *secret) error {
 	mode, err := strconv.ParseUint(secret.Mode, 8, 16)
 	if err != nil {
 		return fmt.Errorf("Invalid number in mode: %d: %s", mode, err)
@@ -247,14 +349,24 @@ func validateSecret(secret *secret) error {
 	}
 
 	if secret.Format != "yaml" && secret.Format != "json" && secret.Format != "binary" {
-		return fmt.Errorf("Unsupported format %s for secret %s",
-			secret.Format, secret.Name)
+		return fmt.Errorf("Unsupported format %s for secret %s", secret.Format, secret.Name)
 	}
 
-	return nil
+	file, ok := app.secretFiles[secret.SopsFile]
+	if !ok {
+		maybeFile, err := app.loadSopsFile(secret)
+		if err != nil {
+			return err
+		}
+		app.secretFiles[secret.SopsFile] = *maybeFile
+		file = *maybeFile
+	}
+
+	return app.validateSopsFile(secret, &file)
 }
 
-func validateManifest(m *manifest) error {
+func (app *appContext) validateManifest() error {
+	m := &app.manifest
 	if m.SecretsMountPoint == "" {
 		m.SecretsMountPoint = "/run/secrets.d"
 	}
@@ -266,7 +378,7 @@ func validateManifest(m *manifest) error {
 			"Both options are mutual exclusive.")
 	}
 	for i := range m.Secrets {
-		if err := validateSecret(&m.Secrets[i]); err != nil {
+		if err := app.validateSecret(&m.Secrets[i]); err != nil {
 			return err
 		}
 	}
@@ -358,11 +470,6 @@ func setupGPGKeyring(sshKeys []string, parentDir string) (*keyring, error) {
 	return &k, nil
 }
 
-type options struct {
-	check    bool
-	manifest string
-}
-
 func parseFlags(args []string) (*options, error) {
 	var opts options
 	fs := flag.NewFlagSet(args[0], flag.ContinueOnError)
@@ -370,11 +477,19 @@ func parseFlags(args []string) (*options, error) {
 		fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [OPTION] manifest.json\n", args[0])
 		fs.PrintDefaults()
 	}
-	fs.BoolVar(&opts.check, "check", false, "Validate manifest instead installing it")
+	var checkMode string
+	fs.StringVar(&checkMode, "check-mode", "off", `Validate configuration without installing it (possible values: "manifest","sopsfile","off")`)
 	if err := fs.Parse(args[1:]); err != nil {
 		return nil, err
 	}
 
+	switch CheckMode(checkMode) {
+	case Manifest, SopsFile, Off:
+		opts.checkMode = CheckMode(checkMode)
+	default:
+		return nil, fmt.Errorf("Invalid value provided for -check-mode flag: %s", opts.checkMode)
+	}
+
 	if fs.NArg() != 1 {
 		flag.Usage()
 		return nil, flag.ErrHelp
@@ -394,11 +509,17 @@ func installSecrets(args []string) error {
 		return err
 	}
 
-	if err := validateManifest(manifest); err != nil {
+	app := appContext{
+		manifest:    *manifest,
+		checkMode:   opts.checkMode,
+		secretFiles: make(map[string]secretFile),
+	}
+
+	if err := app.validateManifest(); err != nil {
 		return fmt.Errorf("Manifest is not valid: %s", err)
 	}
 
-	if opts.check {
+	if app.checkMode != Off {
 		return nil
 	}
 

pkgs/sops-install-secrets/main_test.go

@@ -213,3 +213,34 @@ func TestAll(t *testing.T) {
 	testGPG(t)
 	testSSHKey(t)
 }
+
+func TestValidateManifest(t *testing.T) {
+	assets := testAssetPath()
+
+	testdir := newTestDir(t)
+	defer testdir.Remove()
+
+	s := secret{
+		Name:            "test",
+		Key:             "test_key",
+		Owner:           "nobody",
+		Group:           "nogroup",
+		SopsFile:        path.Join(assets, "secrets.yaml"),
+		Path:            path.Join(testdir.path, "test-target"),
+		Mode:            "0400",
+		RestartServices: []string{},
+		ReloadServices:  make([]string, 0),
+	}
+
+	m := manifest{
+		Secrets:           []secret{s},
+		SecretsMountPoint: testdir.secretsPath,
+		SymlinkPath:       testdir.symlinkPath,
+		SSHKeyPaths:       []string{"non-existing-key"},
+	}
+
+	path := writeManifest(t, testdir.path, &m)
+
+	ok(t, installSecrets([]string{"sops-install-secrets", "-check-mode=manifest", path}))
+	ok(t, installSecrets([]string{"sops-install-secrets", "-check-mode=sopsfile", path}))
+}