nixos/servnerr-3: enable zrepl

Signed-off-by: Matt Layher <mdlayher@gmail.com>

nixos/servnerr-3/configuration.nix

@@ -73,7 +73,12 @@ in {
 
   # Packages specific to this machine. The base package set is defined in
   # lib/system.nix.
-  environment.systemPackages = with pkgs; [ zfs ];
+  environment.systemPackages = with pkgs; [
+    zfs
+
+    # Use latest available zrepl.
+    unstable.zrepl
+  ];
 
   services = {
     apcupsd.enable = true;

nixos/servnerr-3/storage.nix

@@ -1,6 +1,72 @@
 { lib, pkgs, ... }:
 
-let secrets = import ./lib/secrets.nix;
+let
+  secrets = import ./lib/secrets.nix;
+  unstable = import <nixos-unstable-small> { };
+
+  # Make a local zrepl push job from primary to the target zpool.
+  pushLocal = (zpool: {
+    name = pushName zpool;
+    type = "push";
+
+    # Replicate all of primary locally.
+    filesystems."primary<" = true;
+    connect = {
+      type = "local";
+      listener_name = sinkName zpool;
+      client_identity = "local";
+    };
+
+    # Let zfs-auto-snapshot manage the snapshotting.
+    snapshotting.type = "manual";
+    pruning = {
+      # Keep all primary snapshots, zfs-auto-snapshot manages them.
+      keep_sender = [{
+        type = "regex";
+        regex = ".*";
+      }];
+      # Keep the last few snapshots for each dataset for disaster recovery.
+      keep_receiver = [{
+        type = "last_n";
+        # 6 ZFS datasets, 8 snapshots each.
+        count = 48;
+      }];
+    };
+  });
+
+  # Make a local zrepl sink job to the target zpool.
+  sinkLocal = (zpool: {
+    name = sinkName zpool;
+    type = "sink";
+    root_fs = "${zpool}";
+    recv.properties = {
+      # Inherit any encryption properties.
+      "inherit" = [ "encryption" "keyformat" "keylocation" ];
+
+      override = {
+        # Do not mount sink pools.
+        mountpoint = "none";
+
+        # Do not auto-snapshot sink pools.
+        "com.sun:auto-snapshot" = false;
+        "com.sun:auto-snapshot:frequent" = false;
+        "com.sun:auto-snapshot:hourly" = false;
+        "com.sun:auto-snapshot:daily" = false;
+        "com.sun:auto-snapshot:weekly" = false;
+        "com.sun:auto-snapshot:monthly" = false;
+      };
+    };
+    serve = {
+      type = "local";
+      listener_name = "sink_${zpool}";
+    };
+  });
+
+  # Generate the zrepl push job name for a target zpool.
+  pushName = (zpool: "primary_to_${zpool}");
+
+  # Generate the zrepl sink job name for a target zpool.
+  sinkName = (zpool: "sink_${zpool}");
 
 in {
   # ZFS filesystem mounts.
@@ -110,5 +176,52 @@ in {
         };
       };
     };
+
+    # Replicate ZFS pools using zrepl.
+    zrepl = {
+      enable = true;
+      settings = {
+        global.monitoring = [{
+          type = "prometheus";
+          listen = ":9811";
+        }];
+        jobs = [
+          # Replicate from primary pool to sinks.
+          (pushLocal "secondary")
+          (pushLocal "backup0")
+          (pushLocal "backup1")
+
+          # Local sink jobs for backups.
+          (sinkLocal "secondary")
+          (sinkLocal "backup0")
+          (sinkLocal "backup1")
+        ];
+      };
+    };
+  };
+
+  # Manual systemd unit and timer to trigger zrepl jobs. We use
+  # zfs-auto-snapshot integrated into NixOS instead of zrepl's built-in
+  # automatic snapshotting, so we have to signal replication manually.
+  #
+  # TODO(mdlayher): push upstream into services.zrepl configuration.
+  systemd = {
+    services.zrepl-signal-jobs = {
+      serviceConfig.Type = "oneshot";
+      path = with pkgs; [ unstable.zrepl ];
+      script = ''
+        zrepl signal wakeup ${pushName "secondary"}
+        zrepl signal wakeup ${pushName "backup0"}
+        zrepl signal wakeup ${pushName "backup1"}
+      '';
+    };
+    timers.zrepl-signal-jobs = {
+      wantedBy = [ "timers.target" ];
+      partOf = [ "zrepl-signal-jobs.service" ];
+      timerConfig = {
+        OnCalendar = "hourly";
+        Unit = "zrepl-signal-jobs.service";
+      };
+    };
   };
 }