Merge pull request #586 from hercules-ci/add-hercules-ci-agent

Add `services.hercules-ci-agent`

README.md

@@ -225,6 +225,19 @@ information on the `-I` flag look at the nix-build [manpage](https://nixos.org/m
 darwin-rebuild switch -I darwin=.
 ```
 
+If you're adding a module, please add yourself to `meta.maintainers`, for example
+
+```nix
+  meta.maintainers = [
+    lib.maintainers.alice or "alice"
+  ];
+
+  options.services.alicebot = # ...
+```
+
+The `or` operator takes care of graceful degradation when `lib` from Nixpkgs
+goes out of sync.
+
 Also feel free to contact me if you have questions,
 - Matrix - @daiderd:matrix.org, you can find me in [#macos:nixos.org](https://matrix.to/#/#macos:nixos.org)
 - @lnl7 on twitter

modules/meta.nix

@@ -0,0 +1,54 @@
+# This module was derived from
+# https://github.com/NixOS/nixpkgs/blob/000387627d26f245a6d9a0a7a60b7feddecaeec0/nixos/modules/misc/meta.nix
+{ lib, ... }:
+
+with lib;
+
+let
+  maintainer = mkOptionType {
+    name = "maintainer";
+    check = email: elem email (attrValues lib.maintainers);
+    merge = loc: defs: listToAttrs (singleton (nameValuePair (last defs).file (last defs).value));
+  };
+
+  listOfMaintainers = types.listOf maintainer // {
+    # Returns list of
+    #   { "module-file" = [
+    #        "maintainer1 <first@nixos.org>"
+    #        "maintainer2 <second@nixos.org>" ];
+    #   }
+    merge = loc: defs:
+      zipAttrs
+        (flatten (imap1 (n: def: imap1 (m: def':
+          maintainer.merge (loc ++ ["[${toString n}-${toString m}]"])
+            [{ inherit (def) file; value = def'; }]) def.value) defs));
+  };
+
+in
+
+{
+  options = {
+    meta = {
+
+      maintainers = mkOption {
+        type = listOfMaintainers;
+        internal = true;
+        default = [];
+        example = [ lib.maintainers.all ];
+        description = ''
+          List of maintainers of each module.  This option should be defined at
+          most once per module.
+
+          NOTE: <literal>lib</literal> comes from Nixpkgs, which can go out of
+          sync with nix-darwin. For this reason, use definitions like
+          <literal>maintainers.alice or "alice"</literal>.
+        '';
+      };
+
+    };
+  };
+
+  meta.maintainers = [
+    maintainers.roberth or "roberth"
+  ];
+}

modules/module-list.nix

@@ -1,6 +1,7 @@
 [
   ./alias.nix
   ./documentation
+  ./meta.nix
   ./misc/ids.nix
   ./misc/lib.nix
   ./security/pam.nix
@@ -49,6 +50,7 @@
   ./services/dnsmasq.nix
   ./services/emacs.nix
   ./services/gitlab-runner.nix
+  ./services/hercules-ci-agent
   ./services/karabiner-elements
   ./services/khd
   ./services/kwm

modules/services/hercules-ci-agent/common.nix

@@ -0,0 +1,99 @@
+/*
+
+  This file is for options that NixOS and nix-darwin have in common.
+
+  Platform-specific code is in the respective default.nix files.
+
+*/
+
+{ config, lib, options, pkgs, ... }:
+let
+  inherit (lib)
+    filterAttrs
+    literalExpression
+    mkIf
+    mkOption
+    mkRemovedOptionModule
+    mkRenamedOptionModule
+    types
+    ;
+  literalMD = lib.literalMD or (x: lib.literalDocBook "Documentation not rendered. Please upgrade to a newer NixOS with markdown support.");
+  mdDoc = lib.mdDoc or (x: "Documentation not rendered. Please upgrade to a newer NixOS with markdown support.");
+
+  cfg = config.services.hercules-ci-agent;
+
+  inherit (import ./settings.nix { inherit pkgs lib; }) format settingsModule;
+
+in
+{
+  imports = [
+    (mkRenamedOptionModule [ "services" "hercules-ci-agent" "extraOptions" ] [ "services" "hercules-ci-agent" "settings" ])
+    (mkRenamedOptionModule [ "services" "hercules-ci-agent" "baseDirectory" ] [ "services" "hercules-ci-agent" "settings" "baseDirectory" ])
+    (mkRenamedOptionModule [ "services" "hercules-ci-agent" "concurrentTasks" ] [ "services" "hercules-ci-agent" "settings" "concurrentTasks" ])
+    (mkRemovedOptionModule [ "services" "hercules-ci-agent" "patchNix" ] "Nix versions packaged in this version of Nixpkgs don't need a patched nix-daemon to work correctly in Hercules CI Agent clusters.")
+  ];
+
+  options.services.hercules-ci-agent = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = mdDoc ''
+        Enable to run Hercules CI Agent as a system service.
+
+        [Hercules CI](https://hercules-ci.com) is a
+        continuous integation service that is centered around Nix.
+
+        Support is available at [help@hercules-ci.com](mailto:help@hercules-ci.com).
+      '';
+    };
+    package = mkOption {
+      description = mdDoc ''
+        Package containing the bin/hercules-ci-agent executable.
+      '';
+      type = types.package;
+      default = pkgs.hercules-ci-agent;
+      defaultText = literalExpression "pkgs.hercules-ci-agent";
+    };
+    settings = mkOption {
+      description = mdDoc ''
+        These settings are written to the `agent.toml` file.
+
+        Not all settings are listed as options, can be set nonetheless.
+
+        For the exhaustive list of settings, see <https://docs.hercules-ci.com/hercules-ci/reference/agent-config/>.
+      '';
+      type = types.submoduleWith { modules = [ settingsModule ]; };
+    };
+
+    /*
+      Internal and/or computed values.
+
+      These are written as options instead of let binding to allow sharing with
+      default.nix on both NixOS and nix-darwin.
+    */
+    tomlFile = mkOption {
+      type = types.path;
+      internal = true;
+      defaultText = literalMD "generated `hercules-ci-agent.toml`";
+      description = mdDoc ''
+        The fully assembled config file.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    nix.extraOptions = ''
+      # A store path that was missing at first may well have finished building,
+      # even shortly after the previous lookup. This *also* applies to the daemon.
+      narinfo-cache-negative-ttl = 0
+    '';
+    services.hercules-ci-agent = {
+      tomlFile =
+        format.generate "hercules-ci-agent.toml" cfg.settings;
+      settings.config._module.args = {
+        packageOption = options.services.hercules-ci-agent.package;
+        inherit pkgs;
+      };
+    };
+  };
+}

modules/services/hercules-ci-agent/default.nix

@@ -0,0 +1,81 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.hercules-ci-agent;
+  user = config.users.users._hercules-ci-agent;
+in
+{
+  imports = [ ./common.nix ];
+
+  meta.maintainers = [
+    lib.maintainers.roberth or "roberth"
+  ];
+
+  options.services.hercules-ci-agent = {
+
+    logFile = mkOption {
+      type = types.path;
+      default = "/var/log/hercules-ci-agent.log";
+      description = "Stdout and sterr of hercules-ci-agent process.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    launchd.daemons.hercules-ci-agent = {
+      script = "exec ${cfg.package}/bin/hercules-ci-agent --config ${cfg.tomlFile}";
+
+      path = [ config.nix.package ];
+      environment = {
+        NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+      };
+
+      serviceConfig.KeepAlive = true;
+      serviceConfig.RunAtLoad = true;
+      serviceConfig.StandardErrorPath = cfg.logFile;
+      serviceConfig.StandardOutPath = cfg.logFile;
+      serviceConfig.GroupName = "_hercules-ci-agent";
+      serviceConfig.UserName = "_hercules-ci-agent";
+      serviceConfig.WorkingDirectory = user.home;
+      serviceConfig.WatchPaths = [
+        cfg.settings.staticSecretsDirectory
+      ];
+    };
+
+    system.activationScripts.preActivation.text = ''
+      touch '${cfg.logFile}'
+      chown ${toString user.uid}:${toString user.gid} '${cfg.logFile}'
+    '';
+
+    # Trusted user allows simplified configuration and better performance
+    # when operating in a cluster.
+    nix.settings.trusted-users = [ "_hercules-ci-agent" ];
+    services.hercules-ci-agent.settings.nixUserIsTrusted = true;
+
+    users.knownGroups = [ "hercules-ci-agent" "_hercules-ci-agent" ];
+    users.knownUsers = [ "hercules-ci-agent" "_hercules-ci-agent" ];
+
+    users.users._hercules-ci-agent = {
+      uid = mkDefault 399;
+      gid = mkDefault config.users.groups._hercules-ci-agent.gid;
+      home = mkDefault cfg.settings.baseDirectory;
+      name = "_hercules-ci-agent";
+      createHome = true;
+      shell = "/bin/bash";
+      description = "System user for the Hercules CI Agent";
+    };
+    users.groups._hercules-ci-agent = {
+      gid = mkDefault 32001;
+      name = "_hercules-ci-agent";
+      description = "System group for the Hercules CI Agent";
+    };
+
+    services.hercules-ci-agent.settings.labels = {
+      darwin.label = config.system.darwinLabel;
+      darwin.revision = config.system.darwinRevision;
+      darwin.version = config.system.darwinVersion;
+      darwin.nix.daemon = config.nix.useDaemon;
+      darwin.nix.sandbox = config.nix.settings.sandbox;
+    };
+  };
+}

modules/services/hercules-ci-agent/settings.nix

@@ -0,0 +1,155 @@
+# Not a module
+{ pkgs, lib }:
+let
+  inherit (lib)
+    types
+    literalExpression
+    mkOption
+    ;
+  literalMD = lib.literalMD or (x: lib.literalDocBook "Documentation not rendered. Please upgrade to a newer NixOS with markdown support.");
+  mdDoc = lib.mdDoc or (x: "Documentation not rendered. Please upgrade to a newer NixOS with markdown support.");
+
+  format = pkgs.formats.toml { };
+
+  settingsModule = { config, packageOption, pkgs, ... }: {
+    freeformType = format.type;
+    options = {
+      apiBaseUrl = mkOption {
+        description = mdDoc ''
+          API base URL that the agent will connect to.
+
+          When using Hercules CI Enterprise, set this to the URL where your
+          Hercules CI server is reachable.
+        '';
+        type = types.str;
+        default = "https://hercules-ci.com";
+      };
+      baseDirectory = mkOption {
+        type = types.path;
+        default = "/var/lib/hercules-ci-agent";
+        description = mdDoc ''
+          State directory (secrets, work directory, etc) for agent
+        '';
+      };
+      concurrentTasks = mkOption {
+        description = mdDoc ''
+          Number of tasks to perform simultaneously.
+
+          A task is a single derivation build, an evaluation or an effect run.
+          At minimum, you need 2 concurrent tasks for `x86_64-linux`
+          in your cluster, to allow for import from derivation.
+
+          `concurrentTasks` can be around the CPU core count or lower if memory is
+          the bottleneck.
+
+          The optimal value depends on the resource consumption characteristics of your workload,
+          including memory usage and in-task parallelism. This is typically determined empirically.
+
+          When scaling, it is generally better to have a double-size machine than two machines,
+          because each split of resources causes inefficiencies; particularly with regards
+          to build latency because of extra downloads.
+        '';
+        type = types.either types.ints.positive (types.enum [ "auto" ]);
+        default = "auto";
+        defaultText = literalMD ''
+          `"auto"`, meaning equal to the number of CPU cores.
+        '';
+      };
+      labels = mkOption {
+        description = mdDoc ''
+          A key-value map of user data.
+
+          This data will be available to organization members in the dashboard and API.
+
+          The values can be of any TOML type that corresponds to a JSON type, but arrays
+          can not contain tables/objects due to limitations of the TOML library. Values
+          involving arrays of non-primitive types may not be representable currently.
+        '';
+        type = format.type;
+        defaultText = literalExpression ''
+          {
+            agent.source = "..."; # One of "nixpkgs", "flake", "override"
+            lib.version = "...";
+            pkgs.version = "...";
+          }
+        '';
+      };
+      workDirectory = mkOption {
+        description = mdDoc ''
+          The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation.
+        '';
+        type = types.path;
+        default = config.baseDirectory + "/work";
+        defaultText = literalExpression ''baseDirectory + "/work"'';
+      };
+      staticSecretsDirectory = mkOption {
+        description = mdDoc ''
+          This is the default directory to look for statically configured secrets like `cluster-join-token.key`.
+
+          See also `clusterJoinTokenPath` and `binaryCachesPath` for fine-grained configuration.
+        '';
+        type = types.path;
+        default = config.baseDirectory + "/secrets";
+        defaultText = literalExpression ''baseDirectory + "/secrets"'';
+      };
+      clusterJoinTokenPath = mkOption {
+        description = mdDoc ''
+          Location of the cluster-join-token.key file.
+
+          You can retrieve the contents of the file when creating a new agent via
+          <https://hercules-ci.com/dashboard>.
+
+          As this value is confidential, it should not be in the store, but
+          installed using other means, such as agenix, NixOps
+          `deployment.keys`, or manual installation.
+
+          The contents of the file are used for authentication between the agent and the API.
+        '';
+        type = types.path;
+        default = config.staticSecretsDirectory + "/cluster-join-token.key";
+        defaultText = literalExpression ''staticSecretsDirectory + "/cluster-join-token.key"'';
+      };
+      binaryCachesPath = mkOption {
+        description = mdDoc ''
+          Path to a JSON file containing binary cache secret keys.
+
+          As these values are confidential, they should not be in the store, but
+          copied over using other means, such as agenix, NixOps
+          `deployment.keys`, or manual installation.
+
+          The format is described on <https://docs.hercules-ci.com/hercules-ci-agent/binary-caches-json/>.
+        '';
+        type = types.path;
+        default = config.staticSecretsDirectory + "/binary-caches.json";
+        defaultText = literalExpression ''staticSecretsDirectory + "/binary-caches.json"'';
+      };
+      secretsJsonPath = mkOption {
+        description = mdDoc ''
+          Path to a JSON file containing secrets for effects.
+
+          As these values are confidential, they should not be in the store, but
+          copied over using other means, such as agenix, NixOps
+          `deployment.keys`, or manual installation.
+
+          The format is described on <https://docs.hercules-ci.com/hercules-ci-agent/secrets-json/>.
+        '';
+        type = types.path;
+        default = config.staticSecretsDirectory + "/secrets.json";
+        defaultText = literalExpression ''staticSecretsDirectory + "/secrets.json"'';
+      };
+    };
+    config = {
+      labels = {
+        agent.source =
+          if packageOption.highestPrio == (lib.modules.mkOptionDefault { }).priority
+          then "nixpkgs"
+          else lib.mkOptionDefault "override";
+        pkgs.version = pkgs.lib.version;
+        lib.version = lib.version;
+      };
+    };
+  };
+in
+{
+  inherit format settingsModule;
+}