From fcb7c6067f42fa76d2a7cfaba2df8376b354c7ff Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 01/15] flake: Bump nixpkgs-stable to 24.05 --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index e72975b..437ecf1 100644 --- a/flake.lock +++ b/flake.lock @@ -72,16 +72,16 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1720535198, - "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=", + "lastModified": 1724316499, + "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5", + "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index be10223..708eece 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-23.11"; + nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05"; flake-utils.url = "github:numtide/flake-utils"; crane = { From 7712ed2c7f85dc4c632ba38554d710bdf6c5953c Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 02/15] flake: Bump unstable --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 437ecf1..e38f188 100644 --- a/flake.lock +++ b/flake.lock @@ -56,11 +56,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723827930, - "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=", + "lastModified": 1724999960, + "narHash": "sha256-LB3jqSGW5u1ZcUcX6vO/qBOq5oXHlmOCxsTXGMEitp4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d", + "rev": "b96f849e725333eb2b1c7f1cb84ff102062468ba", "type": "github" }, "original": { From fd6e3bdbefbc014eb4eb68f213269b298e118f6c Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 03/15] Add CI-agnostic helper scripts --- .ci/.gitignore | 1 + .ci/cache-shell.sh | 5 +++++ .ci/common.sh | 7 +++++++ .ci/run | 10 ++++++++++ 4 files changed, 23 insertions(+) create mode 100644 .ci/.gitignore create mode 100755 .ci/cache-shell.sh create mode 100644 .ci/common.sh create mode 100755 .ci/run diff --git a/.ci/.gitignore b/.ci/.gitignore new file mode 100644 index 0000000..1fd44ac --- /dev/null +++ b/.ci/.gitignore @@ -0,0 +1 @@ +/cached-shell diff --git a/.ci/cache-shell.sh b/.ci/cache-shell.sh new file mode 100755 index 0000000..b810324 --- /dev/null +++ b/.ci/cache-shell.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +>&2 echo "Caching dev shell" +nix print-dev-env "${base}#" >"${cached_shell}" diff --git a/.ci/common.sh b/.ci/common.sh new file mode 100644 index 0000000..1c8d67c --- /dev/null +++ b/.ci/common.sh @@ -0,0 +1,7 @@ +# Use as: +# +# source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +set -euo pipefail +base="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)" +cached_shell="${base}/.ci/cached-shell" diff --git a/.ci/run b/.ci/run new file mode 100755 index 0000000..67b60b0 --- /dev/null +++ b/.ci/run @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" + +if [[ ! -f "${cached_shell}" ]]; then + >&2 echo "No cached shell in ${cached_shell}" + exit 1 +fi + +. "${cached_shell}" +exec "$@" From 54f4854e9f5521833dad72165e6a19ecf858b5b7 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 04/15] Start migration to flake-parts --- flake.lock | 21 +++ flake.nix | 298 +++++++++++++++++++----------------------- flake/devshells.nix | 111 ++++++++++++++++ flake/distributor.nix | 15 +++ 4 files changed, 284 insertions(+), 161 deletions(-) create mode 100644 flake/devshells.nix create mode 100644 flake/distributor.nix diff --git a/flake.lock b/flake.lock index e38f188..1e31e3b 100644 --- a/flake.lock +++ b/flake.lock @@ -36,6 +36,26 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -90,6 +110,7 @@ "inputs": { "crane": "crane", "flake-compat": "flake-compat", + "flake-parts": "flake-parts", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "nixpkgs-stable": "nixpkgs-stable" diff --git a/flake.nix b/flake.nix index 708eece..2b1fa3e 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,11 @@ nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05"; flake-utils.url = "github:numtide/flake-utils"; + flake-parts = { + url = "github:hercules-ci/flake-parts"; + inputs.nixpkgs-lib.follows = "nixpkgs"; + }; + crane = { url = "github:ipetkov/crane"; inputs.nixpkgs.follows = "nixpkgs"; @@ -17,189 +22,160 @@ }; }; - outputs = { self, nixpkgs, nixpkgs-stable, flake-utils, crane, ... }: let - supportedSystems = flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ]; + outputs = inputs @ { self, flake-parts, ... }: let + supportedSystems = inputs.flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ]; + + inherit (inputs.nixpkgs) lib; makeCranePkgs = pkgs: let - craneLib = crane.mkLib pkgs; + craneLib = inputs.crane.mkLib pkgs; in pkgs.callPackage ./crane.nix { inherit craneLib; }; - in flake-utils.lib.eachSystem supportedSystems (system: let - pkgs = import nixpkgs { - inherit system; - overlays = []; - }; - cranePkgs = makeCranePkgs pkgs; - internalMatrix = lib.mapAttrs (_: nix: let - cranePkgs' = cranePkgs.override { inherit nix; }; - in { - inherit (cranePkgs') attic-tests cargoArtifacts; - }) { - "2.20" = pkgs.nixVersions.nix_2_20; - "2.24" = pkgs.nixVersions.nix_2_24; - "default" = pkgs.nix; - }; + modules = builtins.foldl' (acc: f: f acc) ./flake [ + builtins.readDir + (lib.filterAttrs (name: type: + type == "regular" && lib.hasSuffix ".nix" name + )) + (lib.mapAttrsToList (name: _: + lib.path.append ./flake name + )) + ]; - pkgsStable = import nixpkgs-stable { - inherit system; - overlays = []; - }; - cranePkgsStable = makeCranePkgs pkgsStable; + in flake-parts.lib.mkFlake { inherit inputs; } { + imports = modules; + systems = supportedSystems; - inherit (pkgs) lib; - in rec { - inherit internalMatrix; + debug = true; - packages = { - default = packages.attic; + # old flake + flake = inputs.flake-utils.lib.eachSystem supportedSystems (system: let + pkgs = import inputs.nixpkgs { + inherit system; + overlays = []; + }; + cranePkgs = makeCranePkgs pkgs; - inherit (cranePkgs) attic attic-client attic-server; - - attic-nixpkgs = pkgs.callPackage ./package.nix { }; - - attic-ci-installer = pkgs.callPackage ./ci-installer.nix { - inherit self; + internalMatrix = lib.mapAttrs (_: nix: let + cranePkgs' = cranePkgs.override { inherit nix; }; + in { + inherit (cranePkgs') attic-tests cargoArtifacts; + }) { + "2.20" = pkgs.nixVersions.nix_2_20; + "2.24" = pkgs.nixVersions.nix_2_24; + "default" = pkgs.nix; }; - book = pkgs.callPackage ./book { - attic = packages.attic; + pkgsStable = import inputs.nixpkgs-stable { + inherit system; + overlays = []; }; - } // (lib.optionalAttrs (system != "x86_64-darwin") { - # Unfortunately, x86_64-darwin fails to evaluate static builds - # TODO: Make this work with Crane - attic-static = (pkgs.pkgsStatic.callPackage ./package.nix { - nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { - patches = (old.patches or []) ++ [ - # To be submitted - (pkgs.fetchpatch { - url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff"; - hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c="; - }) + cranePkgsStable = makeCranePkgs pkgsStable; + + inherit (pkgs) lib; + in rec { + inherit internalMatrix; + + packages = { + default = packages.attic; + + inherit (cranePkgs) attic attic-client attic-server; + + attic-nixpkgs = pkgs.callPackage ./package.nix { }; + + attic-ci-installer = pkgs.callPackage ./ci-installer.nix { + inherit self; + }; + + book = pkgs.callPackage ./book { + attic = packages.attic; + }; + } // (lib.optionalAttrs (system != "x86_64-darwin") { + # Unfortunately, x86_64-darwin fails to evaluate static builds + # TODO: Make this work with Crane + attic-static = (pkgs.pkgsStatic.callPackage ./package.nix { + nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { + patches = (old.patches or []) ++ [ + # To be submitted + (pkgs.fetchpatch { + url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff"; + hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c="; + }) + ]; + }); + }).overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + pkgs.nukeReferences ]; + + # Read by pkg_config crate (do some autodetection in build.rs?) + PKG_CONFIG_ALL_STATIC = "1"; + + "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; + RUSTFLAGS = "-C relocation-model=static"; + + postFixup = (old.postFixup or "") + '' + rm -f $out/nix-support/propagated-build-inputs + nuke-refs $out/bin/attic + ''; }); - }).overrideAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ - pkgs.nukeReferences - ]; - # Read by pkg_config crate (do some autodetection in build.rs?) - PKG_CONFIG_ALL_STATIC = "1"; + attic-client-static = packages.attic-static.override { + clientOnly = true; + }; + }) // (lib.optionalAttrs pkgs.stdenv.isLinux { + attic-server-image = pkgs.dockerTools.buildImage { + name = "attic-server"; + tag = "main"; + copyToRoot = [ + # Debugging utilities for `fly ssh console` + pkgs.busybox + packages.attic-server - "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; - RUSTFLAGS = "-C relocation-model=static"; - - postFixup = (old.postFixup or "") + '' - rm -f $out/nix-support/propagated-build-inputs - nuke-refs $out/bin/attic - ''; + # Now required by the fly.io sshd + pkgs.dockerTools.fakeNss + ]; + config = { + Entrypoint = [ "${packages.attic-server}/bin/atticd" ]; + Env = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + }; + }; }); - attic-client-static = packages.attic-static.override { - clientOnly = true; + checks = let + makeIntegrationTests = pkgs: import ./integration-tests { + pkgs = import inputs.nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + }; + flake = self; + }; + unstableTests = makeIntegrationTests pkgs; + stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable); + in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests); + }) // { + overlays = { + default = final: prev: let + cranePkgs = makeCranePkgs final; + in { + inherit (cranePkgs) attic attic-client attic-server; + }; }; - }) // (lib.optionalAttrs pkgs.stdenv.isLinux { - attic-server-image = pkgs.dockerTools.buildImage { - name = "attic-server"; - tag = "main"; - copyToRoot = [ - # Debugging utilities for `fly ssh console` - pkgs.busybox - packages.attic-server - # Now required by the fly.io sshd - pkgs.dockerTools.fakeNss - ]; - config = { - Entrypoint = [ "${packages.attic-server}/bin/atticd" ]; - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + nixosModules = { + atticd = { + imports = [ + ./nixos/atticd.nix + ]; + + services.atticd.useFlakeCompatOverlay = false; + + nixpkgs.overlays = [ + self.overlays.default ]; }; }; - }); - - devShells = { - default = pkgs.mkShell { - inputsFrom = with packages; [ attic book ]; - nativeBuildInputs = with pkgs; [ - rustc - - rustfmt clippy - cargo-expand - # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152 - # cargo-outdated - cargo-edit - tokio-console - - sqlite-interactive - - editorconfig-checker - - flyctl - - wrk - - llvmPackages_latest.bintools - wrangler worker-build wasm-pack wasm-bindgen-cli - ] ++ (lib.optionals pkgs.stdenv.isLinux [ - linuxPackages.perf - ]); - - NIX_PATH = "nixpkgs=${pkgs.path}"; - RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library"; - - # See comment in `attic/build.rs` - NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include"; - - ATTIC_DISTRIBUTOR = "dev"; - }; - - demo = pkgs.mkShell { - nativeBuildInputs = [ - packages.default - ]; - - shellHook = '' - >&2 echo - >&2 echo '🚀 Run `atticd` to get started!' - >&2 echo - ''; - }; - }; - devShell = devShells.default; - - checks = let - makeIntegrationTests = pkgs: import ./integration-tests { - pkgs = import nixpkgs { - inherit system; - overlays = [ self.overlays.default ]; - }; - flake = self; - }; - unstableTests = makeIntegrationTests pkgs; - stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable); - in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests); - }) // { - overlays = { - default = final: prev: let - cranePkgs = makeCranePkgs final; - in { - inherit (cranePkgs) attic attic-client attic-server; - }; - }; - - nixosModules = { - atticd = { - imports = [ - ./nixos/atticd.nix - ]; - - services.atticd.useFlakeCompatOverlay = false; - - nixpkgs.overlays = [ - self.overlays.default - ]; - }; }; }; } diff --git a/flake/devshells.nix b/flake/devshells.nix new file mode 100644 index 0000000..b71a0bd --- /dev/null +++ b/flake/devshells.nix @@ -0,0 +1,111 @@ +# Development shells + +toplevel @ { lib, flake-parts-lib, ... }: +let + inherit (lib) + mkOption + types + ; + inherit (flake-parts-lib) + mkPerSystemOption + ; +in +{ + options = { + perSystem = mkPerSystemOption { + options.attic.devshell = { + packageSets = mkOption { + type = types.attrsOf (types.listOf types.package); + default = {}; + }; + extraPackages = mkOption { + type = types.listOf types.package; + default = []; + }; + extraArgs = mkOption { + type = types.attrsOf types.unspecified; + default = {}; + }; + }; + }; + }; + + config = { + perSystem = { self', pkgs, config, ... }: let + cfg = config.attic.devshell; + in { + attic.devshell.packageSets = with pkgs; { + rust = [ + rustc + + cargo-expand + # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152 + # cargo-outdated + cargo-edit + tokio-console + ]; + + linters = [ + clippy + rustfmt + + editorconfig-checker + ]; + + utils = [ + jq + just + ]; + + ops = [ + postgresql + sqlite-interactive + + flyctl + wrangler + ]; + + bench = [ + wrk + ] ++ lib.optionals pkgs.stdenv.isLinux [ + linuxPackages.perf + ]; + + wasm = [ + llvmPackages_latest.bintools + worker-build wasm-pack wasm-bindgen-cli + ]; + }; + + devShells.default = pkgs.mkShell (lib.recursiveUpdate { + inputsFrom = [ + self'.packages.attic + self'.packages.book + ]; + + packages = lib.flatten (lib.attrValues cfg.packageSets); + + env = { + ATTIC_DISTRIBUTOR = toplevel.config.attic.distributor; + + RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library"; + + NIX_PATH = "nixpkgs=${pkgs.path}"; + + # See comment in `attic/build.rs` + NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include"; + }; + } cfg.extraArgs); + + devShells.demo = pkgs.mkShell { + packages = [ self'.packages.default ]; + + shellHook = '' + >&2 echo + >&2 echo '🚀 Run `atticd` to get started!' + >&2 echo + ''; + }; + }; + }; +} diff --git a/flake/distributor.nix b/flake/distributor.nix new file mode 100644 index 0000000..7b40cb0 --- /dev/null +++ b/flake/distributor.nix @@ -0,0 +1,15 @@ +{ lib, flake-parts-lib, ... }: +let + inherit (lib) + mkOption + types + ; +in +{ + options = { + attic.distributor = mkOption { + type = types.str; + default = "dev"; + }; + }; +} From 11163ab253b12733437b1bde06bc2441c903173b Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 05/15] Migrate packages to flake-parts --- flake.nix | 68 -------------------------------- flake/packages.nix | 96 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 68 deletions(-) create mode 100644 flake/packages.nix diff --git a/flake.nix b/flake.nix index 2b1fa3e..8d0ceca 100644 --- a/flake.nix +++ b/flake.nix @@ -75,74 +75,6 @@ in rec { inherit internalMatrix; - packages = { - default = packages.attic; - - inherit (cranePkgs) attic attic-client attic-server; - - attic-nixpkgs = pkgs.callPackage ./package.nix { }; - - attic-ci-installer = pkgs.callPackage ./ci-installer.nix { - inherit self; - }; - - book = pkgs.callPackage ./book { - attic = packages.attic; - }; - } // (lib.optionalAttrs (system != "x86_64-darwin") { - # Unfortunately, x86_64-darwin fails to evaluate static builds - # TODO: Make this work with Crane - attic-static = (pkgs.pkgsStatic.callPackage ./package.nix { - nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { - patches = (old.patches or []) ++ [ - # To be submitted - (pkgs.fetchpatch { - url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff"; - hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c="; - }) - ]; - }); - }).overrideAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ - pkgs.nukeReferences - ]; - - # Read by pkg_config crate (do some autodetection in build.rs?) - PKG_CONFIG_ALL_STATIC = "1"; - - "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; - RUSTFLAGS = "-C relocation-model=static"; - - postFixup = (old.postFixup or "") + '' - rm -f $out/nix-support/propagated-build-inputs - nuke-refs $out/bin/attic - ''; - }); - - attic-client-static = packages.attic-static.override { - clientOnly = true; - }; - }) // (lib.optionalAttrs pkgs.stdenv.isLinux { - attic-server-image = pkgs.dockerTools.buildImage { - name = "attic-server"; - tag = "main"; - copyToRoot = [ - # Debugging utilities for `fly ssh console` - pkgs.busybox - packages.attic-server - - # Now required by the fly.io sshd - pkgs.dockerTools.fakeNss - ]; - config = { - Entrypoint = [ "${packages.attic-server}/bin/atticd" ]; - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" - ]; - }; - }; - }); - checks = let makeIntegrationTests = pkgs: import ./integration-tests { pkgs = import inputs.nixpkgs { diff --git a/flake/packages.nix b/flake/packages.nix new file mode 100644 index 0000000..c8fdf95 --- /dev/null +++ b/flake/packages.nix @@ -0,0 +1,96 @@ +{ self, inputs, lib, ... }: +let + makeCranePkgs = pkgs: let + craneLib = inputs.crane.mkLib pkgs; + in pkgs.callPackage ../crane.nix { inherit craneLib; }; +in +{ + _module.args.makeCranePkgs = makeCranePkgs; + + perSystem = { self', pkgs, cranePkgs, ... }: (lib.mkMerge [ + { + _module.args.cranePkgs = makeCranePkgs pkgs; + + packages = { + default = self'.packages.attic; + + inherit (cranePkgs) + attic + attic-client + attic-server + ; + + attic-nixpkgs = pkgs.callPackage ../package.nix { }; + + attic-ci-installer = pkgs.callPackage ../ci-installer.nix { + inherit self; + }; + + book = pkgs.callPackage ../book { + attic = self'.packages.attic; + }; + }; + } + + (lib.mkIf pkgs.stdenv.isLinux { + packages = { + attic-server-image = pkgs.dockerTools.buildImage { + name = "attic-server"; + tag = "main"; + copyToRoot = [ + self'.packages.attic-server + + # Debugging utilities for `fly ssh console` + pkgs.busybox + + # Now required by the fly.io sshd + pkgs.dockerTools.fakeNss + ]; + config = { + Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ]; + Env = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + }; + }; + }; + }) + + # Unfortunately, x86_64-darwin fails to evaluate static builds + (lib.mkIf (pkgs.system != "x86_64-darwin") { + packages = { + # TODO: Make this work with Crane + attic-static = (pkgs.pkgsStatic.callPackage ../package.nix { + nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { + patches = (old.patches or []) ++ [ + # To be submitted + (pkgs.fetchpatch { + url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff"; + hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c="; + }) + ]; + }); + }).overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + pkgs.nukeReferences + ]; + + # Read by pkg_config crate (do some autodetection in build.rs?) + PKG_CONFIG_ALL_STATIC = "1"; + + "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; + RUSTFLAGS = "-C relocation-model=static"; + + postFixup = (old.postFixup or "") + '' + rm -f $out/nix-support/propagated-build-inputs + nuke-refs $out/bin/attic + ''; + }); + + attic-client-static = self'.packages.attic-static.override { + clientOnly = true; + }; + }; + }) + ]); +} From 8dc0bdbf7f18b816ebc6945bcf90ae296f2ad0a0 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 06/15] Migrate overlays to flake-parts --- flake.nix | 8 -------- flake/overlays.nix | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 flake/overlays.nix diff --git a/flake.nix b/flake.nix index 8d0ceca..f333523 100644 --- a/flake.nix +++ b/flake.nix @@ -87,14 +87,6 @@ stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable); in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests); }) // { - overlays = { - default = final: prev: let - cranePkgs = makeCranePkgs final; - in { - inherit (cranePkgs) attic attic-client attic-server; - }; - }; - nixosModules = { atticd = { imports = [ diff --git a/flake/overlays.nix b/flake/overlays.nix new file mode 100644 index 0000000..8cd8bef --- /dev/null +++ b/flake/overlays.nix @@ -0,0 +1,14 @@ +{ makeCranePkgs, ... }: +{ + flake.overlays = { + default = final: prev: let + cranePkgs = makeCranePkgs final; + in { + inherit (cranePkgs) + attic + attic-client + attic-server + ; + }; + }; +} From 3ecea8d07fcbc0272d2a3dd4fcbf1fce46cdcf12 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 07/15] Migrate NixOS modules to flake-parts --- flake.nix | 13 ------------- flake/nixos.nix | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 flake/nixos.nix diff --git a/flake.nix b/flake.nix index f333523..1abf156 100644 --- a/flake.nix +++ b/flake.nix @@ -87,19 +87,6 @@ stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable); in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests); }) // { - nixosModules = { - atticd = { - imports = [ - ./nixos/atticd.nix - ]; - - services.atticd.useFlakeCompatOverlay = false; - - nixpkgs.overlays = [ - self.overlays.default - ]; - }; - }; }; }; } diff --git a/flake/nixos.nix b/flake/nixos.nix new file mode 100644 index 0000000..b8141c0 --- /dev/null +++ b/flake/nixos.nix @@ -0,0 +1,16 @@ +{ config, ... }: +{ + flake.nixosModules = { + atticd = { + imports = [ + ../nixos/atticd.nix + ]; + + services.atticd.useFlakeCompatOverlay = false; + + nixpkgs.overlays = [ + config.flake.overlays.default + ]; + }; + }; +} From 09038b7663543cfb75f1fb2fe9c8fcd5796f8a22 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 08/15] flake: Add attic.nix-versions --- flake/nix-versions.nix | 52 ++++++++++++++++++++++++++++++++++++++++++ justfile | 19 +++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 flake/nix-versions.nix create mode 100644 justfile diff --git a/flake/nix-versions.nix b/flake/nix-versions.nix new file mode 100644 index 0000000..66f3697 --- /dev/null +++ b/flake/nix-versions.nix @@ -0,0 +1,52 @@ +{ lib, flake-parts-lib, ... }: +let + inherit (lib) + mkOption + types + ; + inherit (flake-parts-lib) + mkPerSystemOption + ; +in +{ + options = { + perSystem = mkPerSystemOption { + options.attic.nix-versions = { + versions = mkOption { + type = types.attrsOf types.package; + default = {}; + }; + manifestFile = mkOption { + type = types.package; + }; + }; + }; + }; + + config = { + perSystem = { self', pkgs, config, ... }: let + cfg = config.attic.nix-versions; + in { + attic.nix-versions = { + versions = { + default = pkgs.nix; + "2.20" = pkgs.nixVersions.nix_2_20; + "2.24" = pkgs.nixVersions.nix_2_24; + }; + + manifestFile = let + manifest = lib.mapAttrs (_: nix: { + inherit nix; + shellHook = '' + export NIX_INCLUDE_PATH="${lib.getDev nix}/include" + export NIX_CFLAGS_COMPILE="-isystem $NIX_INCLUDE_PATH $NIX_CFLAGS_COMPILE" + export NIX_LDFLAGS="-L${nix}/lib $NIX_LDFLAGS" + export PKG_CONFIG_PATH="${lib.getDev nix}/lib/pkgconfig:$PKG_CONFIG_PATH" + export PATH="${lib.getBin nix}/bin:$PATH" + ''; + }) cfg.versions; + in pkgs.writeText "nix-versions.json" (builtins.toJSON manifest); + }; + }; + }; +} diff --git a/justfile b/justfile new file mode 100644 index 0000000..4fa6931 --- /dev/null +++ b/justfile @@ -0,0 +1,19 @@ +set positional-arguments + +here := env_var_or_default("JUST_INVOCATION_DIR", invocation_directory()) +base := `pwd` + +#@echo "here: {{ here }}" +#@echo "base: {{ base }}" + +# List available targets +list: + @just --list --unsorted + +# Run a command with an alternative Nix version +with-nix version *command: + set -e; \ + hook="$(jq -e -r '.[$version].shellHook' --arg version "{{ version }}" < "$NIX_VERSIONS" || (>&2 echo "Version {{ version }} doesn't exist"; exit 1))"; \ + eval "$hook"; \ + CARGO_TARGET_DIR="{{ base }}/target/nix-{{ version }}" \ + {{ command }} From c6989fa54ed9d64e765f1bdf7f712093f1331228 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 09/15] flake/devshells: Add Nix versions manifest --- flake/devshells.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake/devshells.nix b/flake/devshells.nix index b71a0bd..b480bf3 100644 --- a/flake/devshells.nix +++ b/flake/devshells.nix @@ -62,6 +62,7 @@ in sqlite-interactive flyctl + ] ++ lib.optionals pkgs.stdenv.isLinux [ wrangler ]; @@ -94,6 +95,9 @@ in # See comment in `attic/build.rs` NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include"; + + # Used by `just with-nix` to build/test with alternative Nix versions. + NIX_VERSIONS = config.attic.nix-versions.manifestFile; }; } cfg.extraArgs); From 0d9c1c826fca369b751eec9819811003ecc1a71f Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 10/15] flake/devshells: Re-enable cargo-outdated --- flake/devshells.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flake/devshells.nix b/flake/devshells.nix index b480bf3..c736b2e 100644 --- a/flake/devshells.nix +++ b/flake/devshells.nix @@ -39,8 +39,7 @@ in rustc cargo-expand - # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152 - # cargo-outdated + cargo-outdated cargo-edit tokio-console ]; From 6b1d4520f33d1e9f6f48263788daa60c5ad2b61a Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 11/15] Migrate internalMatrix to flake-parts --- flake.nix | 11 ----------- flake/nix-versions.nix | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index 1abf156..3cbd238 100644 --- a/flake.nix +++ b/flake.nix @@ -55,15 +55,6 @@ }; cranePkgs = makeCranePkgs pkgs; - internalMatrix = lib.mapAttrs (_: nix: let - cranePkgs' = cranePkgs.override { inherit nix; }; - in { - inherit (cranePkgs') attic-tests cargoArtifacts; - }) { - "2.20" = pkgs.nixVersions.nix_2_20; - "2.24" = pkgs.nixVersions.nix_2_24; - "default" = pkgs.nix; - }; pkgsStable = import inputs.nixpkgs-stable { inherit system; @@ -73,8 +64,6 @@ inherit (pkgs) lib; in rec { - inherit internalMatrix; - checks = let makeIntegrationTests = pkgs: import ./integration-tests { pkgs = import inputs.nixpkgs { diff --git a/flake/nix-versions.nix b/flake/nix-versions.nix index 66f3697..b0b8df6 100644 --- a/flake/nix-versions.nix +++ b/flake/nix-versions.nix @@ -1,4 +1,4 @@ -{ lib, flake-parts-lib, ... }: +{ lib, flake-parts-lib, config, ... }: let inherit (lib) mkOption @@ -20,11 +20,17 @@ in type = types.package; }; }; + + options.internalMatrix = mkOption { + type = types.attrsOf (types.attrsOf types.package); + }; }; }; config = { - perSystem = { self', pkgs, config, ... }: let + flake.internalMatrix = lib.mapAttrs (system: ps: ps.internalMatrix) config.allSystems; + + perSystem = { self', pkgs, config, cranePkgs, ... }: let cfg = config.attic.nix-versions; in { attic.nix-versions = { @@ -47,6 +53,12 @@ in }) cfg.versions; in pkgs.writeText "nix-versions.json" (builtins.toJSON manifest); }; + + internalMatrix = lib.mapAttrs (_: nix: let + cranePkgs' = cranePkgs.override { inherit nix; }; + in { + inherit (cranePkgs') attic-tests cargoArtifacts; + }) cfg.versions; }; }; } From eeb4275172c758b2c21fdf03088e7baf83cd874f Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 12/15] .github: Factor WebAssembly build out to justfile --- .github/workflows/build.yml | 16 ++++++---------- justfile | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c1b9bb7..f265478 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,25 +45,21 @@ jobs: ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }} ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} - - name: Build and run tests + - name: Cache dev shell run: | + .ci/cache-shell.sh system=$(nix-instantiate --eval -E 'builtins.currentSystem') echo system=$system >>$GITHUB_ENV + + - name: Build and run tests + run: | tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L) find "$tests/bin" -exec {} \; - name: Build WebAssembly crates if: runner.os == 'Linux' run: | - # https://github.com/rust-lang/rust/issues/122357 - export RUST_MIN_STACK=16777216 - - pushd attic - nix develop .# --command -- cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream - popd - pushd token - nix develop .# --command -- cargo build --target wasm32-unknown-unknown - popd + .ci/run just ci-build-wasm # TODO: Just take a diff of the list of store paths, also abstract all of this out - name: Push build artifacts diff --git a/justfile b/justfile index 4fa6931..2216a49 100644 --- a/justfile +++ b/justfile @@ -17,3 +17,18 @@ with-nix version *command: eval "$hook"; \ CARGO_TARGET_DIR="{{ base }}/target/nix-{{ version }}" \ {{ command }} + +# (CI) Build WebAssembly crates +ci-build-wasm: + #!/usr/bin/env bash + set -euxo pipefail + + # https://github.com/rust-lang/rust/issues/122357 + export RUST_MIN_STACK=16777216 + + pushd attic + cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream + popd + pushd token + cargo build --target wasm32-unknown-unknown + popd From a9cafe829824eb12caa24243fbdfab9c29172deb Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 13/15] .github: Factor unit tests out to justfile --- .github/workflows/build.yml | 5 ++--- justfile | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f265478..75f815c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,10 +51,9 @@ jobs: system=$(nix-instantiate --eval -E 'builtins.currentSystem') echo system=$system >>$GITHUB_ENV - - name: Build and run tests + - name: Run unit tests run: | - tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L) - find "$tests/bin" -exec {} \; + .ci/run just ci-unit-tests ${{ matrix.nix }} - name: Build WebAssembly crates if: runner.os == 'Linux' diff --git a/justfile b/justfile index 2216a49..64dcc2c 100644 --- a/justfile +++ b/justfile @@ -32,3 +32,12 @@ ci-build-wasm: pushd token cargo build --target wasm32-unknown-unknown popd + +# (CI) Run unit tests +ci-unit-tests matrix: + #!/usr/bin/env bash + set -euxo pipefail + + system=$(nix-instantiate --eval -E 'builtins.currentSystem') + tests=$(nix build .#internalMatrix."$system".\"{{ matrix }}\".attic-tests --no-link --print-out-paths -L) + find "$tests/bin" -exec {} \; From c5d84a475df3d82d388a96bd659fd9bfff916593 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 14/15] Migrate integration tests to flake-parts Well, actually also to fix them. --- flake.nix | 35 ---------------------- flake/integration-tests.nix | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 35 deletions(-) create mode 100644 flake/integration-tests.nix diff --git a/flake.nix b/flake.nix index 3cbd238..68667fd 100644 --- a/flake.nix +++ b/flake.nix @@ -27,10 +27,6 @@ inherit (inputs.nixpkgs) lib; - makeCranePkgs = pkgs: let - craneLib = inputs.crane.mkLib pkgs; - in pkgs.callPackage ./crane.nix { inherit craneLib; }; - modules = builtins.foldl' (acc: f: f acc) ./flake [ builtins.readDir (lib.filterAttrs (name: type: @@ -46,36 +42,5 @@ systems = supportedSystems; debug = true; - - # old flake - flake = inputs.flake-utils.lib.eachSystem supportedSystems (system: let - pkgs = import inputs.nixpkgs { - inherit system; - overlays = []; - }; - cranePkgs = makeCranePkgs pkgs; - - - pkgsStable = import inputs.nixpkgs-stable { - inherit system; - overlays = []; - }; - cranePkgsStable = makeCranePkgs pkgsStable; - - inherit (pkgs) lib; - in rec { - checks = let - makeIntegrationTests = pkgs: import ./integration-tests { - pkgs = import inputs.nixpkgs { - inherit system; - overlays = [ self.overlays.default ]; - }; - flake = self; - }; - unstableTests = makeIntegrationTests pkgs; - stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable); - in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests); - }) // { - }; }; } diff --git a/flake/integration-tests.nix b/flake/integration-tests.nix new file mode 100644 index 0000000..a05d742 --- /dev/null +++ b/flake/integration-tests.nix @@ -0,0 +1,60 @@ +{ lib, flake-parts-lib, inputs, self, ... }: +let + inherit (lib) + mkOption + types + ; + inherit (flake-parts-lib) + mkPerSystemOption + ; +in +{ + options = { + perSystem = mkPerSystemOption { + options.attic.integration-tests = { + nixpkgsArgs = mkOption { + type = types.attrsOf types.anything; + default = {}; + }; + tests = mkOption { + type = types.attrsOf types.package; + default = {}; + }; + stableTests = mkOption { + type = types.attrsOf types.package; + default = {}; + }; + }; + }; + }; + + config = { + perSystem = { self', pkgs, config, system, ... }: let + cfg = config.attic.integration-tests; + + vmPkgs = import inputs.nixpkgs ({ + inherit system; + overlays = [ self.overlays.default ]; + } // cfg.nixpkgsArgs); + vmPkgsStable = import inputs.nixpkgs-stable ({ + inherit system; + overlays = [ self.overlays.default ]; + } // cfg.nixpkgsArgs); + + makeIntegrationTests = pkgs: import ../integration-tests { + inherit pkgs; + flake = self; + }; + in { + attic.integration-tests = { + tests = makeIntegrationTests vmPkgs; + stableTests = makeIntegrationTests vmPkgsStable; + }; + + checks = let + tests = cfg.tests; + stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") cfg.stableTests; + in lib.optionalAttrs pkgs.stdenv.isLinux (tests // stableTests); + }; + }; +} From 7f275f00d8932e56cd7929b39c08d3e8f7fad74c Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 30 Aug 2024 12:32:10 -0400 Subject: [PATCH 15/15] .github: Update bash on macOS runners --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75f815c..031d200 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,11 @@ jobs: steps: - uses: actions/checkout@v4.1.1 + - name: Install current Bash on macOS + if: runner.os == 'macOS' + run: | + command -v brew && brew install bash || true + - uses: DeterminateSystems/nix-installer-action@v9 continue-on-error: true # Self-hosted runners already have Nix installed