From 903fb4e39eecb944495b01903b253ba5748d6d20 Mon Sep 17 00:00:00 2001
From: Cole Helbling <cole.e.helbling@outlook.com>
Date: Fri, 26 Jul 2024 10:21:52 -0700
Subject: [PATCH 01/56] fixup: stream error logging

The call to `into_inner()` discards the wrapper type constructed by
`map_err()`. So instead, `map_err()` the actual stream, and call
`Body::from_stream` on the wrapped stream.
---
 server/src/api/binary_cache.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/server/src/api/binary_cache.rs b/server/src/api/binary_cache.rs
index 05e6630..242f0f1 100644
--- a/server/src/api/binary_cache.rs
+++ b/server/src/api/binary_cache.rs
@@ -18,7 +18,7 @@ use axum::{
     Router,
 };
 use futures::stream::BoxStream;
-use http_body_util::BodyExt;
+use futures::TryStreamExt as _;
 use serde::Serialize;
 use tokio_util::io::ReaderStream;
 use tracing::instrument;
@@ -217,11 +217,11 @@ async fn get_nar(
         match storage.download_file_db(remote_file, false).await? {
             Download::Url(url) => Ok(Redirect::temporary(&url).into_response()),
             Download::AsyncRead(stream) => {
-                let stream = ReaderStream::new(stream);
-                let body = Body::from_stream(stream).map_err(|e| {
+                let stream = ReaderStream::new(stream).map_err(|e| {
                     tracing::error!("Stream error: {e}");
                     e
-                }).into_inner();
+                });
+                let body = Body::from_stream(stream);
 
                 Ok(body.into_response())
             }
@@ -254,11 +254,11 @@ async fn get_nar(
 
         // TODO: Make num_prefetch configurable
         // The ideal size depends on the average chunk size
-        let merged = merge_chunks(chunks, streamer, storage, 2);
-        let body = Body::from_stream(merged).map_err(|e| {
+        let merged = merge_chunks(chunks, streamer, storage, 2).map_err(|e| {
             tracing::error!("Stream error: {e}");
             e
-        }).into_inner();
+        });
+        let body = Body::from_stream(merged);
 
         Ok(body.into_response())
     }

From 443ceac40f5b645098235074a273bdef54548ad2 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Tue, 13 Aug 2024 07:39:37 -0600
Subject: [PATCH 02/56] server: Upsert object row on conflict

Upsert instead of doing delete+insert or ignoring the specific error.

Fixes #132.
---
 server/src/api/v1/upload_path.rs     | 23 ++++-------------------
 server/src/database/entity/object.rs | 27 +++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/server/src/api/v1/upload_path.rs b/server/src/api/v1/upload_path.rs
index 3673797..4b04601 100644
--- a/server/src/api/v1/upload_path.rs
+++ b/server/src/api/v1/upload_path.rs
@@ -46,7 +46,7 @@ use crate::database::entity::cache;
 use crate::database::entity::chunk::{self, ChunkState, Entity as Chunk};
 use crate::database::entity::chunkref::{self, Entity as ChunkRef};
 use crate::database::entity::nar::{self, Entity as Nar, NarState};
-use crate::database::entity::object::{self, Entity as Object};
+use crate::database::entity::object::{self, Entity as Object, InsertExt};
 use crate::database::entity::Json as DbJson;
 use crate::database::{AtticDatabase, ChunkGuard, NarGuard};
 
@@ -257,12 +257,6 @@ async fn upload_path_dedup(
         .map_err(ServerError::database_error)?;
 
     // Create a mapping granting the local cache access to the NAR
-    Object::delete_many()
-        .filter(object::Column::CacheId.eq(cache.id))
-        .filter(object::Column::StorePathHash.eq(upload_info.store_path_hash.to_string()))
-        .exec(&txn)
-        .await
-        .map_err(ServerError::database_error)?;
     Object::insert({
         let mut new_object = upload_info.to_active_model();
         new_object.cache_id = Set(cache.id);
@@ -271,6 +265,7 @@ async fn upload_path_dedup(
         new_object.created_by = Set(username);
         new_object
     })
+    .on_conflict_do_update()
     .exec(&txn)
     .await
     .map_err(ServerError::database_error)?;
@@ -487,12 +482,6 @@ async fn upload_path_new_chunked(
     .map_err(ServerError::database_error)?;
 
     // Create a mapping granting the local cache access to the NAR
-    Object::delete_many()
-        .filter(object::Column::CacheId.eq(cache.id))
-        .filter(object::Column::StorePathHash.eq(upload_info.store_path_hash.to_string()))
-        .exec(&txn)
-        .await
-        .map_err(ServerError::database_error)?;
     Object::insert({
         let mut new_object = upload_info.to_active_model();
         new_object.cache_id = Set(cache.id);
@@ -501,6 +490,7 @@ async fn upload_path_new_chunked(
         new_object.created_by = Set(username);
         new_object
     })
+    .on_conflict_do_update()
     .exec(&txn)
     .await
     .map_err(ServerError::database_error)?;
@@ -594,12 +584,6 @@ async fn upload_path_new_unchunked(
     .map_err(ServerError::database_error)?;
 
     // Create a mapping granting the local cache access to the NAR
-    Object::delete_many()
-        .filter(object::Column::CacheId.eq(cache.id))
-        .filter(object::Column::StorePathHash.eq(upload_info.store_path_hash.to_string()))
-        .exec(&txn)
-        .await
-        .map_err(ServerError::database_error)?;
     Object::insert({
         let mut new_object = upload_info.to_active_model();
         new_object.cache_id = Set(cache.id);
@@ -608,6 +592,7 @@ async fn upload_path_new_unchunked(
         new_object.created_by = Set(username);
         new_object
     })
+    .on_conflict_do_update()
     .exec(&txn)
     .await
     .map_err(ServerError::database_error)?;
diff --git a/server/src/database/entity/object.rs b/server/src/database/entity/object.rs
index 37ad893..69b5a62 100644
--- a/server/src/database/entity/object.rs
+++ b/server/src/database/entity/object.rs
@@ -6,6 +6,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 
 use sea_orm::entity::prelude::*;
+use sea_orm::sea_query::OnConflict;
+use sea_orm::Insert;
 
 use super::nar::NarModel;
 use super::Json;
@@ -15,6 +17,10 @@ use attic::hash::Hash;
 
 pub type ObjectModel = Model;
 
+pub trait InsertExt {
+    fn on_conflict_do_update(self) -> Self;
+}
+
 /// An object in a binary cache.
 #[derive(Debug, Clone, PartialEq, Eq, DeriveEntityModel)]
 #[sea_orm(table_name = "object")]
@@ -87,6 +93,27 @@ pub enum Relation {
     Nar,
 }
 
+impl InsertExt for Insert<ActiveModel> {
+    fn on_conflict_do_update(self) -> Self {
+        self.on_conflict(
+            OnConflict::columns([Column::CacheId, Column::StorePathHash])
+                .update_columns([
+                    Column::NarId,
+                    Column::StorePath,
+                    Column::References,
+                    Column::System,
+                    Column::Deriver,
+                    Column::Sigs,
+                    Column::Ca,
+                    Column::CreatedAt,
+                    Column::LastAccessedAt,
+                    Column::CreatedBy,
+                ])
+                .to_owned(),
+        )
+    }
+}
+
 impl Model {
     /// Converts this object to a NarInfo.
     pub fn to_nar_info(&self, nar: &NarModel) -> ServerResult<NarInfo> {

From aecca91fad10a53687065952db1c7ebaf257ec54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 19 Aug 2024 12:16:33 -0400
Subject: [PATCH 03/56] Build with Nix 2.24 in devShell

---
 flake.lock | 48 +++++++++++++++++++++++++++++++++---------------
 flake.nix  |  7 +++++--
 2 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1101918..e72975b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717025063,
-        "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=",
+        "lastModified": 1722960479,
+        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e",
+        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
         "type": "github"
       },
       "original": {
@@ -23,11 +23,11 @@
     "flake-compat": {
       "flake": false,
       "locked": {
-        "lastModified": 1673956053,
-        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
         "type": "github"
       },
       "original": {
@@ -37,12 +37,15 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
@@ -53,11 +56,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1711401922,
-        "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=",
+        "lastModified": 1723827930,
+        "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "07262b18b97000d16a4bdb003418bd2fb067a932",
+        "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d",
         "type": "github"
       },
       "original": {
@@ -69,11 +72,11 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1711460390,
-        "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=",
+        "lastModified": 1720535198,
+        "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "44733514b72e732bd49f5511bd0203dea9b9a434",
+        "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5",
         "type": "github"
       },
       "original": {
@@ -91,6 +94,21 @@
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index ea81e6f..07fd7c7 100644
--- a/flake.nix
+++ b/flake.nix
@@ -113,7 +113,10 @@
           rustc
 
           rustfmt clippy
-          cargo-expand cargo-outdated cargo-edit
+          cargo-expand
+          # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152
+          # cargo-outdated
+          cargo-edit
           tokio-console
 
           sqlite-interactive
@@ -131,7 +134,7 @@
         RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
 
         # See comment in `attic/build.rs`
-        NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include";
+        NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include";
 
         ATTIC_DISTRIBUTOR = "dev";
       };

From 40c0bb406e4e68cd81031300eb9c2bd184a8653c Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 12:16:33 -0400
Subject: [PATCH 04/56] Fix build for Nix 2.20+
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Perform version detection to keep supporting older versions
(and in the future, alternative implementations).

Co-authored-by: Jörg Thalheim <joerg@thalheim.io>
---
 Cargo.lock                           | 20 +++++++--
 attic/Cargo.toml                     |  2 +
 attic/build.rs                       | 63 ++++++++++++++++++++++------
 attic/src/nix_store/bindings/nix.cpp | 10 ++++-
 4 files changed, 79 insertions(+), 16 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 165b9a9..a8bd1f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -230,6 +230,7 @@ dependencies = [
  "async-stream",
  "base64 0.22.1",
  "bytes",
+ "cc",
  "cxx",
  "cxx-build",
  "digest",
@@ -250,6 +251,7 @@ dependencies = [
  "tempfile",
  "tokio",
  "tokio-test",
+ "version-compare",
  "wildmatch",
  "xdg",
 ]
@@ -1050,13 +1052,13 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.98"
+version = "1.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
+checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48"
 dependencies = [
  "jobserver",
  "libc",
- "once_cell",
+ "shlex",
 ]
 
 [[package]]
@@ -3960,6 +3962,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
 
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.2"
@@ -4891,6 +4899,12 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "version-compare"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index c24a771..957bb02 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -43,9 +43,11 @@ serde_json = "1.0.96"
 tokio-test = "0.4.2"
 
 [build-dependencies]
+cc = "1.1.13"
 cxx-build = { version = "1.0", optional = true }
 pkg-config = "0.3.27"
 tempfile = "3"
+version-compare = "0.2.0"
 
 [features]
 default = [ "nix_store", "tokio" ]
diff --git a/attic/build.rs b/attic/build.rs
index 1ebc91e..14151a2 100644
--- a/attic/build.rs
+++ b/attic/build.rs
@@ -2,6 +2,47 @@
 //!
 //! We link against libnixstore to perform actions on the Nix Store.
 
+use cc::Build;
+use version_compare::Version;
+
+struct NixDependency {
+    version: String,
+}
+
+impl NixDependency {
+    fn detect() -> Self {
+        let library = pkg_config::Config::new()
+            .cargo_metadata(false)
+            .atleast_version("2.4")
+            .probe("nix-main")
+            .expect("Failed to find nix-main >=2.4 through pkg-config");
+
+        Self {
+            version: library.version,
+        }
+    }
+
+    fn apply_version_flags(&self, build: &mut Build) {
+        let version = Version::from(&self.version).unwrap();
+
+        if version >= Version::from("2.20").unwrap() {
+            build.flag("-DATTIC_NIX_2_20");
+        }
+    }
+
+    fn emit_cargo_metadata(&self) {
+        pkg_config::Config::new()
+            .atleast_version("2.4")
+            .probe("nix-store")
+            .unwrap();
+
+        pkg_config::Config::new()
+            .atleast_version("2.4")
+            .probe("nix-main")
+            .unwrap();
+    }
+}
+
 fn main() {
     #[cfg(feature = "nix_store")]
     build_bridge();
@@ -9,6 +50,8 @@ fn main() {
 
 #[cfg(feature = "nix_store")]
 fn build_bridge() {
+    let nix_dep = NixDependency::detect();
+
     // Temporary workaround for issue in <https://github.com/NixOS/nix/pull/8484>
     let hacky_include = {
         let dir = tempfile::tempdir().expect("Failed to create temporary directory for workaround");
@@ -16,7 +59,8 @@ fn build_bridge() {
         dir
     };
 
-    cxx_build::bridge("src/nix_store/bindings/mod.rs")
+    let mut build = cxx_build::bridge("src/nix_store/bindings/mod.rs");
+    build
         .file("src/nix_store/bindings/nix.cpp")
         .flag("-std=c++2a")
         .flag("-O2")
@@ -26,19 +70,14 @@ fn build_bridge() {
         .flag(hacky_include.path().to_str().unwrap())
         // In Nix 2.19+, nix/args/root.hh depends on being able to #include "args.hh" (which is in its parent directory), for some reason
         .flag("-I")
-        .flag(concat!(env!("NIX_INCLUDE_PATH"), "/nix"))
-        .compile("nixbinding");
+        .flag(concat!(env!("NIX_INCLUDE_PATH"), "/nix"));
+
+    nix_dep.apply_version_flags(&mut build);
+
+    build.compile("nixbinding");
 
     println!("cargo:rerun-if-changed=src/nix_store/bindings");
 
     // the -l flags must be after -lnixbinding
-    pkg_config::Config::new()
-        .atleast_version("2.4")
-        .probe("nix-store")
-        .unwrap();
-
-    pkg_config::Config::new()
-        .atleast_version("2.4")
-        .probe("nix-main")
-        .unwrap();
+    nix_dep.emit_cargo_metadata();
 }
diff --git a/attic/src/nix_store/bindings/nix.cpp b/attic/src/nix_store/bindings/nix.cpp
index 59ab3ad..11607df 100644
--- a/attic/src/nix_store/bindings/nix.cpp
+++ b/attic/src/nix_store/bindings/nix.cpp
@@ -18,6 +18,14 @@ static nix::StorePath store_path_from_rust(RBasePathSlice base_name) {
 	return nix::StorePath(sv);
 }
 
+static bool hash_is_sha256(const nix::Hash &hash) {
+#ifdef ATTIC_NIX_2_20
+	return hash.algo == nix::HashAlgorithm::SHA256;
+#else
+	return hash.type == nix::htSHA256;
+#endif
+}
+
 // ========
 // RustSink
 // ========
@@ -44,7 +52,7 @@ CPathInfo::CPathInfo(nix::ref<const nix::ValidPathInfo> pi) : pi(pi) {}
 RHashSlice CPathInfo::nar_sha256_hash() {
 	auto &hash = this->pi->narHash;
 
-	if (hash.type != nix::htSHA256) {
+	if (!hash_is_sha256(hash)) {
 		throw nix::Error("Only SHA-256 hashes are supported at the moment");
 	}
 

From 38c42ae9ad510df93fee8dcc4f51b916f1fc8afc Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 12:16:33 -0400
Subject: [PATCH 05/56] Run tests against different Nix versions

---
 .github/workflows/build.yml | 12 +++++++++---
 flake.nix                   | 16 ++++++++++++----
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2440837..af099f7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -12,6 +12,10 @@ jobs:
         os:
           - ubuntu-latest
           - macos-latest
+        nix:
+          - "2.20"
+          - "2.24"
+          - "default"
     runs-on: ${{ matrix.os }}
     permissions:
       contents: read
@@ -45,7 +49,7 @@ jobs:
         run: |
           system=$(nix-instantiate --eval -E 'builtins.currentSystem')
           echo system=$system >>$GITHUB_ENV
-          tests=$(nix build .#internal."$system".attic-tests --no-link --print-out-paths -L)
+          tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L)
           find "$tests/bin" -exec {} \;
 
       # TODO: Just take a diff of the list of store paths, also abstract all of this out
@@ -53,8 +57,10 @@ jobs:
         run: |
           export PATH=$HOME/.nix-profile/bin:$PATH # FIXME
           if [ -n "$ATTIC_TOKEN" ]; then
-            nix build .#internal."$system".attic-tests .#internal."$system".cargoArtifacts --no-link --print-out-paths -L | \
-              xargs attic push "ci:$ATTIC_CACHE"
+            nix build --no-link --print-out-paths -L \
+              .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests \
+              .#internalMatrix."$system".\"${{ matrix.nix }}\".cargoArtifacts \
+            | xargs attic push "ci:$ATTIC_CACHE"
           fi
       - name: Log in to the Container registry
         uses: docker/login-action@v3.0.0
diff --git a/flake.nix b/flake.nix
index 07fd7c7..717e582 100644
--- a/flake.nix
+++ b/flake.nix
@@ -30,6 +30,16 @@
     };
     cranePkgs = makeCranePkgs pkgs;
 
+    internalMatrix = lib.mapAttrs (_: nix: let
+      cranePkgs' = cranePkgs.override { inherit nix; };
+    in {
+      inherit (cranePkgs') attic-tests cargoArtifacts;
+    }) {
+      "2.20" = pkgs.nixVersions.nix_2_20;
+      "2.24" = pkgs.nixVersions.nix_2_24;
+      "default" = pkgs.nix;
+    };
+
     pkgsStable = import nixpkgs-stable {
       inherit system;
       overlays = [];
@@ -38,6 +48,8 @@
 
     inherit (pkgs) lib;
   in rec {
+    inherit internalMatrix;
+
     packages = {
       default = packages.attic;
 
@@ -153,10 +165,6 @@
     };
     devShell = devShells.default;
 
-    internal = {
-      inherit (cranePkgs) attic-tests cargoArtifacts;
-    };
-
     checks = let
       makeIntegrationTests = pkgs: import ./integration-tests {
         pkgs = import nixpkgs {

From 956d6915a575732cb3d228b811789fba9c6bcd36 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 06/56] attic: Split out stream feature

Right now it just depends on tokio, but the goal is to support
alternative async runtimes.
---
 attic/Cargo.toml  | 17 +++++++++++------
 attic/src/lib.rs  |  2 +-
 server/Cargo.toml |  2 +-
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index 957bb02..4cf79f2 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -50,14 +50,19 @@ tempfile = "3"
 version-compare = "0.2.0"
 
 [features]
-default = [ "nix_store", "tokio" ]
+default = [
+	"nix_store",
+	"stream",
+	"tokio",
+]
 
 # Native libnixstore bindings.
 #
 # When disabled, the native Rust portions of nix_store can still be used.
-nix_store = [ "dep:cxx", "dep:cxx-build", "tokio/rt" ]
+nix_store = ["tokio", "dep:cxx", "dep:cxx-build"]
 
-# Tokio.
-#
-# When disabled, any part depending on tokio is unavailable.
-tokio = [ "dep:tokio", "dep:async-stream" ]
+# Stream utilities.
+stream = ["tokio", "dep:async-stream"]
+
+# Tokio runtime.
+tokio = ["dep:tokio", "tokio/rt"]
diff --git a/attic/src/lib.rs b/attic/src/lib.rs
index 05c1f52..04ff0fd 100644
--- a/attic/src/lib.rs
+++ b/attic/src/lib.rs
@@ -22,7 +22,7 @@ pub mod hash;
 pub mod mime;
 pub mod nix_store;
 pub mod signing;
-#[cfg(feature = "tokio")]
+#[cfg(feature = "stream")]
 pub mod stream;
 #[cfg(target_family = "unix")]
 pub mod testing;
diff --git a/server/Cargo.toml b/server/Cargo.toml
index a4794d3..ec50929 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -19,7 +19,7 @@ path = "src/adm/main.rs"
 doc = false
 
 [dependencies]
-attic = { path = "../attic", default-features = false, features = [ "tokio" ] }
+attic = { path = "../attic", default-features = false, features = ["stream", "tokio"] }
 attic-token = { path = "../token" }
 
 anyhow = "1.0.71"

From deff31a85007aeef3a84091940cbe7a3811e3cca Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 07/56] Move chunking to attic crate

---
 Cargo.lock                            | 2 +-
 attic/Cargo.toml                      | 5 +++++
 {server => attic}/src/chunking/mod.rs | 2 +-
 attic/src/lib.rs                      | 2 ++
 server/Cargo.toml                     | 3 +--
 server/src/api/v1/upload_path.rs      | 2 +-
 server/src/lib.rs                     | 1 -
 7 files changed, 11 insertions(+), 6 deletions(-)
 rename {server => attic}/src/chunking/mod.rs (98%)

diff --git a/Cargo.lock b/Cargo.lock
index a8bd1f0..3411db8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -236,6 +236,7 @@ dependencies = [
  "digest",
  "displaydoc",
  "ed25519-compact",
+ "fastcdc",
  "futures",
  "hex",
  "lazy_static",
@@ -309,7 +310,6 @@ dependencies = [
  "digest",
  "displaydoc",
  "enum-as-inner",
- "fastcdc",
  "futures",
  "hex",
  "http-body-util",
diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index 4cf79f2..ceaee24 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -11,6 +11,7 @@ bytes = "1.4.0"
 displaydoc = "0.2.4"
 digest = "0.10.7"
 ed25519-compact = "2.0.4"
+fastcdc = "3.0.3"
 futures = "0.3.28"
 hex = "0.4.3"
 lazy_static = "1.4.0"
@@ -51,11 +52,15 @@ version-compare = "0.2.0"
 
 [features]
 default = [
+	"chunking",
 	"nix_store",
 	"stream",
 	"tokio",
 ]
 
+# Chunking.
+chunking = ["tokio", "dep:async-stream"]
+
 # Native libnixstore bindings.
 #
 # When disabled, the native Rust portions of nix_store can still be used.
diff --git a/server/src/chunking/mod.rs b/attic/src/chunking/mod.rs
similarity index 98%
rename from server/src/chunking/mod.rs
rename to attic/src/chunking/mod.rs
index b567847..00862ef 100644
--- a/server/src/chunking/mod.rs
+++ b/attic/src/chunking/mod.rs
@@ -9,7 +9,7 @@ use fastcdc::ronomon::FastCDC;
 use futures::stream::Stream;
 use tokio::io::AsyncRead;
 
-use attic::stream::read_chunk_async;
+use crate::stream::read_chunk_async;
 
 /// Splits a streams into content-defined chunks.
 ///
diff --git a/attic/src/lib.rs b/attic/src/lib.rs
index 04ff0fd..8cc2cec 100644
--- a/attic/src/lib.rs
+++ b/attic/src/lib.rs
@@ -17,6 +17,8 @@
 
 pub mod api;
 pub mod cache;
+#[cfg(feature = "chunking")]
+pub mod chunking;
 pub mod error;
 pub mod hash;
 pub mod mime;
diff --git a/server/Cargo.toml b/server/Cargo.toml
index ec50929..3fcf108 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -19,7 +19,7 @@ path = "src/adm/main.rs"
 doc = false
 
 [dependencies]
-attic = { path = "../attic", default-features = false, features = ["stream", "tokio"] }
+attic = { path = "../attic", default-features = false, features = ["chunking", "stream", "tokio"] }
 attic-token = { path = "../token" }
 
 anyhow = "1.0.71"
@@ -37,7 +37,6 @@ derivative = "2.2.0"
 digest = "0.10.7"
 displaydoc = "0.2.4"
 enum-as-inner = "0.6.0"
-fastcdc = "3.0.3"
 futures = "0.3.28"
 hex = "0.4.3"
 http-body-util = "0.1.1"
diff --git a/server/src/api/v1/upload_path.rs b/server/src/api/v1/upload_path.rs
index 4b04601..049795d 100644
--- a/server/src/api/v1/upload_path.rs
+++ b/server/src/api/v1/upload_path.rs
@@ -37,11 +37,11 @@ use attic::api::v1::upload_path::{
     UploadPathNarInfo, UploadPathResult, UploadPathResultKind, ATTIC_NAR_INFO,
     ATTIC_NAR_INFO_PREAMBLE_SIZE,
 };
+use attic::chunking::chunk_stream;
 use attic::hash::Hash;
 use attic::stream::{read_chunk_async, StreamHasher};
 use attic::util::Finally;
 
-use crate::chunking::chunk_stream;
 use crate::database::entity::cache;
 use crate::database::entity::chunk::{self, ChunkState, Entity as Chunk};
 use crate::database::entity::chunkref::{self, Entity as ChunkRef};
diff --git a/server/src/lib.rs b/server/src/lib.rs
index 93f2e0b..0314e69 100644
--- a/server/src/lib.rs
+++ b/server/src/lib.rs
@@ -15,7 +15,6 @@
 
 pub mod access;
 mod api;
-mod chunking;
 pub mod config;
 pub mod database;
 pub mod error;

From a41e2d1724a3a68d6ff75bd61f29351cac64195b Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 08/56] attic: Add simple chunking benchmarks

---
 Cargo.lock                | 206 +++++++++++++++++++++++++++++++++++++-
 attic/Cargo.toml          |   6 ++
 attic/benches/chunking.rs |  84 ++++++++++++++++
 attic/src/chunking/mod.rs |  18 +---
 attic/src/testing/mod.rs  |  24 +++++
 5 files changed, 321 insertions(+), 17 deletions(-)
 create mode 100644 attic/benches/chunking.rs

diff --git a/Cargo.lock b/Cargo.lock
index 3411db8..4a04bbc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -92,6 +92,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstream"
 version = "0.6.14"
@@ -231,6 +237,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "cc",
+ "criterion",
  "cxx",
  "cxx-build",
  "digest",
@@ -1050,6 +1057,12 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.1.13"
@@ -1088,6 +1101,33 @@ dependencies = [
  "windows-targets 0.52.5",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
 [[package]]
 name = "clap"
 version = "4.5.4"
@@ -1307,6 +1347,44 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "futures",
+ "is-terminal",
+ "itertools 0.10.5",
+ "num-traits",
+ "once_cell",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "tokio",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools 0.10.5",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.13"
@@ -1316,6 +1394,25 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.11"
@@ -1331,6 +1428,12 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+
 [[package]]
 name = "crypto-bigint"
 version = "0.4.9"
@@ -1698,6 +1801,11 @@ name = "fastcdc"
 version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a71061d097bfa9a5a4d2efdec57990d9a88745020b365191d37e48541a1628f2"
+dependencies = [
+ "async-stream",
+ "tokio",
+ "tokio-stream",
+]
 
 [[package]]
 name = "fastrand"
@@ -1956,6 +2064,16 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
+dependencies = [
+ "cfg-if",
+ "crunchy",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -2388,12 +2506,32 @@ version = "2.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.12.1"
@@ -2776,6 +2914,12 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "oorandom"
+version = "11.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
@@ -3002,6 +3146,34 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
+[[package]]
+name = "plotters"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "portable-atomic"
 version = "1.6.0"
@@ -3088,7 +3260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
 dependencies = [
  "anyhow",
- "itertools",
+ "itertools 0.12.1",
  "proc-macro2",
  "quote",
  "syn 2.0.66",
@@ -3168,6 +3340,26 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.4.1"
@@ -4069,7 +4261,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c"
 dependencies = [
- "itertools",
+ "itertools 0.12.1",
  "nom",
  "unicode_categories",
 ]
@@ -4459,6 +4651,16 @@ dependencies = [
  "time-core",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.6.0"
diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index ceaee24..756519b 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -40,6 +40,8 @@ features = [
 ]
 
 [dev-dependencies]
+criterion = { version = "0.5", features = ["html_reports", "async_tokio"] }
+fastcdc = { version = "*", features = ["tokio"] }
 serde_json = "1.0.96"
 tokio-test = "0.4.2"
 
@@ -71,3 +73,7 @@ stream = ["tokio", "dep:async-stream"]
 
 # Tokio runtime.
 tokio = ["dep:tokio", "tokio/rt"]
+
+[[bench]]
+name = "chunking"
+harness = false
diff --git a/attic/benches/chunking.rs b/attic/benches/chunking.rs
new file mode 100644
index 0000000..1d6be48
--- /dev/null
+++ b/attic/benches/chunking.rs
@@ -0,0 +1,84 @@
+use std::io::Cursor;
+
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use futures::StreamExt;
+
+use attic::chunking::chunk_stream;
+use attic::testing::{get_fake_data, get_runtime};
+
+struct Parameters {
+    min_size: u32,
+    avg_size: u32,
+    max_size: u32,
+}
+
+pub fn bench_chunking(c: &mut Criterion) {
+    let rt = get_runtime();
+    let data = get_fake_data(128 * 1024 * 1024); // 128 MiB
+
+    let cases = [
+        (
+            "2K,4K,8K",
+            Parameters {
+                min_size: 2 * 1024,
+                avg_size: 4 * 1024,
+                max_size: 8 * 1024,
+            },
+        ),
+        (
+            "8K,16K,32K",
+            Parameters {
+                min_size: 8 * 1024,
+                avg_size: 16 * 1024,
+                max_size: 32 * 1024,
+            },
+        ),
+        (
+            "1M,4M,16M",
+            Parameters {
+                min_size: 1024 * 1024,
+                avg_size: 4 * 1024 * 1024,
+                max_size: 16 * 1024 * 1024,
+            },
+        ),
+    ];
+
+    let mut group = c.benchmark_group("chunking");
+    group.throughput(Throughput::Bytes(data.len() as u64));
+
+    for (case, params) in cases {
+        group.bench_with_input(BenchmarkId::new("ronomon", case), &params, |b, params| {
+            b.to_async(&rt).iter(|| async {
+                let cursor = Cursor::new(&data);
+                let mut chunks = chunk_stream(
+                    cursor,
+                    params.min_size as usize,
+                    params.avg_size as usize,
+                    params.max_size as usize,
+                );
+                while let Some(chunk) = chunks.next().await {
+                    black_box(chunk).unwrap();
+                }
+            })
+        });
+        group.bench_with_input(BenchmarkId::new("v2020", case), &params, |b, params| {
+            b.to_async(&rt).iter(|| async {
+                let cursor = Cursor::new(&data);
+                let mut chunks = fastcdc::v2020::AsyncStreamCDC::new(
+                    cursor,
+                    params.min_size,
+                    params.avg_size,
+                    params.max_size,
+                );
+                let mut chunks = Box::pin(chunks.as_stream());
+                while let Some(chunk) = chunks.next().await {
+                    black_box(chunk).unwrap();
+                }
+            })
+        });
+    }
+    group.finish();
+}
+
+criterion_group!(benches, bench_chunking);
+criterion_main!(benches);
diff --git a/attic/src/chunking/mod.rs b/attic/src/chunking/mod.rs
index 00862ef..031aa96 100644
--- a/attic/src/chunking/mod.rs
+++ b/attic/src/chunking/mod.rs
@@ -74,12 +74,14 @@ mod tests {
     use futures::StreamExt;
     use tokio_test::block_on;
 
+    use crate::testing::get_fake_data;
+
     /// Chunks and reconstructs a file.
     #[test]
     fn test_chunking_basic() {
         fn case(size: usize) {
             block_on(async move {
-                let test_file = get_data(size); // 32 MiB
+                let test_file = get_fake_data(size); // 32 MiB
                 let mut reconstructed_file = Vec::new();
 
                 let cursor = Cursor::new(&test_file);
@@ -99,18 +101,4 @@ mod tests {
         case(32 * 1024 * 1024);
         case(32 * 1024 * 1024 + 1);
     }
-
-    /// Returns some fake data.
-    fn get_data(len: usize) -> Vec<u8> {
-        let mut state = 42u32;
-        let mut data = vec![0u8; len];
-
-        for i in 0..data.len() {
-            (state, _) = state.overflowing_mul(1664525u32);
-            (state, _) = state.overflowing_add(1013904223u32);
-            data[i] = ((state >> (i % 24)) & 0xff) as u8;
-        }
-
-        data
-    }
 }
diff --git a/attic/src/testing/mod.rs b/attic/src/testing/mod.rs
index 191925e..393c31d 100644
--- a/attic/src/testing/mod.rs
+++ b/attic/src/testing/mod.rs
@@ -1,3 +1,27 @@
 //! Utilities for testing.
 
 pub mod shadow_store;
+
+use tokio::runtime::Runtime;
+
+/// Returns a new Tokio runtime.
+pub fn get_runtime() -> Runtime {
+    tokio::runtime::Builder::new_current_thread()
+        .enable_time()
+        .build()
+        .unwrap()
+}
+
+/// Returns some fake data.
+pub fn get_fake_data(len: usize) -> Vec<u8> {
+    let mut state = 42u32;
+    let mut data = vec![0u8; len];
+
+    for (i, byte) in data.iter_mut().enumerate() {
+        (state, _) = state.overflowing_mul(1664525u32);
+        (state, _) = state.overflowing_add(1013904223u32);
+        *byte = ((state >> (i % 24)) & 0xff) as u8;
+    }
+
+    data
+}
\ No newline at end of file

From 49c565f7923c61d6ebc40b16fd63c8442ccc3f39 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 09/56] Use tokio::test instead of tokio-test

---
 Cargo.lock                       |  14 ---
 attic/Cargo.toml                 |   2 +-
 attic/src/chunking/mod.rs        |  35 ++++---
 attic/src/nix_store/tests/mod.rs | 159 +++++++++++++++----------------
 attic/src/stream.rs              |  29 +++---
 server/Cargo.toml                |   3 -
 6 files changed, 105 insertions(+), 137 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4a04bbc..b56abab 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -334,7 +334,6 @@ dependencies = [
  "serde_with",
  "sha2",
  "tokio",
- "tokio-test",
  "tokio-util",
  "toml",
  "tower-http",
@@ -4748,19 +4747,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-test"
-version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7"
-dependencies = [
- "async-stream",
- "bytes",
- "futures-core",
- "tokio",
- "tokio-stream",
-]
-
 [[package]]
 name = "tokio-util"
 version = "0.7.11"
diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index 756519b..2f632e2 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -35,6 +35,7 @@ optional = true
 features = [
 	"fs",
 	"io-util",
+	"macros",
 	"process",
 	"sync",
 ]
@@ -43,7 +44,6 @@ features = [
 criterion = { version = "0.5", features = ["html_reports", "async_tokio"] }
 fastcdc = { version = "*", features = ["tokio"] }
 serde_json = "1.0.96"
-tokio-test = "0.4.2"
 
 [build-dependencies]
 cc = "1.1.13"
diff --git a/attic/src/chunking/mod.rs b/attic/src/chunking/mod.rs
index 031aa96..e054219 100644
--- a/attic/src/chunking/mod.rs
+++ b/attic/src/chunking/mod.rs
@@ -72,33 +72,30 @@ mod tests {
     use std::io::Cursor;
 
     use futures::StreamExt;
-    use tokio_test::block_on;
 
     use crate::testing::get_fake_data;
 
     /// Chunks and reconstructs a file.
-    #[test]
-    fn test_chunking_basic() {
-        fn case(size: usize) {
-            block_on(async move {
-                let test_file = get_fake_data(size); // 32 MiB
-                let mut reconstructed_file = Vec::new();
+    #[tokio::test]
+    async fn test_chunking_basic() {
+        async fn case(size: usize) {
+            let test_file = get_fake_data(size); // 32 MiB
+            let mut reconstructed_file = Vec::new();
 
-                let cursor = Cursor::new(&test_file);
-                let mut chunks = chunk_stream(cursor, 8 * 1024, 16 * 1024, 32 * 1024);
+            let cursor = Cursor::new(&test_file);
+            let mut chunks = chunk_stream(cursor, 8 * 1024, 16 * 1024, 32 * 1024);
 
-                while let Some(chunk) = chunks.next().await {
-                    let chunk = chunk.unwrap();
-                    eprintln!("Got a {}-byte chunk", chunk.len());
-                    reconstructed_file.extend(chunk);
-                }
+            while let Some(chunk) = chunks.next().await {
+                let chunk = chunk.unwrap();
+                eprintln!("Got a {}-byte chunk", chunk.len());
+                reconstructed_file.extend(chunk);
+            }
 
-                assert_eq!(reconstructed_file, test_file);
-            });
+            assert_eq!(reconstructed_file, test_file);
         }
 
-        case(32 * 1024 * 1024 - 1);
-        case(32 * 1024 * 1024);
-        case(32 * 1024 * 1024 + 1);
+        case(32 * 1024 * 1024 - 1).await;
+        case(32 * 1024 * 1024).await;
+        case(32 * 1024 * 1024 + 1).await;
     }
 }
diff --git a/attic/src/nix_store/tests/mod.rs b/attic/src/nix_store/tests/mod.rs
index 3d23922..ac5e2f9 100644
--- a/attic/src/nix_store/tests/mod.rs
+++ b/attic/src/nix_store/tests/mod.rs
@@ -6,7 +6,6 @@ use std::os::unix::ffi::OsStrExt;
 use std::process::Command;
 
 use serde::de::DeserializeOwned;
-use tokio_test::block_on;
 
 pub mod test_nar;
 
@@ -143,113 +142,105 @@ fn test_store_path_hash() {
     StorePathHash::new(h).unwrap_err();
 }
 
-#[test]
-fn test_nar_streaming() {
+#[tokio::test]
+async fn test_nar_streaming() {
     let store = NixStore::connect().expect("Failed to connect to the Nix store");
 
-    block_on(async move {
-        let test_nar = test_nar::NO_DEPS;
-        test_nar.import().await.expect("Could not import test NAR");
+    let test_nar = test_nar::NO_DEPS;
+    test_nar.import().await.expect("Could not import test NAR");
 
-        let target = test_nar.get_target().expect("Could not create dump target");
-        let writer = target.get_writer().await.expect("Could not get writer");
+    let target = test_nar.get_target().expect("Could not create dump target");
+    let writer = target.get_writer().await.expect("Could not get writer");
 
-        let store_path = store.parse_store_path(test_nar.path()).unwrap();
+    let store_path = store.parse_store_path(test_nar.path()).unwrap();
 
-        let stream = store.nar_from_path(store_path);
-        stream.write_all(writer).await.unwrap();
+    let stream = store.nar_from_path(store_path);
+    stream.write_all(writer).await.unwrap();
 
-        target
-            .validate()
-            .await
-            .expect("Could not validate resulting dump");
-    });
+    target
+        .validate()
+        .await
+        .expect("Could not validate resulting dump");
 }
 
-#[test]
-fn test_compute_fs_closure() {
+#[tokio::test]
+async fn test_compute_fs_closure() {
+    use test_nar::{WITH_DEPS_A, WITH_DEPS_B, WITH_DEPS_C};
+
     let store = NixStore::connect().expect("Failed to connect to the Nix store");
 
-    block_on(async move {
-        use test_nar::{WITH_DEPS_A, WITH_DEPS_B, WITH_DEPS_C};
+    for nar in [WITH_DEPS_C, WITH_DEPS_B, WITH_DEPS_A] {
+        nar.import().await.expect("Could not import test NAR");
 
-        for nar in [WITH_DEPS_C, WITH_DEPS_B, WITH_DEPS_A] {
-            nar.import().await.expect("Could not import test NAR");
-
-            let path = store
-                .parse_store_path(nar.path())
-                .expect("Could not parse store path");
-
-            let actual: HashSet<StorePath> = store
-                .compute_fs_closure(path, false, false, false)
-                .await
-                .expect("Could not compute closure")
-                .into_iter()
-                .collect();
-
-            assert_eq!(nar.closure(), actual);
-        }
-    });
-}
-
-#[test]
-fn test_compute_fs_closure_multi() {
-    let store = NixStore::connect().expect("Failed to connect to the Nix store");
-
-    block_on(async move {
-        use test_nar::{NO_DEPS, WITH_DEPS_A, WITH_DEPS_B, WITH_DEPS_C};
-
-        for nar in [NO_DEPS, WITH_DEPS_C, WITH_DEPS_B, WITH_DEPS_A] {
-            nar.import().await.expect("Could not import test NAR");
-        }
-
-        let mut expected = NO_DEPS.closure();
-        expected.extend(WITH_DEPS_A.closure());
-
-        let paths = vec![
-            store.parse_store_path(WITH_DEPS_A.path()).unwrap(),
-            store.parse_store_path(NO_DEPS.path()).unwrap(),
-        ];
+        let path = store
+            .parse_store_path(nar.path())
+            .expect("Could not parse store path");
 
         let actual: HashSet<StorePath> = store
-            .compute_fs_closure_multi(paths, false, false, false)
+            .compute_fs_closure(path, false, false, false)
             .await
             .expect("Could not compute closure")
             .into_iter()
             .collect();
 
-        eprintln!("Closure: {:#?}", actual);
-
-        assert_eq!(expected, actual);
-    });
+        assert_eq!(nar.closure(), actual);
+    }
 }
 
-#[test]
-fn test_query_path_info() {
+#[tokio::test]
+async fn test_compute_fs_closure_multi() {
+    use test_nar::{NO_DEPS, WITH_DEPS_A, WITH_DEPS_B, WITH_DEPS_C};
+
     let store = NixStore::connect().expect("Failed to connect to the Nix store");
 
-    block_on(async move {
-        use test_nar::{WITH_DEPS_B, WITH_DEPS_C};
+    for nar in [NO_DEPS, WITH_DEPS_C, WITH_DEPS_B, WITH_DEPS_A] {
+        nar.import().await.expect("Could not import test NAR");
+    }
 
-        for nar in [WITH_DEPS_C, WITH_DEPS_B] {
-            nar.import().await.expect("Could not import test NAR");
-        }
+    let mut expected = NO_DEPS.closure();
+    expected.extend(WITH_DEPS_A.closure());
 
-        let nar = WITH_DEPS_B;
-        let path = store.parse_store_path(nar.path()).unwrap();
-        let path_info = store
-            .query_path_info(path)
-            .await
-            .expect("Could not query path info");
+    let paths = vec![
+        store.parse_store_path(WITH_DEPS_A.path()).unwrap(),
+        store.parse_store_path(NO_DEPS.path()).unwrap(),
+    ];
 
-        eprintln!("Path info: {:?}", path_info);
+    let actual: HashSet<StorePath> = store
+        .compute_fs_closure_multi(paths, false, false, false)
+        .await
+        .expect("Could not compute closure")
+        .into_iter()
+        .collect();
 
-        assert_eq!(nar.nar().len() as u64, path_info.nar_size);
-        assert_eq!(
-            vec![PathBuf::from(
-                "3k1wymic8p7h5pfcqfhh0jan8ny2a712-attic-test-with-deps-c-final"
-            ),],
-            path_info.references
-        );
-    });
+    eprintln!("Closure: {:#?}", actual);
+
+    assert_eq!(expected, actual);
+}
+
+#[tokio::test]
+async fn test_query_path_info() {
+    use test_nar::{WITH_DEPS_B, WITH_DEPS_C};
+
+    let store = NixStore::connect().expect("Failed to connect to the Nix store");
+
+    for nar in [WITH_DEPS_C, WITH_DEPS_B] {
+        nar.import().await.expect("Could not import test NAR");
+    }
+
+    let nar = WITH_DEPS_B;
+    let path = store.parse_store_path(nar.path()).unwrap();
+    let path_info = store
+        .query_path_info(path)
+        .await
+        .expect("Could not query path info");
+
+    eprintln!("Path info: {:?}", path_info);
+
+    assert_eq!(nar.nar().len() as u64, path_info.nar_size);
+    assert_eq!(
+        vec![PathBuf::from(
+            "3k1wymic8p7h5pfcqfhh0jan8ny2a712-attic-test-with-deps-c-final"
+        ),],
+        path_info.references
+    );
 }
diff --git a/attic/src/stream.rs b/attic/src/stream.rs
index fede7b8..077b021 100644
--- a/attic/src/stream.rs
+++ b/attic/src/stream.rs
@@ -176,10 +176,9 @@ mod tests {
     use bytes::{BufMut, BytesMut};
     use futures::future;
     use tokio::io::AsyncReadExt;
-    use tokio_test::block_on;
 
-    #[test]
-    fn test_stream_hasher() {
+    #[tokio::test]
+    async fn test_stream_hasher() {
         let expected = b"hello world";
         let expected_sha256 =
             hex::decode("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9")
@@ -191,10 +190,10 @@ mod tests {
         // force multiple reads
         let mut buf = vec![0u8; 100];
         let mut bytes_read = 0;
-        bytes_read += block_on(read.read(&mut buf[bytes_read..bytes_read + 5])).unwrap();
-        bytes_read += block_on(read.read(&mut buf[bytes_read..bytes_read + 5])).unwrap();
-        bytes_read += block_on(read.read(&mut buf[bytes_read..bytes_read + 5])).unwrap();
-        bytes_read += block_on(read.read(&mut buf[bytes_read..bytes_read + 5])).unwrap();
+        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
+        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
+        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
+        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
 
         assert_eq!(expected.len(), bytes_read);
         assert_eq!(expected, &buf[..bytes_read]);
@@ -206,8 +205,8 @@ mod tests {
         eprintln!("finalized = {:x?}", finalized);
     }
 
-    #[test]
-    fn test_merge_chunks() {
+    #[tokio::test]
+    async fn test_merge_chunks() {
         let chunk_a: BoxStream<Result<Bytes, ()>> = {
             let s = stream! {
                 yield Ok(Bytes::from_static(b"Hello"));
@@ -236,13 +235,11 @@ mod tests {
         let streamer = |c, _| future::ok(c);
         let mut merged = merge_chunks(chunks, streamer, (), 2);
 
-        let bytes = block_on(async move {
-            let mut bytes = BytesMut::with_capacity(100);
-            while let Some(item) = merged.next().await {
-                bytes.put(item.unwrap());
-            }
-            bytes.freeze()
-        });
+        let mut bytes = BytesMut::with_capacity(100);
+        while let Some(item) = merged.next().await {
+            bytes.put(item.unwrap());
+        }
+        let bytes = bytes.freeze();
 
         assert_eq!(&*bytes, b"Hello, world!");
     }
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 3fcf108..977ba2d 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -94,6 +94,3 @@ features = [
 	"rt-multi-thread",
 	"sync",
 ]
-
-[dev-dependencies]
-tokio-test = "0.4.2"

From 7050d8f29262adb613011a67a1a590cedd004743 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 10/56] rustfmt, clippy

---
 attic/src/hash/mod.rs               | 2 +-
 attic/src/nix_store/bindings/mod.rs | 2 +-
 server/src/gc.rs                    | 8 ++++----
 server/src/storage/local.rs         | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/attic/src/hash/mod.rs b/attic/src/hash/mod.rs
index c717d4c..f68d4d0 100644
--- a/attic/src/hash/mod.rs
+++ b/attic/src/hash/mod.rs
@@ -118,7 +118,7 @@ impl Serialize for Hash {
 }
 
 /// Decodes a base16 or base32 encoded hash containing a specified number of bytes.
-fn decode_hash<'s>(s: &'s str, typ: &'static str, expected_bytes: usize) -> AtticResult<Vec<u8>> {
+fn decode_hash(s: &str, typ: &'static str, expected_bytes: usize) -> AtticResult<Vec<u8>> {
     let base16_len = expected_bytes * 2;
     let base32_len = (expected_bytes * 8 - 1) / 5 + 1;
 
diff --git a/attic/src/nix_store/bindings/mod.rs b/attic/src/nix_store/bindings/mod.rs
index 754afcc..a16b52a 100644
--- a/attic/src/nix_store/bindings/mod.rs
+++ b/attic/src/nix_store/bindings/mod.rs
@@ -18,7 +18,7 @@ unsafe impl Send for FfiNixStore {}
 unsafe impl Sync for FfiNixStore {}
 
 impl FfiNixStore {
-    pub fn store<'a>(&'a self) -> Pin<&'a mut ffi::CNixStore> {
+    pub fn store(&self) -> Pin<&mut ffi::CNixStore> {
         unsafe {
             let ptr = self.0.get().as_mut().unwrap();
             ptr.pin_mut()
diff --git a/server/src/gc.rs b/server/src/gc.rs
index a06bdd9..d2332fe 100644
--- a/server/src/gc.rs
+++ b/server/src/gc.rs
@@ -159,12 +159,12 @@ async fn run_reap_orphan_chunks(state: &State) -> Result<()> {
     let storage = state.storage().await?;
 
     let orphan_chunk_limit = match db.get_database_backend() {
-        // Arbitrarily chosen sensible value since there's no good default to choose from for MySQL 
-        sea_orm::DatabaseBackend::MySql    => 1000,
+        // Arbitrarily chosen sensible value since there's no good default to choose from for MySQL
+        sea_orm::DatabaseBackend::MySql => 1000,
         // Panic limit set by sqlx for postgresql: https://github.com/launchbadge/sqlx/issues/671#issuecomment-687043510
         sea_orm::DatabaseBackend::Postgres => u64::from(u16::MAX),
-        // Default statement limit imposed by sqlite: https://www.sqlite.org/limits.html#max_variable_number 
-        sea_orm::DatabaseBackend::Sqlite   => 500,
+        // Default statement limit imposed by sqlite: https://www.sqlite.org/limits.html#max_variable_number
+        sea_orm::DatabaseBackend::Sqlite => 500,
     };
 
     // find all orphan chunks...
diff --git a/server/src/storage/local.rs b/server/src/storage/local.rs
index db9b88b..e35dc0a 100644
--- a/server/src/storage/local.rs
+++ b/server/src/storage/local.rs
@@ -79,8 +79,8 @@ async fn upgrade_0_to_1(storage_path: &Path) -> ServerResult<()> {
             let name = file.file_name();
             let name_bytes = name.as_os_str().as_bytes();
             let parents = storage_path
-              .join(OsStr::from_bytes(&name_bytes[0..1]))
-              .join(OsStr::from_bytes(&name_bytes[0..2]));
+                .join(OsStr::from_bytes(&name_bytes[0..1]))
+                .join(OsStr::from_bytes(&name_bytes[0..2]));
             let new_path = parents.join(name);
             fs::create_dir_all(&parents).await.map_err(|e| {
                 ErrorKind::StorageError(anyhow::anyhow!("Failed to create directory {}", e))

From 7401b14e71c6e308ba6a7b27fcc01bcaa395f375 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 11/56] flake.nix: Add WebAssembly tools

---
 flake.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flake.nix b/flake.nix
index 717e582..10cadea 100644
--- a/flake.nix
+++ b/flake.nix
@@ -138,6 +138,9 @@
           flyctl
 
           wrk
+
+          llvmPackages_latest.bintools
+          wrangler worker-build wasm-pack wasm-bindgen-cli
         ] ++ (lib.optionals pkgs.stdenv.isLinux [
           linuxPackages.perf
         ]);

From c7b1f7195474c05f1f48773ce49846d3d5d5cfbc Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 12/56] attic: Activate unix tokio features for nix_store only

---
 attic/Cargo.toml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index 2f632e2..4714ca7 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -33,10 +33,8 @@ cxx = { version = "1.0", optional = true }
 version = "1.28.2"
 optional = true
 features = [
-	"fs",
 	"io-util",
 	"macros",
-	"process",
 	"sync",
 ]
 
@@ -61,12 +59,18 @@ default = [
 ]
 
 # Chunking.
-chunking = ["tokio", "dep:async-stream"]
+chunking = ["tokio", "stream", "dep:async-stream"]
 
 # Native libnixstore bindings.
 #
 # When disabled, the native Rust portions of nix_store can still be used.
-nix_store = ["tokio", "dep:cxx", "dep:cxx-build"]
+nix_store = [
+	"tokio",
+	"tokio/fs",
+	"tokio/process",
+	"dep:cxx",
+	"dep:cxx-build",
+]
 
 # Stream utilities.
 stream = ["tokio", "dep:async-stream"]

From 93cab5268d02a803e2b4610ef2eca0967edd20b9 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 13/56] .github/build: Add step to build certain crates in
 WebAssembly

---
 .github/workflows/build.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index af099f7..c1b9bb7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -52,6 +52,19 @@ jobs:
           tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L)
           find "$tests/bin" -exec {} \;
 
+      - name: Build WebAssembly crates
+        if: runner.os == 'Linux'
+        run: |
+          # https://github.com/rust-lang/rust/issues/122357
+          export RUST_MIN_STACK=16777216
+
+          pushd attic
+          nix develop .# --command -- cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream
+          popd
+          pushd token
+          nix develop .# --command -- cargo build --target wasm32-unknown-unknown
+          popd
+
       # TODO: Just take a diff of the list of store paths, also abstract all of this out
       - name: Push build artifacts
         run: |

From bb4135c1e0914fef461ecefc23f271730e57f2ae Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 19 Aug 2024 14:49:56 -0400
Subject: [PATCH 14/56] integration-tests: Fix deprecated alias

---
 integration-tests/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/default.nix b/integration-tests/default.nix
index b5a1f4a..89dddfb 100644
--- a/integration-tests/default.nix
+++ b/integration-tests/default.nix
@@ -33,6 +33,6 @@ let
         }
       ];
     };
-  }) (lib.cartesianProductOfSets matrix));
+  }) (lib.cartesianProduct matrix));
 in {
 } // basicTests

From 0f9506ffba106471db0a88f700cb405d2ad153e3 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sat, 24 Aug 2024 18:56:18 -0400
Subject: [PATCH 15/56] Cargo.lock: Fix

---
 Cargo.lock | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index b56abab..fb57a09 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -258,7 +258,6 @@ dependencies = [
  "sha2",
  "tempfile",
  "tokio",
- "tokio-test",
  "version-compare",
  "wildmatch",
  "xdg",

From b4eb905953961816cfd1d7e8d5093be02c9cf689 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sat, 24 Aug 2024 18:56:18 -0400
Subject: [PATCH 16/56] attic/Cargo.toml: Add missing tokio feature for
 crate-level build

---
 attic/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index 4714ca7..b689450 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -76,7 +76,7 @@ nix_store = [
 stream = ["tokio", "dep:async-stream"]
 
 # Tokio runtime.
-tokio = ["dep:tokio", "tokio/rt"]
+tokio = ["dep:tokio", "tokio/rt", "tokio/time"]
 
 [[bench]]
 name = "chunking"

From b92e5ba4b55eab414b7a0bb0238435ed944a1488 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sat, 24 Aug 2024 18:56:18 -0400
Subject: [PATCH 17/56] attic: Update nix-base32 crate

---
 Cargo.lock       | 5 +++--
 attic/Cargo.toml | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index fb57a09..53a47e0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2778,8 +2778,9 @@ dependencies = [
 
 [[package]]
 name = "nix-base32"
-version = "0.1.2-alpha.0"
-source = "git+https://github.com/zhaofengli/nix-base32.git?rev=b850c6e9273d1c39bd93abb704a53345f5be92eb#b850c6e9273d1c39bd93abb704a53345f5be92eb"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2628953ed836273ee4262e3708a8ef63ca38bd8a922070626eef7f9e5d8d536"
 
 [[package]]
 name = "nom"
diff --git a/attic/Cargo.toml b/attic/Cargo.toml
index b689450..b682a52 100644
--- a/attic/Cargo.toml
+++ b/attic/Cargo.toml
@@ -16,7 +16,7 @@ futures = "0.3.28"
 hex = "0.4.3"
 lazy_static = "1.4.0"
 log = "0.4.18"
-nix-base32 = { git = "https://github.com/zhaofengli/nix-base32.git", rev = "b850c6e9273d1c39bd93abb704a53345f5be92eb" }
+nix-base32 = "0.2.0"
 regex = "1.8.3"
 serde = { version = "1.0.163", features = ["derive"] }
 serde_yaml = "0.9.21"

From e81151ed59f73b23101ca57e35c1032aa51f0774 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sat, 24 Aug 2024 19:00:50 -0400
Subject: [PATCH 18/56] flake.nix: Use default Nix headers in devShell

---
 flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 10cadea..be10223 100644
--- a/flake.nix
+++ b/flake.nix
@@ -149,7 +149,7 @@
         RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
 
         # See comment in `attic/build.rs`
-        NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include";
+        NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include";
 
         ATTIC_DISTRIBUTOR = "dev";
       };

From fcb7c6067f42fa76d2a7cfaba2df8376b354c7ff Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 19/56] flake: Bump nixpkgs-stable to 24.05

---
 flake.lock | 8 ++++----
 flake.nix  | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/flake.lock b/flake.lock
index e72975b..437ecf1 100644
--- a/flake.lock
+++ b/flake.lock
@@ -72,16 +72,16 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1720535198,
-        "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=",
+        "lastModified": 1724316499,
+        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5",
+        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
diff --git a/flake.nix b/flake.nix
index be10223..708eece 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,7 +3,7 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
-    nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-23.11";
+    nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
     flake-utils.url = "github:numtide/flake-utils";
 
     crane = {

From 7712ed2c7f85dc4c632ba38554d710bdf6c5953c Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 20/56] flake: Bump unstable

---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 437ecf1..e38f188 100644
--- a/flake.lock
+++ b/flake.lock
@@ -56,11 +56,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1723827930,
-        "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=",
+        "lastModified": 1724999960,
+        "narHash": "sha256-LB3jqSGW5u1ZcUcX6vO/qBOq5oXHlmOCxsTXGMEitp4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d",
+        "rev": "b96f849e725333eb2b1c7f1cb84ff102062468ba",
         "type": "github"
       },
       "original": {

From fd6e3bdbefbc014eb4eb68f213269b298e118f6c Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 21/56] Add CI-agnostic helper scripts

---
 .ci/.gitignore     |  1 +
 .ci/cache-shell.sh |  5 +++++
 .ci/common.sh      |  7 +++++++
 .ci/run            | 10 ++++++++++
 4 files changed, 23 insertions(+)
 create mode 100644 .ci/.gitignore
 create mode 100755 .ci/cache-shell.sh
 create mode 100644 .ci/common.sh
 create mode 100755 .ci/run

diff --git a/.ci/.gitignore b/.ci/.gitignore
new file mode 100644
index 0000000..1fd44ac
--- /dev/null
+++ b/.ci/.gitignore
@@ -0,0 +1 @@
+/cached-shell
diff --git a/.ci/cache-shell.sh b/.ci/cache-shell.sh
new file mode 100755
index 0000000..b810324
--- /dev/null
+++ b/.ci/cache-shell.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+>&2 echo "Caching dev shell"
+nix print-dev-env "${base}#" >"${cached_shell}"
diff --git a/.ci/common.sh b/.ci/common.sh
new file mode 100644
index 0000000..1c8d67c
--- /dev/null
+++ b/.ci/common.sh
@@ -0,0 +1,7 @@
+# Use as:
+#
+# source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+set -euo pipefail
+base="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
+cached_shell="${base}/.ci/cached-shell"
diff --git a/.ci/run b/.ci/run
new file mode 100755
index 0000000..67b60b0
--- /dev/null
+++ b/.ci/run
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+if [[ ! -f "${cached_shell}" ]]; then
+	>&2 echo "No cached shell in ${cached_shell}"
+	exit 1
+fi
+
+. "${cached_shell}"
+exec "$@"

From 54f4854e9f5521833dad72165e6a19ecf858b5b7 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 22/56] Start migration to flake-parts

---
 flake.lock            |  21 +++
 flake.nix             | 298 +++++++++++++++++++-----------------------
 flake/devshells.nix   | 111 ++++++++++++++++
 flake/distributor.nix |  15 +++
 4 files changed, 284 insertions(+), 161 deletions(-)
 create mode 100644 flake/devshells.nix
 create mode 100644 flake/distributor.nix

diff --git a/flake.lock b/flake.lock
index e38f188..1e31e3b 100644
--- a/flake.lock
+++ b/flake.lock
@@ -36,6 +36,26 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": "systems"
@@ -90,6 +110,7 @@
       "inputs": {
         "crane": "crane",
         "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
diff --git a/flake.nix b/flake.nix
index 708eece..2b1fa3e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -6,6 +6,11 @@
     nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
     flake-utils.url = "github:numtide/flake-utils";
 
+    flake-parts = {
+      url = "github:hercules-ci/flake-parts";
+      inputs.nixpkgs-lib.follows = "nixpkgs";
+    };
+
     crane = {
       url = "github:ipetkov/crane";
       inputs.nixpkgs.follows = "nixpkgs";
@@ -17,189 +22,160 @@
     };
   };
 
-  outputs = { self, nixpkgs, nixpkgs-stable, flake-utils, crane, ... }: let
-    supportedSystems = flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];
+  outputs = inputs @ { self, flake-parts, ... }: let
+    supportedSystems = inputs.flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];
+
+    inherit (inputs.nixpkgs) lib;
 
     makeCranePkgs = pkgs: let
-      craneLib = crane.mkLib pkgs;
+      craneLib = inputs.crane.mkLib pkgs;
     in pkgs.callPackage ./crane.nix { inherit craneLib; };
-  in flake-utils.lib.eachSystem supportedSystems (system: let
-    pkgs = import nixpkgs {
-      inherit system;
-      overlays = [];
-    };
-    cranePkgs = makeCranePkgs pkgs;
 
-    internalMatrix = lib.mapAttrs (_: nix: let
-      cranePkgs' = cranePkgs.override { inherit nix; };
-    in {
-      inherit (cranePkgs') attic-tests cargoArtifacts;
-    }) {
-      "2.20" = pkgs.nixVersions.nix_2_20;
-      "2.24" = pkgs.nixVersions.nix_2_24;
-      "default" = pkgs.nix;
-    };
+    modules = builtins.foldl' (acc: f: f acc) ./flake [
+      builtins.readDir
+      (lib.filterAttrs (name: type:
+        type == "regular" && lib.hasSuffix ".nix" name
+      ))
+      (lib.mapAttrsToList (name: _:
+        lib.path.append ./flake name
+      ))
+    ];
 
-    pkgsStable = import nixpkgs-stable {
-      inherit system;
-      overlays = [];
-    };
-    cranePkgsStable = makeCranePkgs pkgsStable;
+  in flake-parts.lib.mkFlake { inherit inputs; } {
+    imports = modules;
+    systems = supportedSystems;
 
-    inherit (pkgs) lib;
-  in rec {
-    inherit internalMatrix;
+    debug = true;
 
-    packages = {
-      default = packages.attic;
+    # old flake
+    flake = inputs.flake-utils.lib.eachSystem supportedSystems (system: let
+      pkgs = import inputs.nixpkgs {
+        inherit system;
+        overlays = [];
+      };
+      cranePkgs = makeCranePkgs pkgs;
 
-      inherit (cranePkgs) attic attic-client attic-server;
-
-      attic-nixpkgs = pkgs.callPackage ./package.nix { };
-
-      attic-ci-installer = pkgs.callPackage ./ci-installer.nix {
-        inherit self;
+      internalMatrix = lib.mapAttrs (_: nix: let
+        cranePkgs' = cranePkgs.override { inherit nix; };
+      in {
+        inherit (cranePkgs') attic-tests cargoArtifacts;
+      }) {
+        "2.20" = pkgs.nixVersions.nix_2_20;
+        "2.24" = pkgs.nixVersions.nix_2_24;
+        "default" = pkgs.nix;
       };
 
-      book = pkgs.callPackage ./book {
-        attic = packages.attic;
+      pkgsStable = import inputs.nixpkgs-stable {
+        inherit system;
+        overlays = [];
       };
-    } // (lib.optionalAttrs (system != "x86_64-darwin") {
-      # Unfortunately, x86_64-darwin fails to evaluate static builds
-      # TODO: Make this work with Crane
-      attic-static = (pkgs.pkgsStatic.callPackage ./package.nix {
-        nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
-          patches = (old.patches or []) ++ [
-            # To be submitted
-            (pkgs.fetchpatch {
-              url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
-              hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
-            })
+      cranePkgsStable = makeCranePkgs pkgsStable;
+
+      inherit (pkgs) lib;
+    in rec {
+      inherit internalMatrix;
+
+      packages = {
+        default = packages.attic;
+
+        inherit (cranePkgs) attic attic-client attic-server;
+
+        attic-nixpkgs = pkgs.callPackage ./package.nix { };
+
+        attic-ci-installer = pkgs.callPackage ./ci-installer.nix {
+          inherit self;
+        };
+
+        book = pkgs.callPackage ./book {
+          attic = packages.attic;
+        };
+      } // (lib.optionalAttrs (system != "x86_64-darwin") {
+        # Unfortunately, x86_64-darwin fails to evaluate static builds
+        # TODO: Make this work with Crane
+        attic-static = (pkgs.pkgsStatic.callPackage ./package.nix {
+          nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
+            patches = (old.patches or []) ++ [
+              # To be submitted
+              (pkgs.fetchpatch {
+                url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
+                hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
+              })
+            ];
+          });
+        }).overrideAttrs (old: {
+          nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
+            pkgs.nukeReferences
           ];
+
+          # Read by pkg_config crate (do some autodetection in build.rs?)
+          PKG_CONFIG_ALL_STATIC = "1";
+
+          "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
+          RUSTFLAGS = "-C relocation-model=static";
+
+          postFixup = (old.postFixup or "") + ''
+            rm -f $out/nix-support/propagated-build-inputs
+            nuke-refs $out/bin/attic
+          '';
         });
-      }).overrideAttrs (old: {
-        nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
-          pkgs.nukeReferences
-        ];
 
-        # Read by pkg_config crate (do some autodetection in build.rs?)
-        PKG_CONFIG_ALL_STATIC = "1";
+        attic-client-static = packages.attic-static.override {
+          clientOnly = true;
+        };
+      }) // (lib.optionalAttrs pkgs.stdenv.isLinux {
+        attic-server-image = pkgs.dockerTools.buildImage {
+          name = "attic-server";
+          tag = "main";
+          copyToRoot = [
+            # Debugging utilities for `fly ssh console`
+            pkgs.busybox
+            packages.attic-server
 
-        "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
-        RUSTFLAGS = "-C relocation-model=static";
-
-        postFixup = (old.postFixup or "") + ''
-          rm -f $out/nix-support/propagated-build-inputs
-          nuke-refs $out/bin/attic
-        '';
+            # Now required by the fly.io sshd
+            pkgs.dockerTools.fakeNss
+          ];
+          config = {
+            Entrypoint = [ "${packages.attic-server}/bin/atticd" ];
+            Env = [
+              "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+            ];
+          };
+        };
       });
 
-      attic-client-static = packages.attic-static.override {
-        clientOnly = true;
+      checks = let
+        makeIntegrationTests = pkgs: import ./integration-tests {
+          pkgs = import inputs.nixpkgs {
+            inherit system;
+            overlays = [ self.overlays.default ];
+          };
+          flake = self;
+        };
+        unstableTests = makeIntegrationTests pkgs;
+        stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
+      in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
+    }) // {
+      overlays = {
+        default = final: prev: let
+          cranePkgs = makeCranePkgs final;
+        in {
+          inherit (cranePkgs) attic attic-client attic-server;
+        };
       };
-    }) // (lib.optionalAttrs pkgs.stdenv.isLinux {
-      attic-server-image = pkgs.dockerTools.buildImage {
-        name = "attic-server";
-        tag = "main";
-        copyToRoot = [
-          # Debugging utilities for `fly ssh console`
-          pkgs.busybox
-          packages.attic-server
 
-          # Now required by the fly.io sshd
-          pkgs.dockerTools.fakeNss
-        ];
-        config = {
-          Entrypoint = [ "${packages.attic-server}/bin/atticd" ];
-          Env = [
-            "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+      nixosModules = {
+        atticd = {
+          imports = [
+            ./nixos/atticd.nix
+          ];
+
+          services.atticd.useFlakeCompatOverlay = false;
+
+          nixpkgs.overlays = [
+            self.overlays.default
           ];
         };
       };
-    });
-
-    devShells = {
-      default = pkgs.mkShell {
-        inputsFrom = with packages; [ attic book ];
-        nativeBuildInputs = with pkgs; [
-          rustc
-
-          rustfmt clippy
-          cargo-expand
-          # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152
-          # cargo-outdated
-          cargo-edit
-          tokio-console
-
-          sqlite-interactive
-
-          editorconfig-checker
-
-          flyctl
-
-          wrk
-
-          llvmPackages_latest.bintools
-          wrangler worker-build wasm-pack wasm-bindgen-cli
-        ] ++ (lib.optionals pkgs.stdenv.isLinux [
-          linuxPackages.perf
-        ]);
-
-        NIX_PATH = "nixpkgs=${pkgs.path}";
-        RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
-
-        # See comment in `attic/build.rs`
-        NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include";
-
-        ATTIC_DISTRIBUTOR = "dev";
-      };
-
-      demo = pkgs.mkShell {
-        nativeBuildInputs = [
-          packages.default
-        ];
-
-        shellHook = ''
-          >&2 echo
-          >&2 echo '🚀 Run `atticd` to get started!'
-          >&2 echo
-        '';
-      };
-    };
-    devShell = devShells.default;
-
-    checks = let
-      makeIntegrationTests = pkgs: import ./integration-tests {
-        pkgs = import nixpkgs {
-          inherit system;
-          overlays = [ self.overlays.default ];
-        };
-        flake = self;
-      };
-      unstableTests = makeIntegrationTests pkgs;
-      stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
-    in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
-  }) // {
-    overlays = {
-      default = final: prev: let
-        cranePkgs = makeCranePkgs final;
-      in {
-        inherit (cranePkgs) attic attic-client attic-server;
-      };
-    };
-
-    nixosModules = {
-      atticd = {
-        imports = [
-          ./nixos/atticd.nix
-        ];
-
-        services.atticd.useFlakeCompatOverlay = false;
-
-        nixpkgs.overlays = [
-          self.overlays.default
-        ];
-      };
     };
   };
 }
diff --git a/flake/devshells.nix b/flake/devshells.nix
new file mode 100644
index 0000000..b71a0bd
--- /dev/null
+++ b/flake/devshells.nix
@@ -0,0 +1,111 @@
+# Development shells
+
+toplevel @ { lib, flake-parts-lib, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+  inherit (flake-parts-lib)
+    mkPerSystemOption
+    ;
+in
+{
+  options = {
+    perSystem = mkPerSystemOption {
+      options.attic.devshell = {
+        packageSets = mkOption {
+          type = types.attrsOf (types.listOf types.package);
+          default = {};
+        };
+        extraPackages = mkOption {
+          type = types.listOf types.package;
+          default = [];
+        };
+        extraArgs = mkOption {
+          type = types.attrsOf types.unspecified;
+          default = {};
+        };
+      };
+    };
+  };
+
+  config = {
+    perSystem = { self', pkgs, config, ... }: let
+      cfg = config.attic.devshell;
+    in {
+      attic.devshell.packageSets = with pkgs; {
+        rust = [
+          rustc
+
+          cargo-expand
+          # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152
+          # cargo-outdated
+          cargo-edit
+          tokio-console
+        ];
+
+        linters = [
+          clippy
+          rustfmt
+
+          editorconfig-checker
+        ];
+
+        utils = [
+          jq
+          just
+        ];
+
+        ops = [
+          postgresql
+          sqlite-interactive
+
+          flyctl
+          wrangler
+        ];
+
+        bench = [
+          wrk
+        ] ++ lib.optionals pkgs.stdenv.isLinux [
+          linuxPackages.perf
+        ];
+
+        wasm = [
+          llvmPackages_latest.bintools
+          worker-build wasm-pack wasm-bindgen-cli
+        ];
+      };
+
+      devShells.default = pkgs.mkShell (lib.recursiveUpdate {
+        inputsFrom = [
+          self'.packages.attic
+          self'.packages.book
+        ];
+
+        packages = lib.flatten (lib.attrValues cfg.packageSets);
+
+        env = {
+          ATTIC_DISTRIBUTOR = toplevel.config.attic.distributor;
+
+          RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
+
+          NIX_PATH = "nixpkgs=${pkgs.path}";
+
+          # See comment in `attic/build.rs`
+          NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include";
+        };
+      } cfg.extraArgs);
+
+      devShells.demo = pkgs.mkShell {
+        packages = [ self'.packages.default ];
+
+        shellHook = ''
+          >&2 echo
+          >&2 echo '🚀 Run `atticd` to get started!'
+          >&2 echo
+        '';
+      };
+    };
+  };
+}
diff --git a/flake/distributor.nix b/flake/distributor.nix
new file mode 100644
index 0000000..7b40cb0
--- /dev/null
+++ b/flake/distributor.nix
@@ -0,0 +1,15 @@
+{ lib, flake-parts-lib, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+in
+{
+  options = {
+    attic.distributor = mkOption {
+      type = types.str;
+      default = "dev";
+    };
+  };
+}

From 11163ab253b12733437b1bde06bc2441c903173b Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 23/56] Migrate packages to flake-parts

---
 flake.nix          | 68 --------------------------------
 flake/packages.nix | 96 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+), 68 deletions(-)
 create mode 100644 flake/packages.nix

diff --git a/flake.nix b/flake.nix
index 2b1fa3e..8d0ceca 100644
--- a/flake.nix
+++ b/flake.nix
@@ -75,74 +75,6 @@
     in rec {
       inherit internalMatrix;
 
-      packages = {
-        default = packages.attic;
-
-        inherit (cranePkgs) attic attic-client attic-server;
-
-        attic-nixpkgs = pkgs.callPackage ./package.nix { };
-
-        attic-ci-installer = pkgs.callPackage ./ci-installer.nix {
-          inherit self;
-        };
-
-        book = pkgs.callPackage ./book {
-          attic = packages.attic;
-        };
-      } // (lib.optionalAttrs (system != "x86_64-darwin") {
-        # Unfortunately, x86_64-darwin fails to evaluate static builds
-        # TODO: Make this work with Crane
-        attic-static = (pkgs.pkgsStatic.callPackage ./package.nix {
-          nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
-            patches = (old.patches or []) ++ [
-              # To be submitted
-              (pkgs.fetchpatch {
-                url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
-                hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
-              })
-            ];
-          });
-        }).overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
-            pkgs.nukeReferences
-          ];
-
-          # Read by pkg_config crate (do some autodetection in build.rs?)
-          PKG_CONFIG_ALL_STATIC = "1";
-
-          "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
-          RUSTFLAGS = "-C relocation-model=static";
-
-          postFixup = (old.postFixup or "") + ''
-            rm -f $out/nix-support/propagated-build-inputs
-            nuke-refs $out/bin/attic
-          '';
-        });
-
-        attic-client-static = packages.attic-static.override {
-          clientOnly = true;
-        };
-      }) // (lib.optionalAttrs pkgs.stdenv.isLinux {
-        attic-server-image = pkgs.dockerTools.buildImage {
-          name = "attic-server";
-          tag = "main";
-          copyToRoot = [
-            # Debugging utilities for `fly ssh console`
-            pkgs.busybox
-            packages.attic-server
-
-            # Now required by the fly.io sshd
-            pkgs.dockerTools.fakeNss
-          ];
-          config = {
-            Entrypoint = [ "${packages.attic-server}/bin/atticd" ];
-            Env = [
-              "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
-            ];
-          };
-        };
-      });
-
       checks = let
         makeIntegrationTests = pkgs: import ./integration-tests {
           pkgs = import inputs.nixpkgs {
diff --git a/flake/packages.nix b/flake/packages.nix
new file mode 100644
index 0000000..c8fdf95
--- /dev/null
+++ b/flake/packages.nix
@@ -0,0 +1,96 @@
+{ self, inputs, lib, ... }:
+let
+  makeCranePkgs = pkgs: let
+    craneLib = inputs.crane.mkLib pkgs;
+  in pkgs.callPackage ../crane.nix { inherit craneLib; };
+in
+{
+  _module.args.makeCranePkgs = makeCranePkgs;
+
+  perSystem = { self', pkgs, cranePkgs, ... }: (lib.mkMerge [
+    {
+      _module.args.cranePkgs = makeCranePkgs pkgs;
+
+      packages = {
+        default = self'.packages.attic;
+
+        inherit (cranePkgs)
+          attic
+          attic-client
+          attic-server
+        ;
+
+        attic-nixpkgs = pkgs.callPackage ../package.nix { };
+
+        attic-ci-installer = pkgs.callPackage ../ci-installer.nix {
+          inherit self;
+        };
+
+        book = pkgs.callPackage ../book {
+          attic = self'.packages.attic;
+        };
+      };
+    }
+
+    (lib.mkIf pkgs.stdenv.isLinux {
+      packages = {
+        attic-server-image = pkgs.dockerTools.buildImage {
+          name = "attic-server";
+          tag = "main";
+          copyToRoot = [
+            self'.packages.attic-server
+
+            # Debugging utilities for `fly ssh console`
+            pkgs.busybox
+
+            # Now required by the fly.io sshd
+            pkgs.dockerTools.fakeNss
+          ];
+          config = {
+            Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ];
+            Env = [
+              "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+            ];
+          };
+        };
+      };
+    })
+
+    # Unfortunately, x86_64-darwin fails to evaluate static builds
+    (lib.mkIf (pkgs.system != "x86_64-darwin") {
+      packages = {
+        # TODO: Make this work with Crane
+        attic-static = (pkgs.pkgsStatic.callPackage ../package.nix {
+          nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
+            patches = (old.patches or []) ++ [
+              # To be submitted
+              (pkgs.fetchpatch {
+                url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
+                hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
+              })
+            ];
+          });
+        }).overrideAttrs (old: {
+          nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
+            pkgs.nukeReferences
+          ];
+
+          # Read by pkg_config crate (do some autodetection in build.rs?)
+          PKG_CONFIG_ALL_STATIC = "1";
+
+          "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
+          RUSTFLAGS = "-C relocation-model=static";
+
+          postFixup = (old.postFixup or "") + ''
+            rm -f $out/nix-support/propagated-build-inputs
+            nuke-refs $out/bin/attic
+          '';
+        });
+
+        attic-client-static = self'.packages.attic-static.override {
+          clientOnly = true;
+        };
+      };
+    })
+  ]);
+}

From 8dc0bdbf7f18b816ebc6945bcf90ae296f2ad0a0 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 24/56] Migrate overlays to flake-parts

---
 flake.nix          |  8 --------
 flake/overlays.nix | 14 ++++++++++++++
 2 files changed, 14 insertions(+), 8 deletions(-)
 create mode 100644 flake/overlays.nix

diff --git a/flake.nix b/flake.nix
index 8d0ceca..f333523 100644
--- a/flake.nix
+++ b/flake.nix
@@ -87,14 +87,6 @@
         stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
       in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
     }) // {
-      overlays = {
-        default = final: prev: let
-          cranePkgs = makeCranePkgs final;
-        in {
-          inherit (cranePkgs) attic attic-client attic-server;
-        };
-      };
-
       nixosModules = {
         atticd = {
           imports = [
diff --git a/flake/overlays.nix b/flake/overlays.nix
new file mode 100644
index 0000000..8cd8bef
--- /dev/null
+++ b/flake/overlays.nix
@@ -0,0 +1,14 @@
+{ makeCranePkgs, ... }:
+{
+  flake.overlays = {
+    default = final: prev: let
+      cranePkgs = makeCranePkgs final;
+    in {
+      inherit (cranePkgs)
+        attic
+        attic-client
+        attic-server
+        ;
+    };
+  };
+}

From 3ecea8d07fcbc0272d2a3dd4fcbf1fce46cdcf12 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 25/56] Migrate NixOS modules to flake-parts

---
 flake.nix       | 13 -------------
 flake/nixos.nix | 16 ++++++++++++++++
 2 files changed, 16 insertions(+), 13 deletions(-)
 create mode 100644 flake/nixos.nix

diff --git a/flake.nix b/flake.nix
index f333523..1abf156 100644
--- a/flake.nix
+++ b/flake.nix
@@ -87,19 +87,6 @@
         stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
       in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
     }) // {
-      nixosModules = {
-        atticd = {
-          imports = [
-            ./nixos/atticd.nix
-          ];
-
-          services.atticd.useFlakeCompatOverlay = false;
-
-          nixpkgs.overlays = [
-            self.overlays.default
-          ];
-        };
-      };
     };
   };
 }
diff --git a/flake/nixos.nix b/flake/nixos.nix
new file mode 100644
index 0000000..b8141c0
--- /dev/null
+++ b/flake/nixos.nix
@@ -0,0 +1,16 @@
+{ config, ... }:
+{
+  flake.nixosModules = {
+    atticd = {
+      imports = [
+        ../nixos/atticd.nix
+      ];
+
+      services.atticd.useFlakeCompatOverlay = false;
+
+      nixpkgs.overlays = [
+        config.flake.overlays.default
+      ];
+    };
+  };
+}

From 09038b7663543cfb75f1fb2fe9c8fcd5796f8a22 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 26/56] flake: Add attic.nix-versions

---
 flake/nix-versions.nix | 52 ++++++++++++++++++++++++++++++++++++++++++
 justfile               | 19 +++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 flake/nix-versions.nix
 create mode 100644 justfile

diff --git a/flake/nix-versions.nix b/flake/nix-versions.nix
new file mode 100644
index 0000000..66f3697
--- /dev/null
+++ b/flake/nix-versions.nix
@@ -0,0 +1,52 @@
+{ lib, flake-parts-lib, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+  inherit (flake-parts-lib)
+    mkPerSystemOption
+    ;
+in
+{
+  options = {
+    perSystem = mkPerSystemOption {
+      options.attic.nix-versions = {
+        versions = mkOption {
+          type = types.attrsOf types.package;
+          default = {};
+        };
+        manifestFile = mkOption {
+          type = types.package;
+        };
+      };
+    };
+  };
+
+  config = {
+    perSystem = { self', pkgs, config, ... }: let
+      cfg = config.attic.nix-versions;
+    in {
+      attic.nix-versions = {
+        versions = {
+          default = pkgs.nix;
+          "2.20" = pkgs.nixVersions.nix_2_20;
+          "2.24" = pkgs.nixVersions.nix_2_24;
+        };
+
+        manifestFile = let
+          manifest = lib.mapAttrs (_: nix: {
+            inherit nix;
+            shellHook = ''
+              export NIX_INCLUDE_PATH="${lib.getDev nix}/include"
+              export NIX_CFLAGS_COMPILE="-isystem $NIX_INCLUDE_PATH $NIX_CFLAGS_COMPILE"
+              export NIX_LDFLAGS="-L${nix}/lib $NIX_LDFLAGS"
+              export PKG_CONFIG_PATH="${lib.getDev nix}/lib/pkgconfig:$PKG_CONFIG_PATH"
+              export PATH="${lib.getBin nix}/bin:$PATH"
+            '';
+          }) cfg.versions;
+        in pkgs.writeText "nix-versions.json" (builtins.toJSON manifest);
+      };
+    };
+  };
+}
diff --git a/justfile b/justfile
new file mode 100644
index 0000000..4fa6931
--- /dev/null
+++ b/justfile
@@ -0,0 +1,19 @@
+set positional-arguments
+
+here := env_var_or_default("JUST_INVOCATION_DIR", invocation_directory())
+base := `pwd`
+
+#@echo "here: {{ here }}"
+#@echo "base: {{ base }}"
+
+# List available targets
+list:
+	@just --list --unsorted
+
+# Run a command with an alternative Nix version
+with-nix version *command:
+	set -e; \
+		hook="$(jq -e -r '.[$version].shellHook' --arg version "{{ version }}" < "$NIX_VERSIONS" || (>&2 echo "Version {{ version }} doesn't exist"; exit 1))"; \
+		eval "$hook"; \
+		CARGO_TARGET_DIR="{{ base }}/target/nix-{{ version }}" \
+		{{ command }}

From c6989fa54ed9d64e765f1bdf7f712093f1331228 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 27/56] flake/devshells: Add Nix versions manifest

---
 flake/devshells.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/flake/devshells.nix b/flake/devshells.nix
index b71a0bd..b480bf3 100644
--- a/flake/devshells.nix
+++ b/flake/devshells.nix
@@ -62,6 +62,7 @@ in
           sqlite-interactive
 
           flyctl
+        ] ++ lib.optionals pkgs.stdenv.isLinux [
           wrangler
         ];
 
@@ -94,6 +95,9 @@ in
 
           # See comment in `attic/build.rs`
           NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include";
+
+          # Used by `just with-nix` to build/test with alternative Nix versions.
+          NIX_VERSIONS = config.attic.nix-versions.manifestFile;
         };
       } cfg.extraArgs);
 

From 0d9c1c826fca369b751eec9819811003ecc1a71f Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 28/56] flake/devshells: Re-enable cargo-outdated

---
 flake/devshells.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flake/devshells.nix b/flake/devshells.nix
index b480bf3..c736b2e 100644
--- a/flake/devshells.nix
+++ b/flake/devshells.nix
@@ -39,8 +39,7 @@ in
           rustc
 
           cargo-expand
-          # Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152
-          # cargo-outdated
+          cargo-outdated
           cargo-edit
           tokio-console
         ];

From 6b1d4520f33d1e9f6f48263788daa60c5ad2b61a Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 29/56] Migrate internalMatrix to flake-parts

---
 flake.nix              | 11 -----------
 flake/nix-versions.nix | 16 ++++++++++++++--
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/flake.nix b/flake.nix
index 1abf156..3cbd238 100644
--- a/flake.nix
+++ b/flake.nix
@@ -55,15 +55,6 @@
       };
       cranePkgs = makeCranePkgs pkgs;
 
-      internalMatrix = lib.mapAttrs (_: nix: let
-        cranePkgs' = cranePkgs.override { inherit nix; };
-      in {
-        inherit (cranePkgs') attic-tests cargoArtifacts;
-      }) {
-        "2.20" = pkgs.nixVersions.nix_2_20;
-        "2.24" = pkgs.nixVersions.nix_2_24;
-        "default" = pkgs.nix;
-      };
 
       pkgsStable = import inputs.nixpkgs-stable {
         inherit system;
@@ -73,8 +64,6 @@
 
       inherit (pkgs) lib;
     in rec {
-      inherit internalMatrix;
-
       checks = let
         makeIntegrationTests = pkgs: import ./integration-tests {
           pkgs = import inputs.nixpkgs {
diff --git a/flake/nix-versions.nix b/flake/nix-versions.nix
index 66f3697..b0b8df6 100644
--- a/flake/nix-versions.nix
+++ b/flake/nix-versions.nix
@@ -1,4 +1,4 @@
-{ lib, flake-parts-lib, ... }:
+{ lib, flake-parts-lib, config, ... }:
 let
   inherit (lib)
     mkOption
@@ -20,11 +20,17 @@ in
           type = types.package;
         };
       };
+
+      options.internalMatrix = mkOption {
+        type = types.attrsOf (types.attrsOf types.package);
+      };
     };
   };
 
   config = {
-    perSystem = { self', pkgs, config, ... }: let
+    flake.internalMatrix = lib.mapAttrs (system: ps: ps.internalMatrix) config.allSystems;
+
+    perSystem = { self', pkgs, config, cranePkgs, ... }: let
       cfg = config.attic.nix-versions;
     in {
       attic.nix-versions = {
@@ -47,6 +53,12 @@ in
           }) cfg.versions;
         in pkgs.writeText "nix-versions.json" (builtins.toJSON manifest);
       };
+
+      internalMatrix = lib.mapAttrs (_: nix: let
+        cranePkgs' = cranePkgs.override { inherit nix; };
+      in {
+        inherit (cranePkgs') attic-tests cargoArtifacts;
+      }) cfg.versions;
     };
   };
 }

From eeb4275172c758b2c21fdf03088e7baf83cd874f Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 30/56] .github: Factor WebAssembly build out to justfile

---
 .github/workflows/build.yml | 16 ++++++----------
 justfile                    | 15 +++++++++++++++
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c1b9bb7..f265478 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -45,25 +45,21 @@ jobs:
           ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }}
           ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
 
-      - name: Build and run tests
+      - name: Cache dev shell
         run: |
+          .ci/cache-shell.sh
           system=$(nix-instantiate --eval -E 'builtins.currentSystem')
           echo system=$system >>$GITHUB_ENV
+
+      - name: Build and run tests
+        run: |
           tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L)
           find "$tests/bin" -exec {} \;
 
       - name: Build WebAssembly crates
         if: runner.os == 'Linux'
         run: |
-          # https://github.com/rust-lang/rust/issues/122357
-          export RUST_MIN_STACK=16777216
-
-          pushd attic
-          nix develop .# --command -- cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream
-          popd
-          pushd token
-          nix develop .# --command -- cargo build --target wasm32-unknown-unknown
-          popd
+          .ci/run just ci-build-wasm
 
       # TODO: Just take a diff of the list of store paths, also abstract all of this out
       - name: Push build artifacts
diff --git a/justfile b/justfile
index 4fa6931..2216a49 100644
--- a/justfile
+++ b/justfile
@@ -17,3 +17,18 @@ with-nix version *command:
 		eval "$hook"; \
 		CARGO_TARGET_DIR="{{ base }}/target/nix-{{ version }}" \
 		{{ command }}
+
+# (CI) Build WebAssembly crates
+ci-build-wasm:
+	#!/usr/bin/env bash
+	set -euxo pipefail
+
+	# https://github.com/rust-lang/rust/issues/122357
+	export RUST_MIN_STACK=16777216
+
+	pushd attic
+	cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream
+	popd
+	pushd token
+	cargo build --target wasm32-unknown-unknown
+	popd

From a9cafe829824eb12caa24243fbdfab9c29172deb Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 31/56] .github: Factor unit tests out to justfile

---
 .github/workflows/build.yml | 5 ++---
 justfile                    | 9 +++++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f265478..75f815c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -51,10 +51,9 @@ jobs:
           system=$(nix-instantiate --eval -E 'builtins.currentSystem')
           echo system=$system >>$GITHUB_ENV
 
-      - name: Build and run tests
+      - name: Run unit tests
         run: |
-          tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L)
-          find "$tests/bin" -exec {} \;
+          .ci/run just ci-unit-tests ${{ matrix.nix }}
 
       - name: Build WebAssembly crates
         if: runner.os == 'Linux'
diff --git a/justfile b/justfile
index 2216a49..64dcc2c 100644
--- a/justfile
+++ b/justfile
@@ -32,3 +32,12 @@ ci-build-wasm:
 	pushd token
 	cargo build --target wasm32-unknown-unknown
 	popd
+
+# (CI) Run unit tests
+ci-unit-tests matrix:
+	#!/usr/bin/env bash
+	set -euxo pipefail
+
+	system=$(nix-instantiate --eval -E 'builtins.currentSystem')
+	tests=$(nix build .#internalMatrix."$system".\"{{ matrix }}\".attic-tests --no-link --print-out-paths -L)
+	find "$tests/bin" -exec {} \;

From c5d84a475df3d82d388a96bd659fd9bfff916593 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 32/56] Migrate integration tests to flake-parts

Well, actually also to fix them.
---
 flake.nix                   | 35 ----------------------
 flake/integration-tests.nix | 60 +++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+), 35 deletions(-)
 create mode 100644 flake/integration-tests.nix

diff --git a/flake.nix b/flake.nix
index 3cbd238..68667fd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -27,10 +27,6 @@
 
     inherit (inputs.nixpkgs) lib;
 
-    makeCranePkgs = pkgs: let
-      craneLib = inputs.crane.mkLib pkgs;
-    in pkgs.callPackage ./crane.nix { inherit craneLib; };
-
     modules = builtins.foldl' (acc: f: f acc) ./flake [
       builtins.readDir
       (lib.filterAttrs (name: type:
@@ -46,36 +42,5 @@
     systems = supportedSystems;
 
     debug = true;
-
-    # old flake
-    flake = inputs.flake-utils.lib.eachSystem supportedSystems (system: let
-      pkgs = import inputs.nixpkgs {
-        inherit system;
-        overlays = [];
-      };
-      cranePkgs = makeCranePkgs pkgs;
-
-
-      pkgsStable = import inputs.nixpkgs-stable {
-        inherit system;
-        overlays = [];
-      };
-      cranePkgsStable = makeCranePkgs pkgsStable;
-
-      inherit (pkgs) lib;
-    in rec {
-      checks = let
-        makeIntegrationTests = pkgs: import ./integration-tests {
-          pkgs = import inputs.nixpkgs {
-            inherit system;
-            overlays = [ self.overlays.default ];
-          };
-          flake = self;
-        };
-        unstableTests = makeIntegrationTests pkgs;
-        stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
-      in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
-    }) // {
-    };
   };
 }
diff --git a/flake/integration-tests.nix b/flake/integration-tests.nix
new file mode 100644
index 0000000..a05d742
--- /dev/null
+++ b/flake/integration-tests.nix
@@ -0,0 +1,60 @@
+{ lib, flake-parts-lib, inputs, self, ... }:
+let
+  inherit (lib)
+    mkOption
+    types
+    ;
+  inherit (flake-parts-lib)
+    mkPerSystemOption
+    ;
+in
+{
+  options = {
+    perSystem = mkPerSystemOption {
+      options.attic.integration-tests = {
+        nixpkgsArgs = mkOption {
+          type = types.attrsOf types.anything;
+          default = {};
+        };
+        tests = mkOption {
+          type = types.attrsOf types.package;
+          default = {};
+        };
+        stableTests = mkOption {
+          type = types.attrsOf types.package;
+          default = {};
+        };
+      };
+    };
+  };
+
+  config = {
+    perSystem = { self', pkgs, config, system, ... }: let
+      cfg = config.attic.integration-tests;
+
+      vmPkgs = import inputs.nixpkgs ({
+        inherit system;
+        overlays = [ self.overlays.default ];
+      } // cfg.nixpkgsArgs);
+      vmPkgsStable = import inputs.nixpkgs-stable ({
+        inherit system;
+        overlays = [ self.overlays.default ];
+      } // cfg.nixpkgsArgs);
+
+      makeIntegrationTests = pkgs: import ../integration-tests {
+        inherit pkgs;
+        flake = self;
+      };
+    in {
+      attic.integration-tests = {
+        tests = makeIntegrationTests vmPkgs;
+        stableTests = makeIntegrationTests vmPkgsStable;
+      };
+
+      checks = let
+        tests = cfg.tests;
+        stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") cfg.stableTests;
+      in lib.optionalAttrs pkgs.stdenv.isLinux (tests // stableTests);
+    };
+  };
+}

From 7f275f00d8932e56cd7929b39c08d3e8f7fad74c Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 30 Aug 2024 12:32:10 -0400
Subject: [PATCH 33/56] .github: Update bash on macOS runners

---
 .github/workflows/build.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 75f815c..031d200 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -23,6 +23,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4.1.1
 
+      - name: Install current Bash on macOS
+        if: runner.os == 'macOS'
+        run: |
+          command -v brew && brew install bash || true
+
       - uses: DeterminateSystems/nix-installer-action@v9
         continue-on-error: true # Self-hosted runners already have Nix installed
 

From 938cb7634b4a69ef8e25e5e9409896f6b33a15b5 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 2 Sep 2024 14:07:37 -0400
Subject: [PATCH 34/56] .cargo: Rename config to config.toml

Support added since 1.39.
---
 .cargo/{config => config.toml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .cargo/{config => config.toml} (100%)

diff --git a/.cargo/config b/.cargo/config.toml
similarity index 100%
rename from .cargo/config
rename to .cargo/config.toml

From 1899dd79f11e3057de0fffab5fafaa39798716e3 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 2 Sep 2024 14:07:37 -0400
Subject: [PATCH 35/56] rustfmt

---
 attic/src/stream.rs      | 20 ++++++++++++++++----
 attic/src/testing/mod.rs |  2 +-
 client/src/cache.rs      |  2 +-
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/attic/src/stream.rs b/attic/src/stream.rs
index 077b021..6619b56 100644
--- a/attic/src/stream.rs
+++ b/attic/src/stream.rs
@@ -190,10 +190,22 @@ mod tests {
         // force multiple reads
         let mut buf = vec![0u8; 100];
         let mut bytes_read = 0;
-        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
-        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
-        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
-        bytes_read += read.read(&mut buf[bytes_read..bytes_read + 5]).await.unwrap();
+        bytes_read += read
+            .read(&mut buf[bytes_read..bytes_read + 5])
+            .await
+            .unwrap();
+        bytes_read += read
+            .read(&mut buf[bytes_read..bytes_read + 5])
+            .await
+            .unwrap();
+        bytes_read += read
+            .read(&mut buf[bytes_read..bytes_read + 5])
+            .await
+            .unwrap();
+        bytes_read += read
+            .read(&mut buf[bytes_read..bytes_read + 5])
+            .await
+            .unwrap();
 
         assert_eq!(expected.len(), bytes_read);
         assert_eq!(expected, &buf[..bytes_read]);
diff --git a/attic/src/testing/mod.rs b/attic/src/testing/mod.rs
index 393c31d..124e075 100644
--- a/attic/src/testing/mod.rs
+++ b/attic/src/testing/mod.rs
@@ -24,4 +24,4 @@ pub fn get_fake_data(len: usize) -> Vec<u8> {
     }
 
     data
-}
\ No newline at end of file
+}
diff --git a/client/src/cache.rs b/client/src/cache.rs
index 66b380c..71b4829 100644
--- a/client/src/cache.rs
+++ b/client/src/cache.rs
@@ -14,7 +14,7 @@ use std::str::FromStr;
 use anyhow::{anyhow, Result};
 use serde::{Deserialize, Serialize};
 
-pub use attic::cache::{CacheName};
+pub use attic::cache::CacheName;
 
 /// A reference to a cache.
 #[derive(Debug, Clone)]

From 20ada8ea5a255bdb250a5555f57fb4976409912d Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Mon, 2 Sep 2024 14:07:37 -0400
Subject: [PATCH 36/56] .github: Add lint workflow

---
 .github/workflows/lint.yml | 44 ++++++++++++++++++++++++++++++++++++++
 justfile                   |  4 ++++
 2 files changed, 48 insertions(+)
 create mode 100644 .github/workflows/lint.yml

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 0000000..a52a61f
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,44 @@
+name: Lint
+
+on:
+  pull_request:
+  push:
+jobs:
+  lint:
+    name: Lint
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - uses: DeterminateSystems/nix-installer-action@v9
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Install Attic
+        run: |
+          if ! command -v attic &> /dev/null; then
+            ./.github/install-attic-ci.sh
+          fi
+
+      - name: Configure Attic
+        run: |
+          : "${ATTIC_SERVER:=https://staging.attic.rs/}"
+          : "${ATTIC_CACHE:=attic-ci}"
+          echo ATTIC_CACHE=$ATTIC_CACHE >>$GITHUB_ENV
+          export PATH=$HOME/.nix-profile/bin:$PATH # FIXME
+          attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN"
+          attic use "$ATTIC_CACHE"
+        env:
+          ATTIC_SERVER: ${{ secrets.ATTIC_SERVER }}
+          ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }}
+          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+
+      - name: Cache dev shell
+        run: |
+          .ci/cache-shell.sh
+          system=$(nix-instantiate --eval -E 'builtins.currentSystem')
+          echo system=$system >>$GITHUB_ENV
+
+      - name: Check rustfmt
+        run: .ci/run just ci-rustfmt
diff --git a/justfile b/justfile
index 64dcc2c..30f4368 100644
--- a/justfile
+++ b/justfile
@@ -41,3 +41,7 @@ ci-unit-tests matrix:
 	system=$(nix-instantiate --eval -E 'builtins.currentSystem')
 	tests=$(nix build .#internalMatrix."$system".\"{{ matrix }}\".attic-tests --no-link --print-out-paths -L)
 	find "$tests/bin" -exec {} \;
+
+# (CI) Run rustfmt check
+ci-rustfmt:
+	cargo fmt --check

From 9f9facd90504283f47c1b2714b2b6b6fb2716ee6 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sun, 8 Sep 2024 12:44:22 -0400
Subject: [PATCH 37/56] flake: Remove flake-utils

---
 flake.lock | 34 ----------------------------------
 flake.nix  |  9 +++++++--
 2 files changed, 7 insertions(+), 36 deletions(-)

diff --git a/flake.lock b/flake.lock
index 1e31e3b..944a4aa 100644
--- a/flake.lock
+++ b/flake.lock
@@ -56,24 +56,6 @@
         "type": "github"
       }
     },
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1724999960,
@@ -111,25 +93,9 @@
         "crane": "crane",
         "flake-compat": "flake-compat",
         "flake-parts": "flake-parts",
-        "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
         "nixpkgs-stable": "nixpkgs-stable"
       }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index 68667fd..868ea70 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,7 +4,6 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
-    flake-utils.url = "github:numtide/flake-utils";
 
     flake-parts = {
       url = "github:hercules-ci/flake-parts";
@@ -23,7 +22,13 @@
   };
 
   outputs = inputs @ { self, flake-parts, ... }: let
-    supportedSystems = inputs.flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];
+    supportedSystems = [
+      "x86_64-linux"
+      "aarch64-linux"
+      "riscv64-linux"
+      "aarch64-darwin"
+      "x86_64-darwin"
+    ];
 
     inherit (inputs.nixpkgs) lib;
 

From 1da3ed6b503774b42fcaecf73ed2529063cc2009 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sun, 8 Sep 2024 12:44:22 -0400
Subject: [PATCH 38/56] flake/packages: Make makeCranePkgs extensible

---
 flake/packages.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake/packages.nix b/flake/packages.nix
index c8fdf95..8cd10e7 100644
--- a/flake/packages.nix
+++ b/flake/packages.nix
@@ -1,11 +1,11 @@
-{ self, inputs, lib, ... }:
+{ self, inputs, lib, makeCranePkgs, ... }:
 let
-  makeCranePkgs = pkgs: let
+  defaultMakeCranePkgs = pkgs: let
     craneLib = inputs.crane.mkLib pkgs;
   in pkgs.callPackage ../crane.nix { inherit craneLib; };
 in
 {
-  _module.args.makeCranePkgs = makeCranePkgs;
+  _module.args.makeCranePkgs = lib.mkDefault defaultMakeCranePkgs;
 
   perSystem = { self', pkgs, cranePkgs, ... }: (lib.mkMerge [
     {

From 8157a3337aac378d5a5497c332aba7796f81b9cf Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sun, 8 Sep 2024 12:44:22 -0400
Subject: [PATCH 39/56] flake/devshells: Move rustc to separate category

---
 flake/devshells.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flake/devshells.nix b/flake/devshells.nix
index c736b2e..2452021 100644
--- a/flake/devshells.nix
+++ b/flake/devshells.nix
@@ -35,9 +35,11 @@ in
       cfg = config.attic.devshell;
     in {
       attic.devshell.packageSets = with pkgs; {
-        rust = [
+        rustc = [
           rustc
+        ];
 
+        rust = [
           cargo-expand
           cargo-outdated
           cargo-edit

From 2c43acd86b1b2d0161dfb937c3df68bca8745a70 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 40/56] flake: Allow overriding Rust toolchain

---
 flake/devshells.nix |   2 +-
 flake/packages.nix  | 201 ++++++++++++++++++++++++++------------------
 2 files changed, 120 insertions(+), 83 deletions(-)

diff --git a/flake/devshells.nix b/flake/devshells.nix
index 2452021..32444a4 100644
--- a/flake/devshells.nix
+++ b/flake/devshells.nix
@@ -35,7 +35,7 @@ in
       cfg = config.attic.devshell;
     in {
       attic.devshell.packageSets = with pkgs; {
-        rustc = [
+        rustc = lib.optionals (config.attic.toolchain == null) [
           rustc
         ];
 
diff --git a/flake/packages.nix b/flake/packages.nix
index 8cd10e7..05b610c 100644
--- a/flake/packages.nix
+++ b/flake/packages.nix
@@ -1,96 +1,133 @@
-{ self, inputs, lib, makeCranePkgs, ... }:
+{ self
+, lib
+, flake-parts-lib
+, inputs
+, config
+, makeCranePkgs
+, getSystem
+, ...
+}:
+
 let
-  defaultMakeCranePkgs = pkgs: let
-    craneLib = inputs.crane.mkLib pkgs;
-  in pkgs.callPackage ../crane.nix { inherit craneLib; };
+  inherit (lib)
+    mkOption
+    types
+    ;
+  inherit (flake-parts-lib)
+    mkPerSystemOption
+    ;
 in
 {
-  _module.args.makeCranePkgs = lib.mkDefault defaultMakeCranePkgs;
-
-  perSystem = { self', pkgs, cranePkgs, ... }: (lib.mkMerge [
-    {
-      _module.args.cranePkgs = makeCranePkgs pkgs;
-
-      packages = {
-        default = self'.packages.attic;
-
-        inherit (cranePkgs)
-          attic
-          attic-client
-          attic-server
-        ;
-
-        attic-nixpkgs = pkgs.callPackage ../package.nix { };
-
-        attic-ci-installer = pkgs.callPackage ../ci-installer.nix {
-          inherit self;
-        };
-
-        book = pkgs.callPackage ../book {
-          attic = self'.packages.attic;
+  options = {
+    perSystem = mkPerSystemOption {
+      options.attic = {
+        toolchain = mkOption {
+          type = types.nullOr types.package;
+          default = null;
         };
       };
-    }
+    };
+  };
 
-    (lib.mkIf pkgs.stdenv.isLinux {
-      packages = {
-        attic-server-image = pkgs.dockerTools.buildImage {
-          name = "attic-server";
-          tag = "main";
-          copyToRoot = [
-            self'.packages.attic-server
+  config = {
+    _module.args.makeCranePkgs = lib.mkDefault (pkgs: let
+      perSystemConfig = getSystem pkgs.system;
+      craneLib = builtins.foldl' (acc: f: f acc) pkgs [
+        inputs.crane.mkLib
+        (craneLib:
+          if perSystemConfig.attic.toolchain == null then craneLib
+          else craneLib.overrideToolchain config.attic.toolchain
+        )
+      ];
+    in pkgs.callPackage ../crane.nix {
+      inherit craneLib;
+    });
 
-            # Debugging utilities for `fly ssh console`
-            pkgs.busybox
+    perSystem = { self', pkgs, config, cranePkgs, ... }: (lib.mkMerge [
+      {
+        _module.args.cranePkgs = makeCranePkgs pkgs;
 
-            # Now required by the fly.io sshd
-            pkgs.dockerTools.fakeNss
-          ];
-          config = {
-            Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ];
-            Env = [
-              "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
-            ];
+        packages = {
+          default = self'.packages.attic;
+
+          inherit (cranePkgs)
+            attic
+            attic-client
+            attic-server
+          ;
+
+          attic-nixpkgs = pkgs.callPackage ../package.nix { };
+
+          attic-ci-installer = pkgs.callPackage ../ci-installer.nix {
+            inherit self;
+          };
+
+          book = pkgs.callPackage ../book {
+            attic = self'.packages.attic;
           };
         };
-      };
-    })
+      }
 
-    # Unfortunately, x86_64-darwin fails to evaluate static builds
-    (lib.mkIf (pkgs.system != "x86_64-darwin") {
-      packages = {
-        # TODO: Make this work with Crane
-        attic-static = (pkgs.pkgsStatic.callPackage ../package.nix {
-          nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
-            patches = (old.patches or []) ++ [
-              # To be submitted
-              (pkgs.fetchpatch {
-                url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
-                hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
-              })
+      (lib.mkIf pkgs.stdenv.isLinux {
+        packages = {
+          attic-server-image = pkgs.dockerTools.buildImage {
+            name = "attic-server";
+            tag = "main";
+            copyToRoot = [
+              self'.packages.attic-server
+
+              # Debugging utilities for `fly ssh console`
+              pkgs.busybox
+
+              # Now required by the fly.io sshd
+              pkgs.dockerTools.fakeNss
             ];
-          });
-        }).overrideAttrs (old: {
-          nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
-            pkgs.nukeReferences
-          ];
-
-          # Read by pkg_config crate (do some autodetection in build.rs?)
-          PKG_CONFIG_ALL_STATIC = "1";
-
-          "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
-          RUSTFLAGS = "-C relocation-model=static";
-
-          postFixup = (old.postFixup or "") + ''
-            rm -f $out/nix-support/propagated-build-inputs
-            nuke-refs $out/bin/attic
-          '';
-        });
-
-        attic-client-static = self'.packages.attic-static.override {
-          clientOnly = true;
+            config = {
+              Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ];
+              Env = [
+                "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
+              ];
+            };
+          };
         };
-      };
-    })
-  ]);
+      })
+
+      # Unfortunately, x86_64-darwin fails to evaluate static builds
+      (lib.mkIf (pkgs.system != "x86_64-darwin") {
+        packages = {
+          # TODO: Make this work with Crane
+          attic-static = (pkgs.pkgsStatic.callPackage ../package.nix {
+            nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
+              patches = (old.patches or []) ++ [
+                # To be submitted
+                (pkgs.fetchpatch {
+                  url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
+                  hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
+                })
+              ];
+            });
+          }).overrideAttrs (old: {
+            nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
+              pkgs.nukeReferences
+            ];
+
+            # Read by pkg_config crate (do some autodetection in build.rs?)
+            PKG_CONFIG_ALL_STATIC = "1";
+
+            "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
+            RUSTFLAGS = "-C relocation-model=static";
+
+            postFixup = (old.postFixup or "") + ''
+              rm -f $out/nix-support/propagated-build-inputs
+              nuke-refs $out/bin/attic
+            '';
+          });
+
+          attic-client-static = self'.packages.attic-static.override {
+            clientOnly = true;
+          };
+        };
+      })
+    ]);
+  };
 }

From b63394302f21c65a319861d7a6bf47628ea02528 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 41/56] flake: Add escape hatch to inject package args

---
 crane.nix          | 18 +++++++++++-------
 flake/packages.nix |  5 +++++
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/crane.nix b/crane.nix
index 4ec2827..73bdd0f 100644
--- a/crane.nix
+++ b/crane.nix
@@ -19,6 +19,8 @@
 , boost
 , darwin
 , libiconv
+
+, extraPackageArgs ? {}
 }:
 
 let
@@ -43,7 +45,9 @@ let
     libiconv
   ];
 
-  cargoArtifacts = craneLib.buildDepsOnly {
+  extraArgs = extraPackageArgs;
+
+  cargoArtifacts = craneLib.buildDepsOnly ({
     pname = "attic";
     inherit src version nativeBuildInputs buildInputs;
 
@@ -54,7 +58,7 @@ let
     # With `use-zstd`, the cargo artifacts are archived in a `tar.zstd`. This is
     # actually set if you use `buildPackage` without passing `cargoArtifacts`.
     installCargoArtifactsMode = "use-zstd";
-  };
+  } // extraArgs);
 
   mkAttic = args: craneLib.buildPackage ({
     pname = "attic";
@@ -86,7 +90,7 @@ let
       maintainers = with maintainers; [ zhaofengli ];
       platforms = platforms.linux ++ platforms.darwin;
     };
-  } // args);
+  } // args // extraArgs);
 
   attic = mkAttic {
     cargoExtraArgs = "-p attic-client -p attic-server";
@@ -106,7 +110,7 @@ let
   #
   # We don't enable fat LTO in the default `attic` package since it
   # dramatically increases build time.
-  attic-server = craneLib.buildPackage {
+  attic-server = craneLib.buildPackage ({
     pname = "attic-server";
 
     # We don't pull in the common cargoArtifacts because the feature flags
@@ -120,13 +124,13 @@ let
 
     CARGO_PROFILE_RELEASE_LTO = "fat";
     CARGO_PROFILE_RELEASE_CODEGEN_UNITS = "1";
-  };
+  } // extraArgs);
 
   # Attic interacts with Nix directly and its tests require trusted-user access
   # to nix-daemon to import NARs, which is not possible in the build sandbox.
   # In the CI pipeline, we build the test executable inside the sandbox, then
   # run it outside.
-  attic-tests = craneLib.mkCargoDerivation {
+  attic-tests = craneLib.mkCargoDerivation ({
     pname = "attic-tests";
 
     inherit src version buildInputs cargoArtifacts;
@@ -151,7 +155,7 @@ let
 
       runHook postInstall
     '';
-  };
+  } // extraArgs);
 in {
   inherit cargoArtifacts attic attic-client attic-server attic-tests;
 }
diff --git a/flake/packages.nix b/flake/packages.nix
index 05b610c..720d804 100644
--- a/flake/packages.nix
+++ b/flake/packages.nix
@@ -25,6 +25,10 @@ in
           type = types.nullOr types.package;
           default = null;
         };
+        extraPackageArgs = mkOption {
+          type = types.attrsOf types.anything;
+          default = {};
+        };
       };
     };
   };
@@ -41,6 +45,7 @@ in
       ];
     in pkgs.callPackage ../crane.nix {
       inherit craneLib;
+      inherit (perSystemConfig.attic) extraPackageArgs;
     });
 
     perSystem = { self', pkgs, config, cranePkgs, ... }: (lib.mkMerge [

From 0f981dab00aab47fb5019483fcc0e7104ef6417e Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 42/56] crane: Add overrides for cross compilation

---
 crane.nix | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/crane.nix b/crane.nix
index 73bdd0f..2907596 100644
--- a/crane.nix
+++ b/crane.nix
@@ -7,8 +7,9 @@
 
 { stdenv
 , lib
+, buildPackages
 , craneLib
-, rustPlatform
+, rust
 , runCommand
 , writeReferencesToFile
 , pkg-config
@@ -45,7 +46,17 @@ let
     libiconv
   ];
 
-  extraArgs = extraPackageArgs;
+  crossArgs = let
+    rustTargetSpec = rust.toRustTargetSpec stdenv.hostPlatform;
+    rustTargetSpecEnv = lib.toUpper (builtins.replaceStrings [ "-" ] [ "_" ] rustTargetSpec);
+  in lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) {
+    depsBuildBuild = [ buildPackages.stdenv.cc ];
+
+    CARGO_BUILD_TARGET = rustTargetSpec;
+    "CARGO_TARGET_${rustTargetSpecEnv}_LINKER" = "${stdenv.cc.targetPrefix}cc";
+  };
+
+  extraArgs = crossArgs // extraPackageArgs;
 
   cargoArtifacts = craneLib.buildDepsOnly ({
     pname = "attic";

From f5463eccc01efbfddc0c20b90cd10d80fb8acecd Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 43/56] crane: Ignore flake directory

---
 crane.nix | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/crane.nix b/crane.nix
index 2907596..ffe9eb0 100644
--- a/crane.nix
+++ b/crane.nix
@@ -27,7 +27,15 @@
 let
   version = "0.1.0";
 
-  ignoredPaths = [ ".github" "target" "book" "nixos" "integration-tests" ];
+  ignoredPaths = [
+    ".ci"
+    ".github"
+    "book"
+    "flake"
+    "integration-tests"
+    "nixos"
+    "target"
+  ];
 
   src = lib.cleanSourceWith {
     filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) ignoredPaths);

From 2a5cce4baad7461e83493b49e95ebf578c5e97d7 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 44/56] flake/packages: Add cross-compiled container image for
 aarch64-linux

---
 flake/packages.nix | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/flake/packages.nix b/flake/packages.nix
index 720d804..c67bb4c 100644
--- a/flake/packages.nix
+++ b/flake/packages.nix
@@ -16,6 +16,17 @@ let
   inherit (flake-parts-lib)
     mkPerSystemOption
     ;
+
+  # Re-evaluate perSystem with cross nixpkgs
+  # HACK before https://github.com/hercules-ci/flake-parts/issues/95 is solved
+  evalCross = { system, pkgs }: config.allSystems.${system}.debug.extendModules {
+    modules = [
+      ({ config, lib, ... }: {
+        _module.args.pkgs = pkgs;
+        _module.args.self' = lib.mkForce config;
+      })
+    ];
+  };
 in
 {
   options = {
@@ -97,6 +108,18 @@ in
         };
       })
 
+      (lib.mkIf (pkgs.system == "x86_64-linux") {
+        packages = {
+          attic-server-image-aarch64 = let
+            eval = evalCross {
+              system = "aarch64-linux";
+              pkgs = pkgs.pkgsCross.aarch64-multiplatform;
+            };
+
+          in eval.config.packages.attic-server-image;
+        };
+      })
+
       # Unfortunately, x86_64-darwin fails to evaluate static builds
       (lib.mkIf (pkgs.system != "x86_64-darwin") {
         packages = {

From 444ea808d5661cf3a944e256d949285492275b84 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 45/56] flake/devshells: Add skopeo and manifest-tool

---
 flake/devshells.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flake/devshells.nix b/flake/devshells.nix
index 32444a4..e7b1145 100644
--- a/flake/devshells.nix
+++ b/flake/devshells.nix
@@ -63,6 +63,8 @@ in
           sqlite-interactive
 
           flyctl
+          skopeo
+          manifest-tool
         ] ++ lib.optionals pkgs.stdenv.isLinux [
           wrangler
         ];

From b45885e67b1a652ee295968c4bbe8e3310bd1e46 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 46/56] Build and push multi-arch images

Fixes #147.
---
 .ci/build-and-push-images.sh | 60 ++++++++++++++++++++++
 .github/workflows/build.yml  | 98 ++++++++++++++++++++++++++++--------
 justfile                     |  4 ++
 3 files changed, 141 insertions(+), 21 deletions(-)
 create mode 100755 .ci/build-and-push-images.sh

diff --git a/.ci/build-and-push-images.sh b/.ci/build-and-push-images.sh
new file mode 100755
index 0000000..740d9f9
--- /dev/null
+++ b/.ci/build-and-push-images.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$#" -lt "2" ]]; then
+	>&2 echo "Usage: $0 <image name> <tag1> ..."
+	>&2 echo "Example: $0 ghcr.io/zhaofengli/attic main abcd123"
+	exit 1
+fi
+
+cleanup() {
+	if [[ -f "${manifest_spec}" ]]; then
+		rm "${manifest_spec}"
+	fi
+}
+trap cleanup EXIT
+
+image_name="$1"
+tags=("${@:2}")
+
+manifest_spec="$(mktemp -t attic-manifest-spec.XXXXXXXXXX)"
+
+declare -a digests
+
+emit_header() {
+	echo "image: ${image_name}"
+	echo "tags:"
+	for tag in "${tags[@]}"; do
+		echo "- ${tag}"
+	done
+	echo "manifests:"
+}
+
+push_digest() {
+	source_image="docker-archive:$1"
+	digest="$(skopeo inspect "${source_image}" | jq -r .Digest)"
+	target_image="docker://${image_name}@${digest}"
+
+	>&2 echo "${source_image} ▸ ${target_image}"
+	>&2 skopeo copy --insecure-policy "${source_image}" "${target_image}"
+
+	echo -n "- "
+	skopeo inspect "${source_image}" | \
+		jq '{platform: {architecture: .Architecture, os: .Os}, image: ($image_name + "@" + .Digest)}' \
+		--arg image_name "${image_name}"
+}
+
+>>"${manifest_spec}" emit_header
+
+nix build .#attic-server-image .#attic-server-image-aarch64 -L --print-out-paths | \
+while read -r output; do
+	>>"${manifest_spec}" push_digest "${output}"
+done
+
+>&2 echo "----------"
+>&2 echo "Generated manifest-tool spec:"
+>&2 echo "----------"
+cat "${manifest_spec}"
+>&2 echo "----------"
+
+manifest-tool push from-spec "${manifest_spec}"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 031d200..990907e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ on:
   push:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ghcr.io/${{ github.repository }}
 jobs:
   tests:
     strategy:
@@ -17,9 +17,6 @@ jobs:
           - "2.24"
           - "default"
     runs-on: ${{ matrix.os }}
-    permissions:
-      contents: read
-      packages: write
     steps:
       - uses: actions/checkout@v4.1.1
 
@@ -38,6 +35,7 @@ jobs:
           fi
 
       - name: Configure Attic
+        continue-on-error: true
         run: |
           : "${ATTIC_SERVER:=https://staging.attic.rs/}"
           : "${ATTIC_CACHE:=attic-ci}"
@@ -75,30 +73,88 @@ jobs:
               .#internalMatrix."$system".\"${{ matrix.nix }}\".cargoArtifacts \
             | xargs attic push "ci:$ATTIC_CACHE"
           fi
+
+  image:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    needs:
+      - tests
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4.1.1
+
+      - name: Install current Bash on macOS
+        if: runner.os == 'macOS'
+        run: |
+          command -v brew && brew install bash || true
+
+      - uses: DeterminateSystems/nix-installer-action@v9
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Install Attic
+        run: |
+          if ! command -v attic &> /dev/null; then
+            ./.github/install-attic-ci.sh
+          fi
+
+      - name: Configure Attic
+        continue-on-error: true
+        run: |
+          : "${ATTIC_SERVER:=https://staging.attic.rs/}"
+          : "${ATTIC_CACHE:=attic-ci}"
+          echo ATTIC_CACHE=$ATTIC_CACHE >>$GITHUB_ENV
+          export PATH=$HOME/.nix-profile/bin:$PATH # FIXME
+          attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN"
+          attic use "$ATTIC_CACHE"
+        env:
+          ATTIC_SERVER: ${{ secrets.ATTIC_SERVER }}
+          ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }}
+          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+
+      - name: Cache dev shell
+        run: |
+          .ci/cache-shell.sh
+          system=$(nix-instantiate --eval -E 'builtins.currentSystem')
+          echo system=$system >>$GITHUB_ENV
+
       - name: Log in to the Container registry
         uses: docker/login-action@v3.0.0
-        if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Push build container image
-        if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+      - name: Build and push container images
         continue-on-error: true
         run: |
-          IMAGE_ID=ghcr.io/${IMAGE_NAME}
-          TARBALL=$(nix build --json .#attic-server-image | jq -r '.[].outputs.out')
-          BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          TAG="${{ github.sha }}"
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && TAG=$(echo $BRANCH | sed -e 's/^v//')
-          docker load < ${TARBALL}
-          echo IMAGE_ID=$IMAGE_ID
-          echo TAG=$TAG
-          docker tag attic-server:main "${IMAGE_ID}:${TAG}"
-          docker push ${IMAGE_ID}:${TAG}
-          if [ "$BRANCH" == "main" ]; then
-            TAG="latest"
-            docker tag attic-server:main "${IMAGE_ID}:${TAG}"
-            docker push ${IMAGE_ID}:${TAG}
+          declare -a tags
+          tags+=("${{ github.sha }}")
+
+          branch=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
+            tags+=("$(echo $branch | sed -e 's/^v//')")
+          else
+            tags+=("${branch}")
+          fi
+
+          if [ "$branch" == "${{ github.event.repository.default_branch }}" ]; then
+            tags+=("latest")
+          fi
+
+          >&2 echo "Image: ${IMAGE_NAME}"
+          >&2 echo "Tags: ${tags[@]}"
+
+          .ci/run just ci-build-and-push-images "${IMAGE_NAME}" "${tags[@]}"
+
+      # TODO: Just take a diff of the list of store paths, also abstract all of this out
+      - name: Push build artifacts
+        run: |
+          export PATH=$HOME/.nix-profile/bin:$PATH # FIXME
+          if [ -n "$ATTIC_TOKEN" ]; then
+            nix build --no-link --print-out-paths -L \
+              .#attic-server-image \
+              .#attic-server-image-aarch64 \
+            | xargs attic push "ci:$ATTIC_CACHE"
           fi
diff --git a/justfile b/justfile
index 30f4368..ffe06e4 100644
--- a/justfile
+++ b/justfile
@@ -45,3 +45,7 @@ ci-unit-tests matrix:
 # (CI) Run rustfmt check
 ci-rustfmt:
 	cargo fmt --check
+
+# (CI) Build and push images
+ci-build-and-push-images *args:
+	.ci/build-and-push-images.sh {{ args }}

From c3618a20e989b1e60528a5ab8f5f589f74dc0430 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 09:59:49 -0400
Subject: [PATCH 47/56] .github: Update deps

---
 .github/workflows/book.yml  |  8 ++++----
 .github/workflows/build.yml | 10 +++++-----
 .github/workflows/lint.yml  |  9 +++++++--
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml
index 4fdd709..2a31963 100644
--- a/.github/workflows/book.yml
+++ b/.github/workflows/book.yml
@@ -16,9 +16,9 @@ jobs:
     if: github.repository == 'zhaofengli/attic'
 
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic
@@ -40,12 +40,12 @@ jobs:
           cp --recursive --dereference --no-preserve=mode,ownership result public
 
       - name: Upload book artifact
-        uses: actions/upload-pages-artifact@v2.0.0
+        uses: actions/upload-pages-artifact@v3.0.1
         with:
           path: public
 
       - name: Deploy book
-        uses: actions/deploy-pages@v3.0.1
+        uses: actions/deploy-pages@v4.0.5
 
       # TODO: Just take a diff of the list of store paths, also abstract all of this out
       - name: Push build artifacts
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 990907e..f9a62a7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -18,14 +18,14 @@ jobs:
           - "default"
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
       - name: Install current Bash on macOS
         if: runner.os == 'macOS'
         run: |
           command -v brew && brew install bash || true
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic
@@ -83,14 +83,14 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
       - name: Install current Bash on macOS
         if: runner.os == 'macOS'
         run: |
           command -v brew && brew install bash || true
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic
@@ -120,7 +120,7 @@ jobs:
           echo system=$system >>$GITHUB_ENV
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v3.0.0
+        uses: docker/login-action@v3.3.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index a52a61f..4ae17ac 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -10,9 +10,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - name: Install current Bash on macOS
+        if: runner.os == 'macOS'
+        run: |
+          command -v brew && brew install bash || true
+
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic

From 2babfb7f0156a008e946e6fb37de8e1448bd5a6a Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 10:57:59 -0400
Subject: [PATCH 48/56] flake.lock: Update nixpkgs

---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 944a4aa..8020ab6 100644
--- a/flake.lock
+++ b/flake.lock
@@ -58,11 +58,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1724999960,
-        "narHash": "sha256-LB3jqSGW5u1ZcUcX6vO/qBOq5oXHlmOCxsTXGMEitp4=",
+        "lastModified": 1726042813,
+        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "b96f849e725333eb2b1c7f1cb84ff102062468ba",
+        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
         "type": "github"
       },
       "original": {

From 1bd0279b79a2dd53c11d40be738536d8b71c53bb Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 10:57:59 -0400
Subject: [PATCH 49/56] attic/build.rs: Fix warnings when nix_store is disabled

---
 attic/build.rs | 137 +++++++++++++++++++++++++------------------------
 1 file changed, 70 insertions(+), 67 deletions(-)

diff --git a/attic/build.rs b/attic/build.rs
index 14151a2..86bbf7f 100644
--- a/attic/build.rs
+++ b/attic/build.rs
@@ -2,82 +2,85 @@
 //!
 //! We link against libnixstore to perform actions on the Nix Store.
 
-use cc::Build;
-use version_compare::Version;
-
-struct NixDependency {
-    version: String,
-}
-
-impl NixDependency {
-    fn detect() -> Self {
-        let library = pkg_config::Config::new()
-            .cargo_metadata(false)
-            .atleast_version("2.4")
-            .probe("nix-main")
-            .expect("Failed to find nix-main >=2.4 through pkg-config");
-
-        Self {
-            version: library.version,
-        }
-    }
-
-    fn apply_version_flags(&self, build: &mut Build) {
-        let version = Version::from(&self.version).unwrap();
-
-        if version >= Version::from("2.20").unwrap() {
-            build.flag("-DATTIC_NIX_2_20");
-        }
-    }
-
-    fn emit_cargo_metadata(&self) {
-        pkg_config::Config::new()
-            .atleast_version("2.4")
-            .probe("nix-store")
-            .unwrap();
-
-        pkg_config::Config::new()
-            .atleast_version("2.4")
-            .probe("nix-main")
-            .unwrap();
-    }
-}
-
 fn main() {
     #[cfg(feature = "nix_store")]
-    build_bridge();
+    nix_store::build_bridge();
 }
 
 #[cfg(feature = "nix_store")]
-fn build_bridge() {
-    let nix_dep = NixDependency::detect();
+mod nix_store {
+    use cc::Build;
+    use version_compare::Version;
 
-    // Temporary workaround for issue in <https://github.com/NixOS/nix/pull/8484>
-    let hacky_include = {
-        let dir = tempfile::tempdir().expect("Failed to create temporary directory for workaround");
-        std::fs::write(dir.path().join("uds-remote-store.md"), "\"\"").unwrap();
-        dir
-    };
+    struct NixDependency {
+        version: String,
+    }
 
-    let mut build = cxx_build::bridge("src/nix_store/bindings/mod.rs");
-    build
-        .file("src/nix_store/bindings/nix.cpp")
-        .flag("-std=c++2a")
-        .flag("-O2")
-        .flag("-include")
-        .flag("nix/config.h")
-        .flag("-idirafter")
-        .flag(hacky_include.path().to_str().unwrap())
-        // In Nix 2.19+, nix/args/root.hh depends on being able to #include "args.hh" (which is in its parent directory), for some reason
-        .flag("-I")
-        .flag(concat!(env!("NIX_INCLUDE_PATH"), "/nix"));
+    impl NixDependency {
+        fn detect() -> Self {
+            let library = pkg_config::Config::new()
+                .cargo_metadata(false)
+                .atleast_version("2.4")
+                .probe("nix-main")
+                .expect("Failed to find nix-main >=2.4 through pkg-config");
 
-    nix_dep.apply_version_flags(&mut build);
+            Self {
+                version: library.version,
+            }
+        }
 
-    build.compile("nixbinding");
+        fn apply_version_flags(&self, build: &mut Build) {
+            let version = Version::from(&self.version).unwrap();
 
-    println!("cargo:rerun-if-changed=src/nix_store/bindings");
+            if version >= Version::from("2.20").unwrap() {
+                build.flag("-DATTIC_NIX_2_20");
+            }
+        }
 
-    // the -l flags must be after -lnixbinding
-    nix_dep.emit_cargo_metadata();
+        fn emit_cargo_metadata(&self) {
+            pkg_config::Config::new()
+                .atleast_version("2.4")
+                .probe("nix-store")
+                .unwrap();
+
+            pkg_config::Config::new()
+                .atleast_version("2.4")
+                .probe("nix-main")
+                .unwrap();
+        }
+    }
+
+    pub fn build_bridge() {
+        let nix_dep = NixDependency::detect();
+
+        // Temporary workaround for issue in <https://github.com/NixOS/nix/pull/8484>
+        let hacky_include = {
+            let dir =
+                tempfile::tempdir().expect("Failed to create temporary directory for workaround");
+            std::fs::write(dir.path().join("uds-remote-store.md"), "\"\"").unwrap();
+            dir
+        };
+
+        let mut build = cxx_build::bridge("src/nix_store/bindings/mod.rs");
+        build
+            .file("src/nix_store/bindings/nix.cpp")
+            .flag("-std=c++2a")
+            .flag("-O2")
+            .flag("-include")
+            .flag("nix/config.h")
+            .flag("-idirafter")
+            .flag(hacky_include.path().to_str().unwrap())
+            // In Nix 2.19+, nix/args/root.hh depends on being able to #include "args.hh" (which is in its parent directory), for some reason
+            .flag("-I")
+            .flag(concat!(env!("NIX_INCLUDE_PATH"), "/nix"));
+
+        nix_dep.apply_version_flags(&mut build);
+
+        build.compile("nixbinding");
+
+        println!("cargo:rerun-if-changed=src/nix_store/bindings");
+
+        // the -l flags must be after -lnixbinding
+        nix_dep.emit_cargo_metadata();
+    }
 }

From 89cefcd50193725e78275317ad7f2b661df6a320 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Wed, 11 Sep 2024 10:57:59 -0400
Subject: [PATCH 50/56] flake: Follow package Nix version in dev shell

---
 crane.nix           | 4 ++++
 flake/devshells.nix | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/crane.nix b/crane.nix
index ffe9eb0..f7abf6e 100644
--- a/crane.nix
+++ b/crane.nix
@@ -109,6 +109,10 @@ let
       maintainers = with maintainers; [ zhaofengli ];
       platforms = platforms.linux ++ platforms.darwin;
     };
+
+    passthru = {
+      inherit nix;
+    };
   } // args // extraArgs);
 
   attic = mkAttic {
diff --git a/flake/devshells.nix b/flake/devshells.nix
index e7b1145..4f30d8e 100644
--- a/flake/devshells.nix
+++ b/flake/devshells.nix
@@ -97,7 +97,7 @@ in
           NIX_PATH = "nixpkgs=${pkgs.path}";
 
           # See comment in `attic/build.rs`
-          NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include";
+          NIX_INCLUDE_PATH = "${lib.getDev self'.packages.attic.passthru.nix}/include";
 
           # Used by `just with-nix` to build/test with alternative Nix versions.
           NIX_VERSIONS = config.attic.nix-versions.manifestFile;

From 265038a109169d21f18c12a45a1901550b4ef1ac Mon Sep 17 00:00:00 2001
From: jzbor <jzbor@posteo.net>
Date: Tue, 24 Sep 2024 12:50:53 +0200
Subject: [PATCH 51/56] attic-client/push: Add flag to read paths from stdin

---
 client/src/command/push.rs | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/client/src/command/push.rs b/client/src/command/push.rs
index 9e1e795..6eabb04 100644
--- a/client/src/command/push.rs
+++ b/client/src/command/push.rs
@@ -1,3 +1,5 @@
+use std::io;
+use std::io::BufRead;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -24,6 +26,10 @@ pub struct Push {
     /// The store paths to push.
     paths: Vec<PathBuf>,
 
+    /// Read paths from stdin
+    #[clap(short = 'i', long)]
+    stdin: bool,
+
     /// Push the specified paths only and do not compute closures.
     #[clap(long)]
     no_closure: bool,
@@ -50,12 +56,20 @@ pub async fn run(opts: Opts) -> Result<()> {
     let config = Config::load()?;
 
     let store = Arc::new(NixStore::connect()?);
-    let roots = sub
-        .paths
-        .clone()
-        .into_iter()
-        .map(|p| store.follow_store_path(p))
-        .collect::<std::result::Result<Vec<_>, _>>()?;
+    let roots = if sub.stdin {
+        io::stdin()
+            .lock()
+            .lines()
+            .flatten()
+            .map(|p| store.follow_store_path(p.trim()))
+            .collect::<std::result::Result<Vec<_>, _>>()?
+    } else {
+        sub.paths
+            .clone()
+            .into_iter()
+            .map(|p| store.follow_store_path(p))
+            .collect::<std::result::Result<Vec<_>, _>>()?
+    };
 
     let (server_name, server, cache) = config.resolve_cache(&sub.cache)?;
 

From e8cf9548a8621658cd55ca241a09b002ce94ebd3 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 4 Oct 2024 08:07:56 -0600
Subject: [PATCH 52/56] client/push: Tweak --stdin flag help text, remove short
 flag

---
 client/src/command/push.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/command/push.rs b/client/src/command/push.rs
index 6eabb04..f1ea560 100644
--- a/client/src/command/push.rs
+++ b/client/src/command/push.rs
@@ -26,8 +26,8 @@ pub struct Push {
     /// The store paths to push.
     paths: Vec<PathBuf>,
 
-    /// Read paths from stdin
-    #[clap(short = 'i', long)]
+    /// Read paths from the standard input.
+    #[clap(long)]
     stdin: bool,
 
     /// Push the specified paths only and do not compute closures.

From d4b365b50329f823428b5167f07acbff0ec29122 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 4 Oct 2024 09:16:22 -0600
Subject: [PATCH 53/56] client/push: Support flushing a PushSession

---
 client/src/push.rs | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/client/src/push.rs b/client/src/push.rs
index 1b7194f..4abbccd 100644
--- a/client/src/push.rs
+++ b/client/src/push.rs
@@ -100,11 +100,17 @@ pub struct Pusher {
 /// seconds since the last path is queued or it's been 10 seconds in total.
 pub struct PushSession {
     /// Sender to the batching future.
-    sender: channel::Sender<Vec<StorePath>>,
+    sender: channel::Sender<SessionQueueCommand>,
+}
+
+enum SessionQueueCommand {
+    Paths(Vec<StorePath>),
+    Flush,
 }
 
 enum SessionQueuePoll {
     Paths(Vec<StorePath>),
+    Flush,
     Closed,
     TimedOut,
 }
@@ -284,7 +290,7 @@ impl PushSession {
         pusher: Arc<Pusher>,
         config: PushSessionConfig,
         known_paths_mutex: Arc<Mutex<HashSet<StorePathHash>>>,
-        receiver: channel::Receiver<Vec<StorePath>>,
+        receiver: channel::Receiver<SessionQueueCommand>,
     ) -> Result<()> {
         let mut roots = HashSet::new();
 
@@ -296,7 +302,8 @@ impl PushSession {
                     loop {
                         let poll = tokio::select! {
                             r = receiver.recv() => match r {
-                                Ok(paths) => SessionQueuePoll::Paths(paths),
+                                Ok(SessionQueueCommand::Paths(paths)) => SessionQueuePoll::Paths(paths),
+                                Ok(SessionQueueCommand::Flush) => SessionQueuePoll::Flush,
                                 _ => SessionQueuePoll::Closed,
                             },
                             _ = time::sleep(Duration::from_secs(2)) => SessionQueuePoll::TimedOut,
@@ -309,7 +316,7 @@ impl PushSession {
                             SessionQueuePoll::Closed => {
                                 break true;
                             }
-                            SessionQueuePoll::TimedOut => {
+                            SessionQueuePoll::Flush | SessionQueuePoll::TimedOut => {
                                 break false;
                             }
                         }
@@ -360,7 +367,14 @@ impl PushSession {
     /// Queues multiple store paths to be pushed.
     pub fn queue_many(&self, store_paths: Vec<StorePath>) -> Result<()> {
         self.sender
-            .send_blocking(store_paths)
+            .send_blocking(SessionQueueCommand::Paths(store_paths))
+            .map_err(|e| anyhow!(e))
+    }
+
+    /// Flushes the worker queue.
+    pub fn flush(&self) -> Result<()> {
+        self.sender
+            .send_blocking(SessionQueueCommand::Flush)
             .map_err(|e| anyhow!(e))
     }
 }

From 99f3fbdc3d5ab4c28923f42dacddfd3b4630c49b Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 4 Oct 2024 09:16:22 -0600
Subject: [PATCH 54/56] client/push: Support retrieving results from a
 PushSession

---
 client/src/push.rs | 59 +++++++++++++++++++++++++++++++---------------
 1 file changed, 40 insertions(+), 19 deletions(-)

diff --git a/client/src/push.rs b/client/src/push.rs
index 4abbccd..309bd4b 100644
--- a/client/src/push.rs
+++ b/client/src/push.rs
@@ -28,7 +28,7 @@ use bytes::Bytes;
 use futures::future::join_all;
 use futures::stream::{Stream, TryStreamExt};
 use indicatif::{HumanBytes, MultiProgress, ProgressBar, ProgressState, ProgressStyle};
-use tokio::sync::Mutex;
+use tokio::sync::{mpsc, Mutex};
 use tokio::task::{spawn, JoinHandle};
 use tokio::time;
 
@@ -101,16 +101,21 @@ pub struct Pusher {
 pub struct PushSession {
     /// Sender to the batching future.
     sender: channel::Sender<SessionQueueCommand>,
+
+    /// Receiver of results.
+    result_receiver: mpsc::Receiver<Result<HashMap<StorePath, Result<()>>>>,
 }
 
 enum SessionQueueCommand {
     Paths(Vec<StorePath>),
     Flush,
+    Terminate,
 }
 
 enum SessionQueuePoll {
     Paths(Vec<StorePath>),
     Flush,
+    Terminate,
     Closed,
     TimedOut,
 }
@@ -261,36 +266,36 @@ impl Pusher {
 impl PushSession {
     pub fn with_pusher(pusher: Pusher, config: PushSessionConfig) -> Self {
         let (sender, receiver) = channel::unbounded();
+        let (result_sender, result_receiver) = mpsc::channel(1);
 
         let known_paths_mutex = Arc::new(Mutex::new(HashSet::new()));
 
-        // FIXME
         spawn(async move {
-            let pusher = Arc::new(pusher);
-            loop {
-                if let Err(e) = Self::worker(
-                    pusher.clone(),
-                    config,
-                    known_paths_mutex.clone(),
-                    receiver.clone(),
-                )
-                .await
-                {
-                    eprintln!("Worker exited: {:?}", e);
-                } else {
-                    break;
-                }
+            if let Err(e) = Self::worker(
+                pusher,
+                config,
+                known_paths_mutex.clone(),
+                receiver.clone(),
+                result_sender.clone(),
+            )
+            .await
+            {
+                let _ = result_sender.send(Err(e)).await;
             }
         });
 
-        Self { sender }
+        Self {
+            sender,
+            result_receiver,
+        }
     }
 
     async fn worker(
-        pusher: Arc<Pusher>,
+        pusher: Pusher,
         config: PushSessionConfig,
         known_paths_mutex: Arc<Mutex<HashSet<StorePathHash>>>,
         receiver: channel::Receiver<SessionQueueCommand>,
+        result_sender: mpsc::Sender<Result<HashMap<StorePath, Result<()>>>>,
     ) -> Result<()> {
         let mut roots = HashSet::new();
 
@@ -304,6 +309,7 @@ impl PushSession {
                             r = receiver.recv() => match r {
                                 Ok(SessionQueueCommand::Paths(paths)) => SessionQueuePoll::Paths(paths),
                                 Ok(SessionQueueCommand::Flush) => SessionQueuePoll::Flush,
+                                Ok(SessionQueueCommand::Terminate) => SessionQueuePoll::Terminate,
                                 _ => SessionQueuePoll::Closed,
                             },
                             _ = time::sleep(Duration::from_secs(2)) => SessionQueuePoll::TimedOut,
@@ -313,7 +319,7 @@ impl PushSession {
                             SessionQueuePoll::Paths(store_paths) => {
                                 roots.extend(store_paths.into_iter());
                             }
-                            SessionQueuePoll::Closed => {
+                            SessionQueuePoll::Closed | SessionQueuePoll::Terminate => {
                                 break true;
                             }
                             SessionQueuePoll::Flush | SessionQueuePoll::TimedOut => {
@@ -359,11 +365,26 @@ impl PushSession {
             drop(known_paths);
 
             if done {
+                let result = pusher.wait().await;
+                result_sender.send(Ok(result)).await?;
                 return Ok(());
             }
         }
     }
 
+    /// Waits for all workers to terminate, returning all results.
+    pub async fn wait(mut self) -> Result<HashMap<StorePath, Result<()>>> {
+        self.flush()?;
+
+        // The worker might have died
+        let _ = self.sender.send(SessionQueueCommand::Terminate).await;
+
+        self.result_receiver
+            .recv()
+            .await
+            .expect("Nothing in result channel")
+    }
+
     /// Queues multiple store paths to be pushed.
     pub fn queue_many(&self, store_paths: Vec<StorePath>) -> Result<()> {
         self.sender

From 46b83a04d9e4b601b8c14bf9f61809d968c556b1 Mon Sep 17 00:00:00 2001
From: Albert Peschar <albert@peschar.net>
Date: Fri, 11 Aug 2023 16:08:41 +0300
Subject: [PATCH 55/56] Use --pipe for systemd-run

Without --pipe stderr and stdout are combined, breaking scripts that use
atticd-atticadm when it generates warnings.
---
 integration-tests/basic/default.nix | 4 ++--
 nixos/atticd.nix                    | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/integration-tests/basic/default.nix b/integration-tests/basic/default.nix
index d6b17bc..fbdc66e 100644
--- a/integration-tests/basic/default.nix
+++ b/integration-tests/basic/default.nix
@@ -186,8 +186,8 @@ in {
       server.wait_for_unit('atticd.service')
       client.wait_until_succeeds("curl -sL http://server:8080", timeout=40)
 
-      root_token = server.succeed("${cmd.atticadm} make-token --sub 'e2e-root' --validity '1 month' --push '*' --pull '*' --delete '*' --create-cache '*' --destroy-cache '*' --configure-cache '*' --configure-cache-retention '*'").strip()
-      readonly_token = server.succeed("${cmd.atticadm} make-token --sub 'e2e-root' --validity '1 month' --pull 'test'").strip()
+      root_token = server.succeed("${cmd.atticadm} make-token --sub 'e2e-root' --validity '1 month' --push '*' --pull '*' --delete '*' --create-cache '*' --destroy-cache '*' --configure-cache '*' --configure-cache-retention '*' </dev/null").strip()
+      readonly_token = server.succeed("${cmd.atticadm} make-token --sub 'e2e-root' --validity '1 month' --pull 'test' </dev/null").strip()
 
       client.succeed(f"attic login --set-default root http://server:8080 {root_token}")
       client.succeed(f"attic login readonly http://server:8080 {readonly_token}")
diff --git a/nixos/atticd.nix b/nixos/atticd.nix
index f14141a..7697e94 100644
--- a/nixos/atticd.nix
+++ b/nixos/atticd.nix
@@ -36,6 +36,7 @@ let
   atticadmWrapper = pkgs.writeShellScriptBin "atticd-atticadm" ''
     exec systemd-run \
       --quiet \
+      --pipe \
       --pty \
       --same-dir \
       --wait \

From c5764fca3bd9678efe544dc71c9243783ca41e35 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Fri, 4 Oct 2024 09:16:22 -0600
Subject: [PATCH 56/56] client/push: Use PushSession for --stdin

Instead of eagerly consuming stdin, read line-by-line and feed into
PushSession. This allows for a `nix-build | attic push` workflow.
---
 client/src/command/push.rs | 146 ++++++++++++++++++++++++-------------
 1 file changed, 97 insertions(+), 49 deletions(-)

diff --git a/client/src/command/push.rs b/client/src/command/push.rs
index f1ea560..c93d22d 100644
--- a/client/src/command/push.rs
+++ b/client/src/command/push.rs
@@ -1,17 +1,16 @@
-use std::io;
-use std::io::BufRead;
 use std::path::PathBuf;
 use std::sync::Arc;
 
 use anyhow::{anyhow, Result};
 use clap::Parser;
 use indicatif::MultiProgress;
+use tokio::io::{self, AsyncBufReadExt, BufReader};
 
 use crate::api::ApiClient;
-use crate::cache::CacheRef;
+use crate::cache::{CacheName, CacheRef, ServerName};
 use crate::cli::Opts;
 use crate::config::Config;
-use crate::push::{PushConfig, Pusher};
+use crate::push::{PushConfig, PushSessionConfig, Pusher};
 use attic::nix_store::NixStore;
 
 /// Push closures to a binary cache.
@@ -47,6 +46,79 @@ pub struct Push {
     force_preamble: bool,
 }
 
+struct PushContext {
+    store: Arc<NixStore>,
+    cache_name: CacheName,
+    server_name: ServerName,
+    pusher: Pusher,
+    no_closure: bool,
+    ignore_upstream_cache_filter: bool,
+}
+
+impl PushContext {
+    async fn push_static(self, paths: Vec<PathBuf>) -> Result<()> {
+        let roots = paths
+            .into_iter()
+            .map(|p| self.store.follow_store_path(p))
+            .collect::<std::result::Result<Vec<_>, _>>()?;
+
+        let plan = self
+            .pusher
+            .plan(roots, self.no_closure, self.ignore_upstream_cache_filter)
+            .await?;
+
+        if plan.store_path_map.is_empty() {
+            if plan.num_all_paths == 0 {
+                eprintln!("🤷 Nothing selected.");
+            } else {
+                eprintln!(
+                    "✅ All done! ({num_already_cached} already cached, {num_upstream} in upstream)",
+                    num_already_cached = plan.num_already_cached,
+                    num_upstream = plan.num_upstream,
+                );
+            }
+
+            return Ok(());
+        } else {
+            eprintln!("⚙️ Pushing {num_missing_paths} paths to \"{cache}\" on \"{server}\" ({num_already_cached} already cached, {num_upstream} in upstream)...",
+                cache = self.cache_name.as_str(),
+                server = self.server_name.as_str(),
+                num_missing_paths = plan.store_path_map.len(),
+                num_already_cached = plan.num_already_cached,
+                num_upstream = plan.num_upstream,
+            );
+        }
+
+        for (_, path_info) in plan.store_path_map {
+            self.pusher.queue(path_info).await?;
+        }
+
+        let results = self.pusher.wait().await;
+        results.into_values().collect::<Result<Vec<()>>>()?;
+
+        Ok(())
+    }
+
+    async fn push_stdin(self) -> Result<()> {
+        let session = self.pusher.into_push_session(PushSessionConfig {
+            no_closure: self.no_closure,
+            ignore_upstream_cache_filter: self.ignore_upstream_cache_filter,
+        });
+
+        let stdin = BufReader::new(io::stdin());
+        let mut lines = stdin.lines();
+        while let Some(line) = lines.next_line().await? {
+            let path = self.store.follow_store_path(line)?;
+            session.queue_many(vec![path])?;
+        }
+
+        let results = session.wait().await?;
+        results.into_values().collect::<Result<Vec<()>>>()?;
+
+        Ok(())
+    }
+}
+
 pub async fn run(opts: Opts) -> Result<()> {
     let sub = opts.command.as_push().unwrap();
     if sub.jobs == 0 {
@@ -56,27 +128,13 @@ pub async fn run(opts: Opts) -> Result<()> {
     let config = Config::load()?;
 
     let store = Arc::new(NixStore::connect()?);
-    let roots = if sub.stdin {
-        io::stdin()
-            .lock()
-            .lines()
-            .flatten()
-            .map(|p| store.follow_store_path(p.trim()))
-            .collect::<std::result::Result<Vec<_>, _>>()?
-    } else {
-        sub.paths
-            .clone()
-            .into_iter()
-            .map(|p| store.follow_store_path(p))
-            .collect::<std::result::Result<Vec<_>, _>>()?
-    };
 
-    let (server_name, server, cache) = config.resolve_cache(&sub.cache)?;
+    let (server_name, server, cache_name) = config.resolve_cache(&sub.cache)?;
 
     let mut api = ApiClient::from_server_config(server.clone())?;
 
     // Confirm remote cache validity, query cache config
-    let cache_config = api.get_cache_config(cache).await?;
+    let cache_config = api.get_cache_config(cache_name).await?;
 
     if let Some(api_endpoint) = &cache_config.api_endpoint {
         // Use delegated API endpoint
@@ -90,39 +148,29 @@ pub async fn run(opts: Opts) -> Result<()> {
 
     let mp = MultiProgress::new();
 
-    let pusher = Pusher::new(store, api, cache.to_owned(), cache_config, mp, push_config);
-    let plan = pusher
-        .plan(roots, sub.no_closure, sub.ignore_upstream_cache_filter)
-        .await?;
+    let pusher = Pusher::new(
+        store.clone(),
+        api,
+        cache_name.to_owned(),
+        cache_config,
+        mp,
+        push_config,
+    );
 
-    if plan.store_path_map.is_empty() {
-        if plan.num_all_paths == 0 {
-            eprintln!("🤷 Nothing selected.");
-        } else {
-            eprintln!(
-                "✅ All done! ({num_already_cached} already cached, {num_upstream} in upstream)",
-                num_already_cached = plan.num_already_cached,
-                num_upstream = plan.num_upstream,
-            );
-        }
+    let push_ctx = PushContext {
+        store,
+        cache_name: cache_name.clone(),
+        server_name: server_name.clone(),
+        pusher,
+        no_closure: sub.no_closure,
+        ignore_upstream_cache_filter: sub.ignore_upstream_cache_filter,
+    };
 
-        return Ok(());
+    if sub.stdin {
+        push_ctx.push_stdin().await?;
     } else {
-        eprintln!("⚙️ Pushing {num_missing_paths} paths to \"{cache}\" on \"{server}\" ({num_already_cached} already cached, {num_upstream} in upstream)...",
-            cache = cache.as_str(),
-            server = server_name.as_str(),
-            num_missing_paths = plan.store_path_map.len(),
-            num_already_cached = plan.num_already_cached,
-            num_upstream = plan.num_upstream,
-        );
+        push_ctx.push_static(sub.paths.clone()).await?;
     }
 
-    for (_, path_info) in plan.store_path_map {
-        pusher.queue(path_info).await?;
-    }
-
-    let results = pusher.wait().await;
-    results.into_values().collect::<Result<Vec<()>>>()?;
-
     Ok(())
 }