mirrors/attic
1
0
Fork 0
mirror of https://github.com/zhaofengli/attic.git synced 2024-12-14 11:57:30 +00:00
Code Activity

Merge pull request #165 from zhaofengli/flake-parts

Browse source 
Migrate to flake-parts
This commit is contained in:
Zhaofeng Li 2024-08-30 14:08:15 -06:00 committed by GitHub
commit f74cee0036
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 507 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.ci/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/cached-shell

5
.ci/cache-shell.sh Executable file
View file

@ -0,0 +1,5 @@
#!/usr/bin/env bash
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
>&2 echo "Caching dev shell"
nix print-dev-env "${base}#" >"${cached_shell}"

7
.ci/common.sh Normal file
View file

@ -0,0 +1,7 @@
# Use as:
#
# source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
set -euo pipefail
base="$(readlink -f $(dirname "${BASH_SOURCE[0]}")/..)"
cached_shell="${base}/.ci/cached-shell"

10
.ci/run Executable file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env bash
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
if [[ ! -f "${cached_shell}" ]]; then
>&2 echo "No cached shell in ${cached_shell}"
exit 1
fi
. "${cached_shell}"
exec "$@"

24
.github/workflows/build.yml vendored
View file

@ -23,6 +23,11 @@ jobs:
steps:
- uses: actions/checkout@v4.1.1
- name: Install current Bash on macOS
if: runner.os == 'macOS'
run: |
command -v brew && brew install bash || true
- uses: DeterminateSystems/nix-installer-action@v9
continue-on-error: true # Self-hosted runners already have Nix installed
@ -45,25 +50,20 @@ jobs:
ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }}
ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
- name: Build and run tests
- name: Cache dev shell
run: |
.ci/cache-shell.sh
system=$(nix-instantiate --eval -E 'builtins.currentSystem')
echo system=$system >>$GITHUB_ENV
tests=$(nix build .#internalMatrix."$system".\"${{ matrix.nix }}\".attic-tests --no-link --print-out-paths -L)
find "$tests/bin" -exec {} \;
- name: Run unit tests
run: |
.ci/run just ci-unit-tests ${{ matrix.nix }}
- name: Build WebAssembly crates
if: runner.os == 'Linux'
run: |
# https://github.com/rust-lang/rust/issues/122357
export RUST_MIN_STACK=16777216
pushd attic
nix develop .# --command -- cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream
popd
pushd token
nix develop .# --command -- cargo build --target wasm32-unknown-unknown
popd
.ci/run just ci-build-wasm
# TODO: Just take a diff of the list of store paths, also abstract all of this out
- name: Push build artifacts

35
flake.lock
View file

@ -36,6 +36,26 @@
"type": "github"
}
},
"flake-parts": {
"inputs": {
"nixpkgs-lib": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1722555600,
"narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
@ -56,11 +76,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1723827930,
"narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=",
"lastModified": 1724999960,
"narHash": "sha256-LB3jqSGW5u1ZcUcX6vO/qBOq5oXHlmOCxsTXGMEitp4=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d",
"rev": "b96f849e725333eb2b1c7f1cb84ff102062468ba",
"type": "github"
},
"original": {
@ -72,16 +92,16 @@
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1720535198,
"narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=",
"lastModified": 1724316499,
"narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5",
"rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"ref": "nixos-24.05",
"repo": "nixpkgs",
"type": "github"
}
@ -90,6 +110,7 @@
"inputs": {
"crane": "crane",
"flake-compat": "flake-compat",
"flake-parts": "flake-parts",
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"nixpkgs-stable": "nixpkgs-stable"

203
flake.nix
View file

@ -3,9 +3,14 @@
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-23.11";
nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
flake-utils.url = "github:numtide/flake-utils";
flake-parts = {
url = "github:hercules-ci/flake-parts";
inputs.nixpkgs-lib.follows = "nixpkgs";
};
crane = {
url = "github:ipetkov/crane";
inputs.nixpkgs.follows = "nixpkgs";
@ -17,189 +22,25 @@
};
};
outputs = { self, nixpkgs, nixpkgs-stable, flake-utils, crane, ... }: let
supportedSystems = flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];
outputs = inputs @ { self, flake-parts, ... }: let
supportedSystems = inputs.flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];
makeCranePkgs = pkgs: let
craneLib = crane.mkLib pkgs;
in pkgs.callPackage ./crane.nix { inherit craneLib; };
in flake-utils.lib.eachSystem supportedSystems (system: let
pkgs = import nixpkgs {
inherit system;
overlays = [];
};
cranePkgs = makeCranePkgs pkgs;
inherit (inputs.nixpkgs) lib;
internalMatrix = lib.mapAttrs (_: nix: let
cranePkgs' = cranePkgs.override { inherit nix; };
in {
inherit (cranePkgs') attic-tests cargoArtifacts;
}) {
"2.20" = pkgs.nixVersions.nix_2_20;
"2.24" = pkgs.nixVersions.nix_2_24;
"default" = pkgs.nix;
};
modules = builtins.foldl' (acc: f: f acc) ./flake [
builtins.readDir
(lib.filterAttrs (name: type:
type == "regular" && lib.hasSuffix ".nix" name
))
(lib.mapAttrsToList (name: _:
lib.path.append ./flake name
))
];
pkgsStable = import nixpkgs-stable {
inherit system;
overlays = [];
};
cranePkgsStable = makeCranePkgs pkgsStable;
in flake-parts.lib.mkFlake { inherit inputs; } {
imports = modules;
systems = supportedSystems;
inherit (pkgs) lib;
in rec {
inherit internalMatrix;
packages = {
default = packages.attic;
inherit (cranePkgs) attic attic-client attic-server;
attic-nixpkgs = pkgs.callPackage ./package.nix { };
attic-ci-installer = pkgs.callPackage ./ci-installer.nix {
inherit self;
};
book = pkgs.callPackage ./book {
attic = packages.attic;
};
} // (lib.optionalAttrs (system != "x86_64-darwin") {
# Unfortunately, x86_64-darwin fails to evaluate static builds
# TODO: Make this work with Crane
attic-static = (pkgs.pkgsStatic.callPackage ./package.nix {
nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
patches = (old.patches or []) ++ [
# To be submitted
(pkgs.fetchpatch {
url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
})
];
});
}).overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
pkgs.nukeReferences
];
# Read by pkg_config crate (do some autodetection in build.rs?)
PKG_CONFIG_ALL_STATIC = "1";
"NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
RUSTFLAGS = "-C relocation-model=static";
postFixup = (old.postFixup or "") + ''
rm -f $out/nix-support/propagated-build-inputs
nuke-refs $out/bin/attic
'';
});
attic-client-static = packages.attic-static.override {
clientOnly = true;
};
}) // (lib.optionalAttrs pkgs.stdenv.isLinux {
attic-server-image = pkgs.dockerTools.buildImage {
name = "attic-server";
tag = "main";
copyToRoot = [
# Debugging utilities for `fly ssh console`
pkgs.busybox
packages.attic-server
# Now required by the fly.io sshd
pkgs.dockerTools.fakeNss
];
config = {
Entrypoint = [ "${packages.attic-server}/bin/atticd" ];
Env = [
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
];
};
};
});
devShells = {
default = pkgs.mkShell {
inputsFrom = with packages; [ attic book ];
nativeBuildInputs = with pkgs; [
rustc
rustfmt clippy
cargo-expand
# Temporary broken: https://github.com/NixOS/nixpkgs/pull/335152
# cargo-outdated
cargo-edit
tokio-console
sqlite-interactive
editorconfig-checker
flyctl
wrk
llvmPackages_latest.bintools
wrangler worker-build wasm-pack wasm-bindgen-cli
] ++ (lib.optionals pkgs.stdenv.isLinux [
linuxPackages.perf
]);
NIX_PATH = "nixpkgs=${pkgs.path}";
RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
# See comment in `attic/build.rs`
NIX_INCLUDE_PATH = "${lib.getDev pkgs.nix}/include";
ATTIC_DISTRIBUTOR = "dev";
};
demo = pkgs.mkShell {
nativeBuildInputs = [
packages.default
];
shellHook = ''
>&2 echo
>&2 echo '🚀 Run `atticd` to get started!'
>&2 echo
'';
};
};
devShell = devShells.default;
checks = let
makeIntegrationTests = pkgs: import ./integration-tests {
pkgs = import nixpkgs {
inherit system;
overlays = [ self.overlays.default ];
};
flake = self;
};
unstableTests = makeIntegrationTests pkgs;
stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") (makeIntegrationTests pkgsStable);
in lib.optionalAttrs pkgs.stdenv.isLinux (unstableTests // stableTests);
}) // {
overlays = {
default = final: prev: let
cranePkgs = makeCranePkgs final;
in {
inherit (cranePkgs) attic attic-client attic-server;
};
};
nixosModules = {
atticd = {
imports = [
./nixos/atticd.nix
];
services.atticd.useFlakeCompatOverlay = false;
nixpkgs.overlays = [
self.overlays.default
];
};
};
debug = true;
};
}

114
flake/devshells.nix Normal file
View file

@ -0,0 +1,114 @@
# Development shells
toplevel @ { lib, flake-parts-lib, ... }:
let
inherit (lib)
mkOption
types
;
inherit (flake-parts-lib)
mkPerSystemOption
;
in
{
options = {
perSystem = mkPerSystemOption {
options.attic.devshell = {
packageSets = mkOption {
type = types.attrsOf (types.listOf types.package);
default = {};
};
extraPackages = mkOption {
type = types.listOf types.package;
default = [];
};
extraArgs = mkOption {
type = types.attrsOf types.unspecified;
default = {};
};
};
};
};
config = {
perSystem = { self', pkgs, config, ... }: let
cfg = config.attic.devshell;
in {
attic.devshell.packageSets = with pkgs; {
rust = [
rustc
cargo-expand
cargo-outdated
cargo-edit
tokio-console
];
linters = [
clippy
rustfmt
editorconfig-checker
];
utils = [
jq
just
];
ops = [
postgresql
sqlite-interactive
flyctl
] ++ lib.optionals pkgs.stdenv.isLinux [
wrangler
];
bench = [
wrk
] ++ lib.optionals pkgs.stdenv.isLinux [
linuxPackages.perf
];
wasm = [
llvmPackages_latest.bintools
worker-build wasm-pack wasm-bindgen-cli
];
};
devShells.default = pkgs.mkShell (lib.recursiveUpdate {
inputsFrom = [
self'.packages.attic
self'.packages.book
];
packages = lib.flatten (lib.attrValues cfg.packageSets);
env = {
ATTIC_DISTRIBUTOR = toplevel.config.attic.distributor;
RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
NIX_PATH = "nixpkgs=${pkgs.path}";
# See comment in `attic/build.rs`
NIX_INCLUDE_PATH = "${lib.getDev pkgs.nixVersions.nix_2_24}/include";
# Used by `just with-nix` to build/test with alternative Nix versions.
NIX_VERSIONS = config.attic.nix-versions.manifestFile;
};
} cfg.extraArgs);
devShells.demo = pkgs.mkShell {
packages = [ self'.packages.default ];
shellHook = ''
>&2 echo
>&2 echo '🚀 Run `atticd` to get started!'
>&2 echo
'';
};
};
};
}

15
flake/distributor.nix Normal file
View file

@ -0,0 +1,15 @@
{ lib, flake-parts-lib, ... }:
let
inherit (lib)
mkOption
types
;
in
{
options = {
attic.distributor = mkOption {
type = types.str;
default = "dev";
};
};
}

60
flake/integration-tests.nix Normal file
View file

@ -0,0 +1,60 @@
{ lib, flake-parts-lib, inputs, self, ... }:
let
inherit (lib)
mkOption
types
;
inherit (flake-parts-lib)
mkPerSystemOption
;
in
{
options = {
perSystem = mkPerSystemOption {
options.attic.integration-tests = {
nixpkgsArgs = mkOption {
type = types.attrsOf types.anything;
default = {};
};
tests = mkOption {
type = types.attrsOf types.package;
default = {};
};
stableTests = mkOption {
type = types.attrsOf types.package;
default = {};
};
};
};
};
config = {
perSystem = { self', pkgs, config, system, ... }: let
cfg = config.attic.integration-tests;
vmPkgs = import inputs.nixpkgs ({
inherit system;
overlays = [ self.overlays.default ];
} // cfg.nixpkgsArgs);
vmPkgsStable = import inputs.nixpkgs-stable ({
inherit system;
overlays = [ self.overlays.default ];
} // cfg.nixpkgsArgs);
makeIntegrationTests = pkgs: import ../integration-tests {
inherit pkgs;
flake = self;
};
in {
attic.integration-tests = {
tests = makeIntegrationTests vmPkgs;
stableTests = makeIntegrationTests vmPkgsStable;
};
checks = let
tests = cfg.tests;
stableTests = lib.mapAttrs' (name: lib.nameValuePair "stable-${name}") cfg.stableTests;
in lib.optionalAttrs pkgs.stdenv.isLinux (tests // stableTests);
};
};
}

64
flake/nix-versions.nix Normal file
View file

@ -0,0 +1,64 @@
{ lib, flake-parts-lib, config, ... }:
let
inherit (lib)
mkOption
types
;
inherit (flake-parts-lib)
mkPerSystemOption
;
in
{
options = {
perSystem = mkPerSystemOption {
options.attic.nix-versions = {
versions = mkOption {
type = types.attrsOf types.package;
default = {};
};
manifestFile = mkOption {
type = types.package;
};
};
options.internalMatrix = mkOption {
type = types.attrsOf (types.attrsOf types.package);
};
};
};
config = {
flake.internalMatrix = lib.mapAttrs (system: ps: ps.internalMatrix) config.allSystems;
perSystem = { self', pkgs, config, cranePkgs, ... }: let
cfg = config.attic.nix-versions;
in {
attic.nix-versions = {
versions = {
default = pkgs.nix;
"2.20" = pkgs.nixVersions.nix_2_20;
"2.24" = pkgs.nixVersions.nix_2_24;
};
manifestFile = let
manifest = lib.mapAttrs (_: nix: {
inherit nix;
shellHook = ''
export NIX_INCLUDE_PATH="${lib.getDev nix}/include"
export NIX_CFLAGS_COMPILE="-isystem $NIX_INCLUDE_PATH $NIX_CFLAGS_COMPILE"
export NIX_LDFLAGS="-L${nix}/lib $NIX_LDFLAGS"
export PKG_CONFIG_PATH="${lib.getDev nix}/lib/pkgconfig:$PKG_CONFIG_PATH"
export PATH="${lib.getBin nix}/bin:$PATH"
'';
}) cfg.versions;
in pkgs.writeText "nix-versions.json" (builtins.toJSON manifest);
};
internalMatrix = lib.mapAttrs (_: nix: let
cranePkgs' = cranePkgs.override { inherit nix; };
in {
inherit (cranePkgs') attic-tests cargoArtifacts;
}) cfg.versions;
};
};
}

16
flake/nixos.nix Normal file
View file

@ -0,0 +1,16 @@
{ config, ... }:
{
flake.nixosModules = {
atticd = {
imports = [
../nixos/atticd.nix
];
services.atticd.useFlakeCompatOverlay = false;
nixpkgs.overlays = [
config.flake.overlays.default
];
};
};
}

14
flake/overlays.nix Normal file
View file

@ -0,0 +1,14 @@
{ makeCranePkgs, ... }:
{
flake.overlays = {
default = final: prev: let
cranePkgs = makeCranePkgs final;
in {
inherit (cranePkgs)
attic
attic-client
attic-server
;
};
};
}

96
flake/packages.nix Normal file
View file

@ -0,0 +1,96 @@
{ self, inputs, lib, ... }:
let
makeCranePkgs = pkgs: let
craneLib = inputs.crane.mkLib pkgs;
in pkgs.callPackage ../crane.nix { inherit craneLib; };
in
{
_module.args.makeCranePkgs = makeCranePkgs;
perSystem = { self', pkgs, cranePkgs, ... }: (lib.mkMerge [
{
_module.args.cranePkgs = makeCranePkgs pkgs;
packages = {
default = self'.packages.attic;
inherit (cranePkgs)
attic
attic-client
attic-server
;
attic-nixpkgs = pkgs.callPackage ../package.nix { };
attic-ci-installer = pkgs.callPackage ../ci-installer.nix {
inherit self;
};
book = pkgs.callPackage ../book {
attic = self'.packages.attic;
};
};
}
(lib.mkIf pkgs.stdenv.isLinux {
packages = {
attic-server-image = pkgs.dockerTools.buildImage {
name = "attic-server";
tag = "main";
copyToRoot = [
self'.packages.attic-server
# Debugging utilities for `fly ssh console`
pkgs.busybox
# Now required by the fly.io sshd
pkgs.dockerTools.fakeNss
];
config = {
Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ];
Env = [
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
];
};
};
};
})
# Unfortunately, x86_64-darwin fails to evaluate static builds
(lib.mkIf (pkgs.system != "x86_64-darwin") {
packages = {
# TODO: Make this work with Crane
attic-static = (pkgs.pkgsStatic.callPackage ../package.nix {
nix = pkgs.pkgsStatic.nix.overrideAttrs (old: {
patches = (old.patches or []) ++ [
# To be submitted
(pkgs.fetchpatch {
url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff";
hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c=";
})
];
});
}).overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or []) ++ [
pkgs.nukeReferences
];
# Read by pkg_config crate (do some autodetection in build.rs?)
PKG_CONFIG_ALL_STATIC = "1";
"NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc";
RUSTFLAGS = "-C relocation-model=static";
postFixup = (old.postFixup or "") + ''
rm -f $out/nix-support/propagated-build-inputs
nuke-refs $out/bin/attic
'';
});
attic-client-static = self'.packages.attic-static.override {
clientOnly = true;
};
};
})
]);
}

43
justfile Normal file
View file

@ -0,0 +1,43 @@
set positional-arguments
here := env_var_or_default("JUST_INVOCATION_DIR", invocation_directory())
base := `pwd`
#@echo "here: {{ here }}"
#@echo "base: {{ base }}"
# List available targets
list:
@just --list --unsorted
# Run a command with an alternative Nix version
with-nix version *command:
set -e; \
hook="$(jq -e -r '.[$version].shellHook' --arg version "{{ version }}" < "$NIX_VERSIONS" || (>&2 echo "Version {{ version }} doesn't exist"; exit 1))"; \
eval "$hook"; \
CARGO_TARGET_DIR="{{ base }}/target/nix-{{ version }}" \
{{ command }}
# (CI) Build WebAssembly crates
ci-build-wasm:
#!/usr/bin/env bash
set -euxo pipefail
# https://github.com/rust-lang/rust/issues/122357
export RUST_MIN_STACK=16777216
pushd attic
cargo build --target wasm32-unknown-unknown --no-default-features -F chunking -F stream
popd
pushd token
cargo build --target wasm32-unknown-unknown
popd
# (CI) Run unit tests
ci-unit-tests matrix:
#!/usr/bin/env bash
set -euxo pipefail
system=$(nix-instantiate --eval -E 'builtins.currentSystem')
tests=$(nix build .#internalMatrix."$system".\"{{ matrix }}\".attic-tests --no-link --print-out-paths -L)
find "$tests/bin" -exec {} \;