mirror of
https://github.com/zhaofengli/attic.git
synced 2024-12-14 11:57:30 +00:00
Merge remote-tracking branch 'upstream/main' into rs256-support
This commit is contained in:
Cole Helbling
commit
41b42b62d5
20 changed files with 1631 additions and 1157 deletions
8
.github/workflows/book.yml
vendored
8
.github/workflows/book.yml
vendored
|
|
@ -16,9 +16,9 @@ jobs:
|
|||
if: github.repository == 'zhaofengli/attic'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3.0.2
|
||||
- uses: actions/checkout@v4.1.1
|
||||
|
||||
- uses: DeterminateSystems/nix-installer-action@v1
|
||||
- uses: DeterminateSystems/nix-installer-action@v9
|
||||
continue-on-error: true # Self-hosted runners already have Nix installed
|
||||
|
||||
- name: Install Attic
|
||||
|
|
@ -40,12 +40,12 @@ jobs:
|
|||
cp --recursive --dereference --no-preserve=mode,ownership result public
|
||||
|
||||
- name: Upload book artifact
|
||||
uses: actions/upload-pages-artifact@v1.0.7
|
||||
uses: actions/upload-pages-artifact@v2.0.0
|
||||
with:
|
||||
path: public
|
||||
|
||||
- name: Deploy book
|
||||
uses: actions/deploy-pages@v1.2.3
|
||||
uses: actions/deploy-pages@v3.0.1
|
||||
|
||||
# TODO: Just take a diff of the list of store paths, also abstract all of this out
|
||||
- name: Push build artifacts
|
||||
|
|
|
|||
6
.github/workflows/build.yml
vendored
6
.github/workflows/build.yml
vendored
|
|
@ -17,9 +17,9 @@ jobs:
|
|||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v3.3.0
|
||||
- uses: actions/checkout@v4.1.1
|
||||
|
||||
- uses: DeterminateSystems/nix-installer-action@v1
|
||||
- uses: DeterminateSystems/nix-installer-action@v9
|
||||
continue-on-error: true # Self-hosted runners already have Nix installed
|
||||
|
||||
- name: Install Attic
|
||||
|
|
@ -57,7 +57,7 @@ jobs:
|
|||
xargs attic push "ci:$ATTIC_CACHE"
|
||||
fi
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
||||
uses: docker/login-action@v3.0.0
|
||||
if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
|
|
|
|||
2432
Cargo.lock
generated
2432
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -17,9 +17,9 @@ bytes = "1.4.0"
|
|||
clap = { version = "4.3", features = ["derive"] }
|
||||
clap_complete = "4.3.0"
|
||||
const_format = "0.2.30"
|
||||
dialoguer = "0.10.4"
|
||||
dialoguer = "0.11.0"
|
||||
displaydoc = "0.2.4"
|
||||
enum-as-inner = "0.5.2"
|
||||
enum-as-inner = "0.6.0"
|
||||
futures = "0.3.28"
|
||||
humantime = "2.1.0"
|
||||
indicatif = "0.17.3"
|
||||
|
|
@ -29,7 +29,7 @@ regex = "1.8.3"
|
|||
reqwest = { version = "0.11.18", default-features = false, features = ["json", "rustls-tls", "rustls-tls-native-roots", "stream"] }
|
||||
serde = { version = "1.0.163", features = ["derive"] }
|
||||
serde_json = "1.0.96"
|
||||
toml = "0.7.4"
|
||||
toml = "0.8.8"
|
||||
tracing = "0.1.37"
|
||||
tracing-subscriber = "0.3.17"
|
||||
xdg = "2.5.0"
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ pub struct StructuredApiError {
|
|||
|
||||
impl ApiClient {
|
||||
pub fn from_server_config(config: ServerConfig) -> Result<Self> {
|
||||
let client = build_http_client(config.token.as_deref());
|
||||
let client = build_http_client(config.token()?.as_deref());
|
||||
|
||||
Ok(Self {
|
||||
endpoint: Url::parse(&config.endpoint)?,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ use std::str::FromStr;
|
|||
use anyhow::{anyhow, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use attic::cache::{CacheName, CacheNamePattern};
|
||||
pub use attic::cache::{CacheName};
|
||||
|
||||
/// A reference to a cache.
|
||||
#[derive(Debug, Clone)]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use clap::Parser;
|
|||
|
||||
use crate::cache::ServerName;
|
||||
use crate::cli::Opts;
|
||||
use crate::config::{Config, ServerConfig};
|
||||
use crate::config::{Config, ServerConfig, ServerTokenConfig};
|
||||
|
||||
/// Log into an Attic server.
|
||||
#[derive(Debug, Parser)]
|
||||
|
|
@ -32,8 +32,10 @@ pub async fn run(opts: Opts) -> Result<()> {
|
|||
|
||||
server.endpoint = sub.endpoint.to_owned();
|
||||
|
||||
if sub.token.is_some() {
|
||||
server.token = sub.token.to_owned();
|
||||
if let Some(token) = &sub.token {
|
||||
server.token = Some(ServerTokenConfig::Raw {
|
||||
token: token.clone(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
eprintln!("✍️ Configuring server \"{}\"", sub.name.as_str());
|
||||
|
|
@ -42,7 +44,10 @@ pub async fn run(opts: Opts) -> Result<()> {
|
|||
sub.name.to_owned(),
|
||||
ServerConfig {
|
||||
endpoint: sub.endpoint.to_owned(),
|
||||
token: sub.token.to_owned(),
|
||||
token: sub
|
||||
.token
|
||||
.to_owned()
|
||||
.map(|token| ServerTokenConfig::Raw { token }),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ pub async fn run(opts: Opts) -> Result<()> {
|
|||
nix_config.add_trusted_public_key(&public_key);
|
||||
|
||||
// Modify netrc
|
||||
if let Some(token) = &server.token {
|
||||
if let Some(token) = server.token()? {
|
||||
eprintln!("+ Access Token");
|
||||
|
||||
let mut nix_netrc = NixNetrc::load().await?;
|
||||
|
|
|
|||
|
|
@ -5,13 +5,13 @@
|
|||
//! experience (e.g., `attic login`).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, OpenOptions, Permissions};
|
||||
use std::fs::{self, read_to_string, OpenOptions, Permissions};
|
||||
use std::io::Write;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use xdg::BaseDirectories;
|
||||
|
||||
|
|
@ -52,7 +52,38 @@ pub struct ConfigData {
|
|||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ServerConfig {
|
||||
pub endpoint: String,
|
||||
pub token: Option<String>,
|
||||
#[serde(flatten)]
|
||||
pub token: Option<ServerTokenConfig>,
|
||||
}
|
||||
|
||||
impl ServerConfig {
|
||||
pub fn token(&self) -> Result<Option<String>> {
|
||||
self.token.as_ref().map(|token| token.get()).transpose()
|
||||
}
|
||||
}
|
||||
|
||||
/// Configured server token
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ServerTokenConfig {
|
||||
Raw {
|
||||
token: String,
|
||||
},
|
||||
File {
|
||||
#[serde(rename = "token-file")]
|
||||
token_file: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl ServerTokenConfig {
|
||||
/// Get the token either directly from the config or through the token file
|
||||
pub fn get(&self) -> Result<String> {
|
||||
match self {
|
||||
ServerTokenConfig::Raw { token } => Ok(token.clone()),
|
||||
ServerTokenConfig::File { token_file } => Ok(read_to_string(token_file)
|
||||
.with_context(|| format!("Failed to read token from {token_file}"))?),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that automatically saves the config once dropped.
|
||||
|
|
|
|||
10
crane.nix
10
crane.nix
|
|
@ -62,6 +62,11 @@ let
|
|||
|
||||
ATTIC_DISTRIBUTOR = "attic";
|
||||
|
||||
# Workaround for https://github.com/NixOS/nixpkgs/issues/166205
|
||||
env = lib.optionalAttrs stdenv.cc.isClang {
|
||||
NIX_LDFLAGS = "-l${stdenv.cc.libcxx.cxxabi.libName}";
|
||||
};
|
||||
|
||||
# See comment in `attic-tests`
|
||||
doCheck = false;
|
||||
|
||||
|
|
@ -130,6 +135,11 @@ let
|
|||
|
||||
nativeBuildInputs = nativeBuildInputs ++ [ jq ];
|
||||
|
||||
# Workaround for https://github.com/NixOS/nixpkgs/issues/166205
|
||||
env = lib.optionalAttrs stdenv.cc.isClang {
|
||||
NIX_LDFLAGS = "-l${stdenv.cc.libcxx.cxxabi.libName}";
|
||||
};
|
||||
|
||||
doCheck = true;
|
||||
|
||||
buildPhaseCargoCommand = "";
|
||||
|
|
|
|||
54
flake.lock
54
flake.lock
|
|
@ -2,23 +2,16 @@
|
|||
"nodes": {
|
||||
"crane": {
|
||||
"inputs": {
|
||||
"flake-compat": [
|
||||
"flake-compat"
|
||||
],
|
||||
"flake-utils": [
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"rust-overlay": "rust-overlay"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1677892403,
|
||||
"narHash": "sha256-/Wi0L1spSWLFj+UQxN3j0mPYMoc7ZoAujpUF/juFVII=",
|
||||
"lastModified": 1702918879,
|
||||
"narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "105e27adb70a9890986b6d543a67761cbc1964a2",
|
||||
"rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -60,11 +53,11 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1691853136,
|
||||
"narHash": "sha256-wTzDsRV4HN8A2Sl0SVQY0q8ILs90CD43Ha//7gNZE+E=",
|
||||
"lastModified": 1702539185,
|
||||
"narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "f0451844bbdf545f696f029d1448de4906c7f753",
|
||||
"rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
@ -76,16 +69,16 @@
|
|||
},
|
||||
"nixpkgs-stable": {
|
||||
"locked": {
|
||||
"lastModified": 1685004253,
|
||||
"narHash": "sha256-AbVL1nN/TDicUQ5wXZ8xdLERxz/eJr7+o8lqkIOVuaE=",
|
||||
"lastModified": 1702780907,
|
||||
"narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "3e01645c40b92d29f3ae76344a6d654986a91a91",
|
||||
"rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixos-23.05",
|
||||
"ref": "nixos-23.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
|
|
@ -98,31 +91,6 @@
|
|||
"nixpkgs": "nixpkgs",
|
||||
"nixpkgs-stable": "nixpkgs-stable"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"flake-utils": [
|
||||
"crane",
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"crane",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1675391458,
|
||||
"narHash": "sha256-ukDKZw922BnK5ohL9LhwtaDAdCsJL7L6ScNEyF1lO9w=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "383a4acfd11d778d5c2efcf28376cbd845eeaedf",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
|
|
|
|||
|
|
@ -3,14 +3,12 @@
|
|||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||
nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-23.05";
|
||||
nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-23.11";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
|
||||
crane = {
|
||||
url = "github:ipetkov/crane";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.flake-compat.follows = "flake-compat";
|
||||
inputs.flake-utils.follows = "flake-utils";
|
||||
};
|
||||
|
||||
flake-compat = {
|
||||
|
|
@ -20,7 +18,7 @@
|
|||
};
|
||||
|
||||
outputs = { self, nixpkgs, nixpkgs-stable, flake-utils, crane, ... }: let
|
||||
supportedSystems = flake-utils.lib.defaultSystems;
|
||||
supportedSystems = flake-utils.lib.defaultSystems ++ [ "riscv64-linux" ];
|
||||
|
||||
makeCranePkgs = pkgs: let
|
||||
craneLib = crane.mkLib pkgs;
|
||||
|
|
@ -101,7 +99,6 @@
|
|||
];
|
||||
config = {
|
||||
Entrypoint = [ "${packages.attic-server}/bin/atticd" ];
|
||||
Cmd = [ "--mode" "api-server" ];
|
||||
Env = [
|
||||
"SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"
|
||||
];
|
||||
|
|
|
|||
|
|
@ -29,7 +29,18 @@ let
|
|||
'';
|
||||
|
||||
databaseModules = {
|
||||
sqlite = {};
|
||||
sqlite = {
|
||||
testScriptPost = ''
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
schema = server.succeed("${pkgs.sqlite}/bin/sqlite3 /var/lib/atticd/server.db '.schema --indent'")
|
||||
|
||||
schema_path = Path(os.environ.get("out", os.getcwd())) / "schema.sql"
|
||||
with open(schema_path, 'w') as f:
|
||||
f.write(schema)
|
||||
'';
|
||||
};
|
||||
postgres = {
|
||||
server = {
|
||||
services.postgresql = {
|
||||
|
|
@ -38,9 +49,6 @@ let
|
|||
ensureUsers = [
|
||||
{
|
||||
name = "atticd";
|
||||
ensurePermissions = {
|
||||
"DATABASE attic" = "ALL PRIVILEGES";
|
||||
};
|
||||
}
|
||||
|
||||
# For testing only - Don't actually do this
|
||||
|
|
@ -53,10 +61,24 @@ let
|
|||
];
|
||||
};
|
||||
|
||||
systemd.services.postgresql.postStart = lib.mkAfter ''
|
||||
$PSQL -tAc 'ALTER DATABASE "attic" OWNER TO "atticd"'
|
||||
'';
|
||||
|
||||
services.atticd.settings = {
|
||||
database.url = "postgresql:///attic?host=/run/postgresql";
|
||||
};
|
||||
};
|
||||
testScriptPost = ''
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
schema = server.succeed("pg_dump --schema-only attic")
|
||||
|
||||
schema_path = Path(os.environ.get("out", os.getcwd())) / "schema.sql"
|
||||
with open(schema_path, 'w') as f:
|
||||
f.write(schema)
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -214,9 +236,9 @@ in {
|
|||
|
||||
${lib.optionalString (config.storage == "local") ''
|
||||
with subtest("Check that all chunks are actually deleted after GC"):
|
||||
files = server.succeed("find /var/lib/atticd/storage -type f")
|
||||
files = server.succeed("find /var/lib/atticd/storage -type f ! -name 'VERSION'")
|
||||
print(f"Remaining files: {files}")
|
||||
assert files.strip() == ""
|
||||
assert files.strip() == "", "Some files remain after GC: " + files
|
||||
''}
|
||||
|
||||
with subtest("Check that we can include the upload info in the payload"):
|
||||
|
|
@ -231,6 +253,9 @@ in {
|
|||
client.succeed("attic cache destroy --no-confirm test")
|
||||
client.fail("attic cache info test")
|
||||
client.fail("curl -sL --fail-with-body http://server:8080/test/nix-cache-info")
|
||||
|
||||
${databaseModules.${config.database}.testScriptPost or ""}
|
||||
${storageModules.${config.storage}.testScriptPost or ""}
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,6 +116,24 @@ in
|
|||
defaultText = "generated from `services.atticd.settings`";
|
||||
};
|
||||
|
||||
mode = lib.mkOption {
|
||||
description = ''
|
||||
Mode in which to run the server.
|
||||
|
||||
'monolithic' runs all components, and is suitable for single-node deployments.
|
||||
|
||||
'api-server' runs only the API server, and is suitable for clustering.
|
||||
|
||||
'garbage-collector' only runs the garbage collector periodically.
|
||||
|
||||
A simple NixOS-based Attic deployment will typically have one 'monolithic' and any number of 'api-server' nodes.
|
||||
|
||||
There are several other supported modes that perform one-off operations, but these are the only ones that make sense to run via the NixOS module.
|
||||
'';
|
||||
type = lib.types.enum ["monolithic" "api-server" "garbage-collector"];
|
||||
default = "monolithic";
|
||||
};
|
||||
|
||||
# Internal flags
|
||||
useFlakeCompatOverlay = lib.mkOption {
|
||||
description = ''
|
||||
|
|
@ -168,7 +186,7 @@ in
|
|||
after = [ "network.target" ]
|
||||
++ lib.optionals hasLocalPostgresDB [ "postgresql.service" "nss-lookup.target" ];
|
||||
serviceConfig = {
|
||||
ExecStart = "${cfg.package}/bin/atticd -f ${checkedConfigFile}";
|
||||
ExecStart = "${cfg.package}/bin/atticd -f ${checkedConfigFile} --mode ${cfg.mode}";
|
||||
EnvironmentFile = cfg.credentialsFile;
|
||||
StateDirectory = "atticd"; # for usage with local storage and sqlite
|
||||
DynamicUser = true;
|
||||
|
|
@ -181,10 +199,16 @@ in
|
|||
ProtectKernelTunables = true;
|
||||
ProtectProc = "invisible";
|
||||
ProtectSystem = "strict";
|
||||
Restart = "on-failure";
|
||||
RestartSec = 10;
|
||||
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
ReadWritePaths = let
|
||||
path = cfg.settings.storage.path;
|
||||
isDefaultStateDirectory = path == "/var/lib/atticd" || lib.hasPrefix "/var/lib/atticd/" path;
|
||||
in lib.optionals (cfg.settings.storage.type or "" == "local" && !isDefaultStateDirectory) [ path ];
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -49,6 +49,11 @@ in rustPlatform.buildRustPackage rec {
|
|||
|
||||
ATTIC_DISTRIBUTOR = "attic";
|
||||
|
||||
# Workaround for https://github.com/NixOS/nixpkgs/issues/166205
|
||||
env = lib.optionalAttrs stdenv.cc.isClang {
|
||||
NIX_LDFLAGS = "-l${stdenv.cc.libcxx.cxxabi.libName}";
|
||||
};
|
||||
|
||||
# Recursive Nix is not stable yet
|
||||
doCheck = false;
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ attic-token = { path = "../token" }
|
|||
anyhow = "1.0.71"
|
||||
async-stream = "0.3.5"
|
||||
async-trait = "0.1.68"
|
||||
aws-config = "0.55.3"
|
||||
aws-sdk-s3 = "0.28.0"
|
||||
aws-config = "0.57.1"
|
||||
aws-sdk-s3 = "0.35.0"
|
||||
axum = "0.6.18"
|
||||
axum-macros = "0.3.7"
|
||||
base64 = "0.21.2"
|
||||
|
|
@ -36,7 +36,7 @@ clap = { version = "4.3", features = ["derive"] }
|
|||
derivative = "2.2.0"
|
||||
digest = "0.10.7"
|
||||
displaydoc = "0.2.4"
|
||||
enum-as-inner = "0.5.2"
|
||||
enum-as-inner = "0.6.0"
|
||||
fastcdc = "3.0.3"
|
||||
futures = "0.3.28"
|
||||
hex = "0.4.3"
|
||||
|
|
@ -52,13 +52,13 @@ serde = "1.0.163"
|
|||
serde_json = "1.0.96"
|
||||
serde_with = "3.0.0"
|
||||
tokio-util = { version = "0.7.8", features = [ "io" ] }
|
||||
toml = "0.7.4"
|
||||
toml = "0.8.8"
|
||||
tower-http = { version = "0.4.0", features = [ "catch-panic", "trace" ] }
|
||||
tracing = "0.1.37"
|
||||
tracing-error = "0.2.0"
|
||||
tracing-subscriber = { version = "0.3.17", features = [ "json" ] }
|
||||
uuid = { version = "1.3.3", features = ["v4"] }
|
||||
console-subscriber = "0.1.9"
|
||||
console-subscriber = "0.2.0"
|
||||
xdg = "2.5.0"
|
||||
rsa = "0.9.3"
|
||||
|
||||
|
|
@ -72,7 +72,7 @@ features = [
|
|||
]
|
||||
|
||||
[dependencies.sea-orm]
|
||||
version = "0.11.3"
|
||||
version = "0.12.10"
|
||||
features = [
|
||||
"runtime-tokio-rustls",
|
||||
"macros",
|
||||
|
|
@ -82,7 +82,7 @@ features = [
|
|||
]
|
||||
|
||||
[dependencies.sea-orm-migration]
|
||||
version = "0.11.3"
|
||||
version = "0.12.10"
|
||||
|
||||
[dependencies.tokio]
|
||||
version = "1.28.2"
|
||||
|
|
|
|||
|
|
@ -215,10 +215,6 @@ async fn get_nar(
|
|||
let storage = state.storage().await?;
|
||||
match storage.download_file_db(remote_file, false).await? {
|
||||
Download::Url(url) => Ok(Redirect::temporary(&url).into_response()),
|
||||
Download::Stream(stream) => {
|
||||
let body = StreamBody::new(stream);
|
||||
Ok(body.into_response())
|
||||
}
|
||||
Download::AsyncRead(stream) => {
|
||||
let stream = ReaderStream::new(stream);
|
||||
let body = StreamBody::new(stream);
|
||||
|
|
@ -241,7 +237,6 @@ async fn get_nar(
|
|||
IoErrorKind::Other,
|
||||
"URLs not supported for NAR reassembly",
|
||||
)),
|
||||
Download::Stream(stream) => Ok(stream),
|
||||
Download::AsyncRead(stream) => {
|
||||
let stream: BoxStream<_> = Box::pin(ReaderStream::new(stream));
|
||||
Ok(stream)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
//! Local file storage.
|
||||
|
||||
use std::ffi::OsStr;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
|
@ -30,17 +33,95 @@ pub struct LocalRemoteFile {
|
|||
pub name: String,
|
||||
}
|
||||
|
||||
impl LocalBackend {
|
||||
pub async fn new(config: LocalStorageConfig) -> ServerResult<Self> {
|
||||
fs::create_dir_all(&config.path)
|
||||
async fn read_version(storage_path: &Path) -> ServerResult<u32> {
|
||||
let version_path = storage_path.join("VERSION");
|
||||
let v = match fs::read_to_string(&version_path).await {
|
||||
Ok(version) => version
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| ErrorKind::StorageError(anyhow::anyhow!("Invalid version file")))?,
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => 0,
|
||||
Err(e) => {
|
||||
return Err(ErrorKind::StorageError(anyhow::anyhow!(
|
||||
"Failed to read version file: {}",
|
||||
e
|
||||
))
|
||||
.into());
|
||||
}
|
||||
};
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
async fn write_version(storage_path: &Path, version: u32) -> ServerResult<()> {
|
||||
let version_path = storage_path.join("VERSION");
|
||||
fs::write(&version_path, format!("{}", version))
|
||||
.await
|
||||
.map_err(ServerError::storage_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upgrade_0_to_1(storage_path: &Path) -> ServerResult<()> {
|
||||
let mut files = fs::read_dir(storage_path)
|
||||
.await
|
||||
.map_err(ServerError::storage_error)?;
|
||||
// move all files to subdirectory using the first two characters of the filename
|
||||
while let Some(file) = files
|
||||
.next_entry()
|
||||
.await
|
||||
.map_err(ServerError::storage_error)?
|
||||
{
|
||||
if file
|
||||
.file_type()
|
||||
.await
|
||||
.map_err(ServerError::storage_error)?
|
||||
.is_file()
|
||||
{
|
||||
let name = file.file_name();
|
||||
let name_bytes = name.as_os_str().as_bytes();
|
||||
let parents = storage_path
|
||||
.join(OsStr::from_bytes(&name_bytes[0..1]))
|
||||
.join(OsStr::from_bytes(&name_bytes[0..2]));
|
||||
let new_path = parents.join(name);
|
||||
fs::create_dir_all(&parents).await.map_err(|e| {
|
||||
ErrorKind::StorageError(anyhow::anyhow!("Failed to create directory {}", e))
|
||||
})?;
|
||||
fs::rename(&file.path(), &new_path).await.map_err(|e| {
|
||||
ErrorKind::StorageError(anyhow::anyhow!(
|
||||
"Failed to move file {} to {}: {}",
|
||||
file.path().display(),
|
||||
new_path.display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl LocalBackend {
|
||||
pub async fn new(config: LocalStorageConfig) -> ServerResult<Self> {
|
||||
fs::create_dir_all(&config.path).await.map_err(|e| {
|
||||
ErrorKind::StorageError(anyhow::anyhow!(
|
||||
"Failed to create storage directory {}: {}",
|
||||
config.path.display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
let version = read_version(&config.path).await?;
|
||||
if version == 0 {
|
||||
upgrade_0_to_1(&config.path).await?;
|
||||
}
|
||||
write_version(&config.path, 1).await?;
|
||||
|
||||
Ok(Self { config })
|
||||
}
|
||||
|
||||
fn get_path(&self, p: &str) -> PathBuf {
|
||||
self.config.path.join(p)
|
||||
let level1 = &p[0..1];
|
||||
let level2 = &p[0..2];
|
||||
self.config.path.join(level1).join(level2).join(p)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -51,9 +132,23 @@ impl StorageBackend for LocalBackend {
|
|||
name: String,
|
||||
mut stream: &mut (dyn AsyncRead + Unpin + Send),
|
||||
) -> ServerResult<RemoteFile> {
|
||||
let mut file = File::create(self.get_path(&name))
|
||||
let path = self.get_path(&name);
|
||||
fs::create_dir_all(path.parent().unwrap())
|
||||
.await
|
||||
.map_err(ServerError::storage_error)?;
|
||||
.map_err(|e| {
|
||||
ErrorKind::StorageError(anyhow::anyhow!(
|
||||
"Failed to create directory {}: {}",
|
||||
path.parent().unwrap().display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
let mut file = File::create(self.get_path(&name)).await.map_err(|e| {
|
||||
ErrorKind::StorageError(anyhow::anyhow!(
|
||||
"Failed to create file {}: {}",
|
||||
self.get_path(&name).display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
io::copy(&mut stream, &mut file)
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@
|
|||
mod local;
|
||||
mod s3;
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::stream::BoxStream;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
|
|
@ -38,9 +36,6 @@ pub enum Download {
|
|||
/// A possibly ephemeral URL.
|
||||
Url(String),
|
||||
|
||||
/// A stream.
|
||||
Stream(BoxStream<'static, std::io::Result<Bytes>>),
|
||||
|
||||
/// An AsyncRead.
|
||||
AsyncRead(Box<dyn AsyncRead + Unpin + Send>),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
//! S3 remote files.
|
||||
|
||||
use std::io::{Error as IoError, ErrorKind as IoErrorKind};
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
|
@ -14,7 +13,6 @@ use aws_sdk_s3::{
|
|||
};
|
||||
use bytes::BytesMut;
|
||||
use futures::future::join_all;
|
||||
use futures::stream::StreamExt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::AsyncRead;
|
||||
|
||||
|
|
@ -127,7 +125,7 @@ impl S3Backend {
|
|||
};
|
||||
|
||||
// FIXME: Ugly
|
||||
let client = if self.client.conf().region().unwrap().as_ref() == file.region {
|
||||
let client = if self.client.config().region().unwrap().as_ref() == file.region {
|
||||
self.client.clone()
|
||||
} else {
|
||||
// FIXME: Cache the client instance
|
||||
|
|
@ -149,11 +147,7 @@ impl S3Backend {
|
|||
if prefer_stream {
|
||||
let output = req.send().await.map_err(ServerError::storage_error)?;
|
||||
|
||||
let stream = StreamExt::map(output.body, |item| {
|
||||
item.map_err(|e| IoError::new(IoErrorKind::Other, e))
|
||||
});
|
||||
|
||||
Ok(Download::Stream(Box::pin(stream)))
|
||||
Ok(Download::AsyncRead(Box::new(output.body.into_async_read())))
|
||||
} else {
|
||||
// FIXME: Configurable expiration
|
||||
let presign_config = PresigningConfig::expires_in(Duration::from_secs(600))
|
||||
|
|
|
|||
Loading…
Reference in a new issue