mirrors/attic
1
0
Fork 0
mirror of https://github.com/zhaofengli/attic.git synced 2024-12-14 11:57:30 +00:00
Code Activity

server/s3: Fix loading of AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY

Browse source
This commit is contained in:
Zhaofeng Li 2023-01-04 21:05:07 -07:00
parent 3fd587315f
commit 14d8b8f1b0
4 changed files with 111 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
Cargo.lock generated
View file

@ -198,6 +198,7 @@ dependencies = [
"async-compression",
"async-trait",
"attic",
"aws-config",
"aws-sdk-s3",
"axum",
"axum-macros",
@ -254,6 +255,34 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "aws-config"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7688e1dfbb9f7804fab0a830820d7e827b8d973906763cf1a855ce4719292f5"
dependencies = [
"aws-http",
"aws-sdk-sso",
"aws-sdk-sts",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-json",
"aws-smithy-types",
"aws-types",
"bytes",
"hex",
"http",
"hyper",
"ring",
"time 0.3.17",
"tokio",
"tower",
"tracing",
"zeroize",
]
[[package]]
name = "aws-endpoint"
version = "0.52.0"
@ -315,6 +344,51 @@ dependencies = [
"tracing",
]
[[package]]
name = "aws-sdk-sso"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf03342c2b3f52b180f484e60586500765474f2bfc7dcd4ffe893a7a1929db1d"
dependencies = [
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-json",
"aws-smithy-types",
"aws-types",
"bytes",
"http",
"tokio-stream",
"tower",
]
[[package]]
name = "aws-sdk-sts"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa1de4e07ea87a30a317c7b563b3a40fd18a843ad794216dda81672b6e174bce"
dependencies = [
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-query",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes",
"http",
"tower",
"tracing",
]
[[package]]
name = "aws-sig-auth"
version = "0.52.0"
@ -456,6 +530,25 @@ dependencies = [
"tracing",
]
[[package]]
name = "aws-smithy-json"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e3ddd9275b167bc59e9446469eca56177ec0b51225632f90aaa2cd5f41c940e"
dependencies = [
"aws-smithy-types",
]
[[package]]
name = "aws-smithy-query"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b19d2e0b3ce20e460bad0d0d974238673100edebba6978c2c1aadd925602f7"
dependencies = [
"aws-smithy-types",
"urlencoding",
]
[[package]]
name = "aws-smithy-types"
version = "0.52.0"
@ -3635,6 +3728,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "urlencoding"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9"
[[package]]
name = "uuid"
version = "1.2.2"

1
server/Cargo.toml
View file

@ -23,6 +23,7 @@ attic = { path = "../attic", default-features = false }
anyhow = "1.0.68"
async-trait = "0.1.60"
aws-config = "0.52.0"
aws-sdk-s3 = "0.22.0"
axum = "0.6.1"
axum-macros = "0.3.0"

2
server/src/lib.rs
View file

@ -111,7 +111,7 @@ impl StateInner {
Ok(Arc::new(boxed))
}
StorageConfig::S3(s3_config) => {
let s3 = S3Backend::new(s3_config.clone())?;
let s3 = S3Backend::new(s3_config.clone()).await?;
let boxed: Box<dyn StorageBackend> = Box::new(s3);
Ok(Arc::new(boxed))
}

19
server/src/storage/s3.rs
View file

@ -5,7 +5,7 @@ use std::time::Duration;
use async_trait::async_trait;
use aws_sdk_s3::{
config::Builder as S3ConfigBuilder, model::CompletedMultipartUpload, model::CompletedPart,
presigning::config::PresigningConfig, Client, Config as S3Config, Credentials, Endpoint,
presigning::config::PresigningConfig, Client, Credentials, Endpoint,
Region,
};
use futures::future::join_all;
@ -73,8 +73,8 @@ pub struct S3RemoteFile {
}
impl S3Backend {
pub fn new(config: S3StorageConfig) -> ServerResult<Self> {
let s3_config = Self::config_builder(&config)?
pub async fn new(config: S3StorageConfig) -> ServerResult<Self> {
let s3_config = Self::config_builder(&config).await?
.region(Region::new(config.region.to_owned()))
.build();
@ -84,8 +84,9 @@ impl S3Backend {
})
}
fn config_builder(config: &S3StorageConfig) -> ServerResult<S3ConfigBuilder> {
let mut builder = S3Config::builder();
async fn config_builder(config: &S3StorageConfig) -> ServerResult<S3ConfigBuilder> {
let shared_config = aws_config::load_from_env().await;
let mut builder = S3ConfigBuilder::from(&shared_config);
if let Some(credentials) = &config.credentials {
builder = builder.credentials_provider(Credentials::new(
@ -105,7 +106,7 @@ impl S3Backend {
Ok(builder)
}
fn get_client_from_db_ref<'a>(
async fn get_client_from_db_ref<'a>(
&self,
file: &'a RemoteFile,
) -> ServerResult<(Client, &'a S3RemoteFile)> {
@ -122,7 +123,7 @@ impl S3Backend {
self.client.clone()
} else {
// FIXME: Cache the client instance
let s3_conf = Self::config_builder(&self.config)?
let s3_conf = Self::config_builder(&self.config).await?
.region(Region::new(file.region.to_owned()))
.build();
Client::from_conf(s3_conf)
@ -290,7 +291,7 @@ impl StorageBackend for S3Backend {
}
async fn delete_file_db(&self, file: &RemoteFile) -> ServerResult<()> {
let (client, file) = self.get_client_from_db_ref(file)?;
let (client, file) = self.get_client_from_db_ref(file).await?;
let deletion = client
.delete_object()
@ -323,7 +324,7 @@ impl StorageBackend for S3Backend {
}
async fn download_file_db(&self, file: &RemoteFile) -> ServerResult<Download> {
let (client, file) = self.get_client_from_db_ref(file)?;
let (client, file) = self.get_client_from_db_ref(file).await?;
let presign_config = PresigningConfig::expires_in(Duration::from_secs(600))
.map_err(ServerError::remote_file_error)?;