Fixed clippy issues and cleanup of unnecessary imports (#182)

This commit is contained in:
Henk-Jan Lebbink 2025-08-26 11:04:51 +02:00 committed by GitHub
parent 5080bf9b85
commit 25d424b97f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
58 changed files with 265 additions and 197 deletions

View File

@ -10,11 +10,6 @@ readme = "README.md"
keywords = ["object-storage", "minio", "s3"] keywords = ["object-storage", "minio", "s3"]
categories = ["api-bindings", "web-programming::http-client"] categories = ["api-bindings", "web-programming::http-client"]
[dependencies.reqwest]
version = "0.12.22"
default-features = false
features = ["stream"]
[features] [features]
default = ["default-tls", "default-crypto"] default = ["default-tls", "default-crypto"]
default-tls = ["reqwest/default-tls"] default-tls = ["reqwest/default-tls"]
@ -23,21 +18,29 @@ rustls-tls = ["reqwest/rustls-tls"]
default-crypto = ["dep:sha2", "dep:hmac"] default-crypto = ["dep:sha2", "dep:hmac"]
ring = ["dep:ring"] ring = ["dep:ring"]
[workspace.dependencies]
uuid = "1.18"
futures-util = "0.3"
reqwest = { version = "0.12", default-features = false }
bytes = "1.10"
async-std = "1.13"
[dependencies] [dependencies]
uuid = { workspace = true, features = ["v4"] }
futures-util = { workspace = true }
bytes = { workspace = true }
async-std = { workspace = true, features = ["attributes"] }
reqwest = { workspace = true, features = ["stream"] }
async-recursion = "1.1.1" async-recursion = "1.1.1"
async-std = { version = "1.13.1", features = ["attributes"] }
async-stream = "0.3.6" async-stream = "0.3.6"
async-trait = "0.1.88" async-trait = "0.1.88"
base64 = "0.22.1" base64 = "0.22.1"
byteorder = "1.5.0"
bytes = "1.10.1"
chrono = "0.4.41" chrono = "0.4.41"
crc = "3.3.0" crc = "3.3.0"
dashmap = "6.1.0" dashmap = "6.1.0"
derivative = "2.2.0"
env_logger = "0.11.8" env_logger = "0.11.8"
futures-util = "0.3.31"
hex = "0.4.3"
hmac = { version = "0.12.1", optional = true } hmac = { version = "0.12.1", optional = true }
hyper = { version = "1.6.0", features = ["full"] } hyper = { version = "1.6.0", features = ["full"] }
lazy_static = "1.5.0" lazy_static = "1.5.0"
@ -46,26 +49,25 @@ md5 = "0.8.0"
multimap = "0.10.1" multimap = "0.10.1"
percent-encoding = "2.3.1" percent-encoding = "2.3.1"
url = "2.5.4" url = "2.5.4"
rand = { version = "0.8.5", features = ["small_rng"] }
regex = "1.11.1" regex = "1.11.1"
ring = { version = "0.17.14", optional = true, default-features = false, features = ["alloc"] } ring = { version = "0.17.14", optional = true, default-features = false, features = ["alloc"] }
serde = { version = "1.0.219", features = ["derive"] } serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140" serde_json = "1.0.142"
sha2 = { version = "0.10.9", optional = true } sha2 = { version = "0.10.9", optional = true }
urlencoding = "2.1.3" urlencoding = "2.1.3"
xmltree = "0.11.0" xmltree = "0.11.0"
futures = "0.3.31"
http = "1.3.1" http = "1.3.1"
thiserror = "2.0.12" thiserror = "2.0.14"
[dev-dependencies] [dev-dependencies]
minio-common = { path = "./common" }
minio-macros = { path = "./macros" }
tokio = { version = "1.47.1", features = ["full"] } tokio = { version = "1.47.1", features = ["full"] }
minio_common = { path = "./common" }
async-std = { version = "1.13.1", features = ["attributes", "tokio1"] } async-std = { version = "1.13.1", features = ["attributes", "tokio1"] }
clap = { version = "4.5.44", features = ["derive"] } clap = { version = "4.5.44", features = ["derive"] }
rand = { version = "0.9.2", features = ["small_rng"] }
quickcheck = "1.0.3" quickcheck = "1.0.3"
criterion = "0.7.0" criterion = "0.7.0"
minio-macros = { path = "./macros" }
[lib] [lib]
name = "minio" name = "minio"

View File

@ -1,20 +1,23 @@
[package] [package]
name = "minio_common" name = "minio-common"
version = "0.1.0" version = "0.1.0"
edition = "2024" edition = "2024"
[dependencies] [dependencies]
minio = {path = ".." } minio = {path = ".." }
uuid = { workspace = true, features = ["v4"] }
reqwest = { workspace = true }
bytes = { workspace = true }
async-std = { workspace = true }
futures-io = "0.3.31"
tokio = { version = "1.47.1", features = ["full"] } tokio = { version = "1.47.1", features = ["full"] }
async-std = "1.13.1" rand = { version = "0.9.2", features = ["small_rng"] }
rand = { version = "0.8.5", features = ["small_rng"] }
bytes = "1.10.1"
log = "0.4.27" log = "0.4.27"
chrono = "0.4.41" chrono = "0.4.41"
reqwest = "0.12.22"
http = "1.3.1" http = "1.3.1"
futures = "0.3.31"
uuid = { version = "1.18.0", features = ["v4"] }
[lib] [lib]
name = "minio_common" name = "minio_common"

View File

@ -38,7 +38,7 @@ impl CleanupGuard {
pub async fn cleanup(client: Client, bucket_name: &str) { pub async fn cleanup(client: Client, bucket_name: &str) {
tokio::select!( tokio::select!(
_ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => {
eprintln!("Cleanup timeout after 60s while removing bucket {}", bucket_name); eprintln!("Cleanup timeout after 60s while removing bucket {bucket_name}");
}, },
outcome = client.delete_and_purge_bucket(bucket_name) => { outcome = client.delete_and_purge_bucket(bucket_name) => {
match outcome { match outcome {
@ -46,7 +46,7 @@ pub async fn cleanup(client: Client, bucket_name: &str) {
//eprintln!("Bucket {} removed successfully", bucket_name); //eprintln!("Bucket {} removed successfully", bucket_name);
} }
Err(e) => { Err(e) => {
eprintln!("Error removing bucket '{}':\n{}", bucket_name, e); eprintln!("Error removing bucket '{bucket_name}':\n{e}");
} }
} }
} }

View File

@ -140,7 +140,7 @@ pub fn create_bucket_replication_config_example(dst_bucket: &str) -> Replication
rules: vec![ReplicationRule { rules: vec![ReplicationRule {
id: Some(String::from("rule1")), id: Some(String::from("rule1")),
destination: Destination { destination: Destination {
bucket_arn: String::from(&format!("arn:aws:s3:::{}", dst_bucket)), bucket_arn: String::from(&format!("arn:aws:s3:::{dst_bucket}")),
..Default::default() ..Default::default()
}, },
filter: Some(Filter { filter: Some(Filter {
@ -170,8 +170,8 @@ pub fn create_object_lock_config_example() -> ObjectLockConfig {
pub fn create_post_policy_example(bucket_name: &str, object_name: &str) -> PostPolicy { pub fn create_post_policy_example(bucket_name: &str, object_name: &str) -> PostPolicy {
let expiration: DateTime<Utc> = utc_now() + chrono::Duration::days(5); let expiration: DateTime<Utc> = utc_now() + chrono::Duration::days(5);
let mut policy = PostPolicy::new(&bucket_name, expiration).unwrap(); let mut policy = PostPolicy::new(bucket_name, expiration).unwrap();
policy.add_equals_condition("key", &object_name).unwrap(); policy.add_equals_condition("key", object_name).unwrap();
policy policy
.add_content_length_range_condition(1024 * 1024, 4 * 1024 * 1024) .add_content_length_range_condition(1024 * 1024, 4 * 1024 * 1024)
.unwrap(); .unwrap();
@ -189,7 +189,7 @@ pub fn create_select_content_data() -> (String, String) {
(body, data) (body, data)
} }
pub fn create_select_content_request() -> SelectRequest { pub fn create_select_content_request() -> SelectRequest {
let request = SelectRequest::new_csv_input_output( SelectRequest::new_csv_input_output(
"select * from S3Object", "select * from S3Object",
CsvInputSerialization { CsvInputSerialization {
compression_type: None, compression_type: None,
@ -209,6 +209,5 @@ pub fn create_select_content_request() -> SelectRequest {
record_delimiter: None, record_delimiter: None,
}, },
) )
.unwrap(); .unwrap()
request
} }

View File

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use futures::AsyncRead; use futures_io::AsyncRead;
use std::io; use std::io;
use std::pin::Pin; use std::pin::Pin;
use std::task::{Context, Poll}; use std::task::{Context, Poll};
@ -34,7 +34,7 @@ impl io::Read for RandReader {
let bytes_read = buf.len().min(self.size as usize); let bytes_read = buf.len().min(self.size as usize);
if bytes_read > 0 { if bytes_read > 0 {
let random: &mut dyn rand::RngCore = &mut rand::thread_rng(); let random: &mut dyn rand::RngCore = &mut rand::rng();
random.fill_bytes(&mut buf[0..bytes_read]); random.fill_bytes(&mut buf[0..bytes_read]);
} }
@ -53,7 +53,7 @@ impl AsyncRead for RandReader {
let bytes_read = buf.len().min(self.size as usize); let bytes_read = buf.len().min(self.size as usize);
if bytes_read > 0 { if bytes_read > 0 {
let random: &mut dyn rand::RngCore = &mut rand::thread_rng(); let random: &mut dyn rand::RngCore = &mut rand::rng();
random.fill_bytes(&mut buf[0..bytes_read]); random.fill_bytes(&mut buf[0..bytes_read]);
} }

View File

@ -15,7 +15,7 @@
use async_std::stream::Stream; use async_std::stream::Stream;
use bytes::Bytes; use bytes::Bytes;
use futures::io::AsyncRead; use futures_io::AsyncRead;
use rand::prelude::SmallRng; use rand::prelude::SmallRng;
use rand::{RngCore, SeedableRng}; use rand::{RngCore, SeedableRng};
use std::io; use std::io;
@ -30,7 +30,7 @@ pub struct RandSrc {
impl RandSrc { impl RandSrc {
#[allow(dead_code)] #[allow(dead_code)]
pub fn new(size: u64) -> RandSrc { pub fn new(size: u64) -> RandSrc {
let rng = SmallRng::from_entropy(); let rng: SmallRng = SmallRng::from_os_rng();
RandSrc { size, rng } RandSrc { size, rng }
} }
} }

View File

@ -84,10 +84,10 @@ impl TestContext {
let host: String = let host: String =
std::env::var("SERVER_ENDPOINT").unwrap_or(DEFAULT_SERVER_ENDPOINT.to_string()); std::env::var("SERVER_ENDPOINT").unwrap_or(DEFAULT_SERVER_ENDPOINT.to_string());
log::debug!("SERVER_ENDPOINT={}", host); log::debug!("SERVER_ENDPOINT={host}");
let access_key: String = let access_key: String =
std::env::var("ACCESS_KEY").unwrap_or(DEFAULT_ACCESS_KEY.to_string()); std::env::var("ACCESS_KEY").unwrap_or(DEFAULT_ACCESS_KEY.to_string());
log::debug!("ACCESS_KEY={}", access_key); log::debug!("ACCESS_KEY={access_key}");
let secret_key: String = let secret_key: String =
std::env::var("SECRET_KEY").unwrap_or(DEFAULT_SECRET_KEY.to_string()); std::env::var("SECRET_KEY").unwrap_or(DEFAULT_SECRET_KEY.to_string());
log::debug!("SECRET_KEY=*****"); log::debug!("SECRET_KEY=*****");
@ -95,19 +95,19 @@ impl TestContext {
.unwrap_or(DEFAULT_ENABLE_HTTPS.to_string()) .unwrap_or(DEFAULT_ENABLE_HTTPS.to_string())
.parse() .parse()
.unwrap_or(false); .unwrap_or(false);
log::debug!("ENABLE_HTTPS={}", secure); log::debug!("ENABLE_HTTPS={secure}");
let ssl_cert: String = let ssl_cert: String =
std::env::var("MINIO_SSL_CERT_FILE").unwrap_or(DEFAULT_SSL_CERT_FILE.to_string()); std::env::var("MINIO_SSL_CERT_FILE").unwrap_or(DEFAULT_SSL_CERT_FILE.to_string());
log::debug!("MINIO_SSL_CERT_FILE={}", ssl_cert); log::debug!("MINIO_SSL_CERT_FILE={ssl_cert}");
let ssl_cert_file: PathBuf = ssl_cert.into(); let ssl_cert_file: PathBuf = ssl_cert.into();
let ignore_cert_check: bool = std::env::var("IGNORE_CERT_CHECK") let ignore_cert_check: bool = std::env::var("IGNORE_CERT_CHECK")
.unwrap_or(DEFAULT_IGNORE_CERT_CHECK.to_string()) .unwrap_or(DEFAULT_IGNORE_CERT_CHECK.to_string())
.parse() .parse()
.unwrap_or(true); .unwrap_or(true);
log::debug!("IGNORE_CERT_CHECK={}", ignore_cert_check); log::debug!("IGNORE_CERT_CHECK={ignore_cert_check}");
let region: String = let region: String =
std::env::var("SERVER_REGION").unwrap_or(DEFAULT_SERVER_REGION.to_string()); std::env::var("SERVER_REGION").unwrap_or(DEFAULT_SERVER_REGION.to_string());
log::debug!("SERVER_REGION={:?}", region); log::debug!("SERVER_REGION={region:?}");
let mut base_url: BaseUrl = host.parse().unwrap(); let mut base_url: BaseUrl = host.parse().unwrap();
base_url.https = secure; base_url.https = secure;

View File

@ -15,8 +15,9 @@
use http::{Response as HttpResponse, StatusCode}; use http::{Response as HttpResponse, StatusCode};
use minio::s3::error::Error; use minio::s3::error::Error;
use rand::distributions::Standard;
use rand::{Rng, thread_rng}; use rand::Rng;
use rand::distr::StandardUniform;
use uuid::Uuid; use uuid::Uuid;
pub fn rand_bucket_name() -> String { pub fn rand_bucket_name() -> String {
@ -28,9 +29,9 @@ pub fn rand_object_name() -> String {
} }
pub fn rand_object_name_utf8(len: usize) -> String { pub fn rand_object_name_utf8(len: usize) -> String {
let rng = thread_rng(); let rng = rand::rng();
rng.sample_iter::<char, _>(Standard) rng.sample_iter(StandardUniform)
.filter(|c| !c.is_control()) .filter(|c: &char| !c.is_control())
.take(len) .take(len)
.collect() .collect()
} }
@ -39,9 +40,9 @@ pub async fn get_bytes_from_response(v: Result<reqwest::Response, Error>) -> byt
match v { match v {
Ok(r) => match r.bytes().await { Ok(r) => match r.bytes().await {
Ok(b) => b, Ok(b) => b,
Err(e) => panic!("{:?}", e), Err(e) => panic!("{e:?}"),
}, },
Err(e) => panic!("{:?}", e), Err(e) => panic!("{e:?}"),
} }
} }
@ -52,5 +53,5 @@ pub fn get_response_from_bytes(bytes: bytes::Bytes) -> reqwest::Response {
.body(bytes) .body(bytes)
.expect("Failed to build HTTP response"); .expect("Failed to build HTTP response");
reqwest::Response::try_from(http_response).expect("Failed to convert to reqwest::Response") reqwest::Response::from(http_response)
} }

View File

@ -22,7 +22,7 @@ use minio::s3::response::{AppendObjectResponse, StatObjectResponse};
use minio::s3::segmented_bytes::SegmentedBytes; use minio::s3::segmented_bytes::SegmentedBytes;
use minio::s3::types::S3Api; use minio::s3::types::S3Api;
use rand::Rng; use rand::Rng;
use rand::distributions::Alphanumeric; use rand::distr::Alphanumeric;
#[tokio::main] #[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> { async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@ -78,7 +78,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
} }
fn random_string(len: usize) -> String { fn random_string(len: usize) -> String {
rand::thread_rng() rand::rng()
.sample_iter(&Alphanumeric) .sample_iter(&Alphanumeric)
.take(len) .take(len)
.map(char::from) .map(char::from)

View File

@ -3,18 +3,19 @@ name = "minio-macros"
version = "0.1.0" version = "0.1.0"
edition = "2024" edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
proc-macro = true
[dependencies] [dependencies]
uuid = { workspace = true, features = ["v4"] }
futures-util = { workspace = true }
syn = "2.0.104" syn = "2.0.104"
proc-macro2 = "1.0.95" proc-macro2 = "1.0.97"
quote = "1.0.40" quote = "1.0.40"
darling = "0.21.0" darling = "0.21.0"
darling_core = "0.21.0" darling_core = "0.21.0"
uuid = { version = "1.17.0", features = ["v4"] }
[dev-dependencies] [dev-dependencies]
minio_common = { path = "../common" } minio-common = { path = "../common" }
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
proc-macro = true

View File

@ -104,6 +104,6 @@ pub fn test(
// Expand the macro // Expand the macro
match test_attr::expand_test_macro(args, input_fn) { match test_attr::expand_test_macro(args, input_fn) {
Ok(expanded) => expanded.into(), Ok(expanded) => expanded.into(),
Err(err) => err.into(), Err(err) => err,
} }
} }

View File

@ -55,7 +55,7 @@ impl MacroArgs {
} }
// Validate that the function has exactly two arguments: ctx and bucket_name // Validate that the function has exactly two arguments: ctx and bucket_name
if func.sig.inputs.len() != 2 && !self.no_bucket.is_present() { if (func.sig.inputs.len() != 2) && !self.no_bucket.is_present() {
let error_msg = "Minio test function must have exactly two arguments: (ctx: TestContext, bucket_name: String)"; let error_msg = "Minio test function must have exactly two arguments: (ctx: TestContext, bucket_name: String)";
return Err(proc_macro::TokenStream::from( return Err(proc_macro::TokenStream::from(
Error::custom(error_msg) Error::custom(error_msg)
@ -80,18 +80,18 @@ impl MacroArgs {
} }
} }
// Check second argument (bucket_name: String) // Check the second argument (bucket_name: String)
if !self.no_bucket.is_present() { if !self.no_bucket.is_present()
if let Some(FnArg::Typed(pat_type)) = iter.next() { && let Some(FnArg::Typed(pat_type)) = iter.next()
let type_str = pat_type.ty.to_token_stream().to_string(); {
if !type_str.contains("String") { let type_str = pat_type.ty.to_token_stream().to_string();
let error_msg = "Second argument must be of type String"; if !type_str.contains("String") {
return Err(proc_macro::TokenStream::from( let error_msg = "Second argument must be of type String";
Error::custom(error_msg) return Err(proc_macro::TokenStream::from(
.with_span(&pat_type.span()) Error::custom(error_msg)
.write_errors(), .with_span(&pat_type.span())
)); .write_errors(),
} ));
} }
} }
@ -124,7 +124,7 @@ pub(crate) fn expand_test_macro(
// Setup common prelude // Setup common prelude
let prelude = quote!( let prelude = quote!(
use ::futures::FutureExt; use ::futures_util::FutureExt;
use ::std::panic::AssertUnwindSafe; use ::std::panic::AssertUnwindSafe;
use ::minio::s3::types::S3Api; use ::minio::s3::types::S3Api;
use ::minio::s3::response::a_response_traits::HasBucket; use ::minio::s3::response::a_response_traits::HasBucket;

View File

@ -20,7 +20,7 @@ use crate::s3::builders::{
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::error::{Error, IoError}; use crate::s3::error::{Error, IoError};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::HasObjectSize; use crate::s3::response::a_response_traits::HasObjectSize;
use crate::s3::response::{AppendObjectResponse, StatObjectResponse}; use crate::s3::response::{AppendObjectResponse, StatObjectResponse};
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;

View File

@ -14,7 +14,7 @@
// limitations under the License. // limitations under the License.
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use std::marker::PhantomData; use std::marker::PhantomData;
/// Common parameters for bucket operations /// Common parameters for bucket operations

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::client::{MAX_MULTIPART_COUNT, MAX_PART_SIZE}; use crate::s3::client::{MAX_MULTIPART_COUNT, MAX_PART_SIZE};
use crate::s3::error::{Error, ValidationErr}; use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::HasEtagFromBody; use crate::s3::response::a_response_traits::HasEtagFromBody;
use crate::s3::response::{ use crate::s3::response::{
AbortMultipartUploadResponse, CompleteMultipartUploadResponse, ComposeObjectResponse, AbortMultipartUploadResponse, CompleteMultipartUploadResponse, ComposeObjectResponse,

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::client::DEFAULT_REGION; use crate::s3::client::DEFAULT_REGION;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::CreateBucketResponse; use crate::s3::response::CreateBucketResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::DeleteObjectTaggingResponse; use crate::s3::response::DeleteObjectTaggingResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert}; use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::client::MAX_MULTIPART_COUNT; use crate::s3::client::MAX_MULTIPART_COUNT;
use crate::s3::error::{Error, ValidationErr}; use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::{DeleteError, DeleteObjectResponse, DeleteObjectsResponse}; use crate::s3::response::{DeleteError, DeleteObjectResponse, DeleteObjectsResponse};
use crate::s3::types::{ListEntry, S3Api, S3Request, ToS3Request, ToStream}; use crate::s3::types::{ListEntry, S3Api, S3Request, ToS3Request, ToStream};
use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash}; use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetBucketLifecycleResponse; use crate::s3::response::GetBucketLifecycleResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert}; use crate::s3::utils::{check_bucket_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::GetBucketTaggingResponse; use crate::s3::response::GetBucketTaggingResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert}; use crate::s3::utils::{check_bucket_name, insert};

View File

@ -16,7 +16,7 @@
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectResponse; use crate::s3::response::GetObjectResponse;
use crate::s3::sse::{Sse, SseCustomerKey}; use crate::s3::sse::{Sse, SseCustomerKey};
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectLegalHoldResponse; use crate::s3::response::GetObjectLegalHoldResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert}; use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectPromptResponse; use crate::s3::response::GetObjectPromptResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::sse::SseCustomerKey; use crate::s3::sse::SseCustomerKey;

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectRetentionResponse; use crate::s3::response::GetObjectRetentionResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert}; use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectTaggingResponse; use crate::s3::response::GetObjectTaggingResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert}; use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::creds::Credentials; use crate::s3::creds::Credentials;
use crate::s3::error::Error; use crate::s3::error::Error;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetPresignedObjectUrlResponse; use crate::s3::response::GetPresignedObjectUrlResponse;
use crate::s3::signer::presign_v4; use crate::s3::signer::presign_v4;
use crate::s3::utils::{UtcTime, check_bucket_name, check_object_name, utc_now}; use crate::s3::utils::{UtcTime, check_bucket_name, check_object_name, utc_now};

View File

@ -19,7 +19,7 @@ use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::signer::post_presign_v4; use crate::s3::signer::post_presign_v4;
use crate::s3::utils::{ use crate::s3::utils::{
UtcTime, b64encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now, UtcTime, b64_encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now,
}; };
use serde_json::{Value, json}; use serde_json::{Value, json};
use std::collections::HashMap; use std::collections::HashMap;
@ -341,7 +341,7 @@ impl PostPolicy {
"conditions": conditions, "conditions": conditions,
}); });
let encoded_policy = b64encode(policy.to_string()); let encoded_policy = b64_encode(policy.to_string());
let signature = post_presign_v4(&encoded_policy, &secret_key, date, &region); let signature = post_presign_v4(&encoded_policy, &secret_key, date, &region);
let mut data: HashMap<String, String> = HashMap::new(); let mut data: HashMap<String, String> = HashMap::new();

View File

@ -16,7 +16,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::client::DEFAULT_REGION; use crate::s3::client::DEFAULT_REGION;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::GetRegionResponse; use crate::s3::response::GetRegionResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert}; use crate::s3::utils::{check_bucket_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::ListBucketsResponse; use crate::s3::response::ListBucketsResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use http::Method; use http::Method;

View File

@ -14,7 +14,7 @@
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::error::{Error, ValidationErr}; use crate::s3::error::{Error, ValidationErr};
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::ListObjectsResponse; use crate::s3::response::ListObjectsResponse;
use crate::s3::response::list_objects::{ use crate::s3::response::list_objects::{
ListObjectVersionsResponse, ListObjectsV1Response, ListObjectsV2Response, ListObjectVersionsResponse, ListObjectsV1Response, ListObjectsV2Response,

View File

@ -15,7 +15,7 @@
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::error::{Error, ValidationErr}; use crate::s3::error::{Error, ValidationErr};
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::ListenBucketNotificationResponse; use crate::s3::response::ListenBucketNotificationResponse;
use crate::s3::types::{NotificationRecords, S3Api, S3Request, ToS3Request}; use crate::s3::types::{NotificationRecords, S3Api, S3Request, ToS3Request};
use crate::s3::utils::check_bucket_name; use crate::s3::utils::check_bucket_name;

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketEncryptionResponse; use crate::s3::response::PutBucketEncryptionResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, SseConfig, ToS3Request}; use crate::s3::types::{S3Api, S3Request, SseConfig, ToS3Request};

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::lifecycle_config::LifecycleConfig; use crate::s3::lifecycle_config::LifecycleConfig;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutBucketLifecycleResponse; use crate::s3::response::PutBucketLifecycleResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert, md5sum_hash}; use crate::s3::utils::{check_bucket_name, insert, md5sum_hash};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketNotificationResponse; use crate::s3::response::PutBucketNotificationResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{NotificationConfig, S3Api, S3Request, ToS3Request}; use crate::s3::types::{NotificationConfig, S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketPolicyResponse; use crate::s3::response::PutBucketPolicyResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketReplicationResponse; use crate::s3::response::PutBucketReplicationResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{ReplicationConfig, S3Api, S3Request, ToS3Request}; use crate::s3::types::{ReplicationConfig, S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketTaggingResponse; use crate::s3::response::PutBucketTaggingResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketVersioningResponse; use crate::s3::response::PutBucketVersioningResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -18,7 +18,7 @@ use crate::s3::builders::{ContentStream, Size};
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::error::{Error, IoError, ValidationErr}; use crate::s3::error::{Error, IoError, ValidationErr};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::HasEtagFromHeaders; use crate::s3::response::a_response_traits::HasEtagFromHeaders;
use crate::s3::response::{ use crate::s3::response::{
AbortMultipartUploadResponse, CompleteMultipartUploadResponse, CreateMultipartUploadResponse, AbortMultipartUploadResponse, CompleteMultipartUploadResponse, CreateMultipartUploadResponse,

View File

@ -16,7 +16,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutObjectLegalHoldResponse; use crate::s3::response::PutObjectLegalHoldResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash}; use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash};

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutObjectLockConfigResponse; use crate::s3::response::PutObjectLockConfigResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{ObjectLockConfig, S3Api, S3Request, ToS3Request}; use crate::s3::types::{ObjectLockConfig, S3Api, S3Request, ToS3Request};

View File

@ -16,7 +16,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutObjectRetentionResponse; use crate::s3::response::PutObjectRetentionResponse;
use crate::s3::types::{RetentionMode, S3Api, S3Request, ToS3Request}; use crate::s3::types::{RetentionMode, S3Api, S3Request, ToS3Request};
use crate::s3::utils::{ use crate::s3::utils::{

View File

@ -15,7 +15,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutObjectTaggingResponse; use crate::s3::response::PutObjectTaggingResponse;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -16,7 +16,7 @@
use crate::s3::Client; use crate::s3::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::SelectObjectContentResponse; use crate::s3::response::SelectObjectContentResponse;
use crate::s3::sse::SseCustomerKey; use crate::s3::sse::SseCustomerKey;
use crate::s3::types::{S3Api, S3Request, SelectRequest, ToS3Request}; use crate::s3::types::{S3Api, S3Request, SelectRequest, ToS3Request};

View File

@ -16,7 +16,7 @@
use crate::s3::client::Client; use crate::s3::client::Client;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::StatObjectResponse; use crate::s3::response::StatObjectResponse;
use crate::s3::sse::{Sse, SseCustomerKey}; use crate::s3::sse::{Sse, SseCustomerKey};
use crate::s3::types::{S3Api, S3Request, ToS3Request}; use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,32 +15,31 @@
//! S3 client to perform bucket and object operations //! S3 client to perform bucket and object operations
use bytes::Bytes;
use dashmap::DashMap;
use http::HeaderMap;
use hyper::http::Method;
use reqwest::{Body, Response};
use std::fs::File; use std::fs::File;
use std::io::prelude::*; use std::io::prelude::*;
use std::mem; use std::mem;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, OnceLock};
use uuid::Uuid;
use crate::s3::builders::{BucketExists, ComposeSource}; use crate::s3::builders::{BucketExists, ComposeSource};
use crate::s3::creds::Provider; use crate::s3::creds::Provider;
use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::http::BaseUrl; use crate::s3::http::BaseUrl;
use crate::s3::minio_error_response::{MinioErrorCode, MinioErrorResponse}; use crate::s3::minio_error_response::{MinioErrorCode, MinioErrorResponse};
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::{HasEtagFromHeaders, HasS3Fields}; use crate::s3::response::a_response_traits::{HasEtagFromHeaders, HasS3Fields};
use crate::s3::response::*; use crate::s3::response::*;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::signer::sign_v4_s3; use crate::s3::signer::sign_v4_s3;
use crate::s3::utils::{EMPTY_SHA256, check_ssec_with_log, sha256_hash_sb, to_amz_date, utc_now}; use crate::s3::utils::{EMPTY_SHA256, check_ssec_with_log, sha256_hash_sb, to_amz_date, utc_now};
use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr};
use bytes::Bytes;
use dashmap::DashMap;
use http::HeaderMap;
use hyper::http::Method;
use rand::Rng;
use reqwest::{Body, Response};
mod append_object; mod append_object;
mod bucket_exists; mod bucket_exists;
mod copy_object; mod copy_object;
@ -276,12 +275,7 @@ impl Client {
*val *val
} else { } else {
// Create a random bucket name // Create a random bucket name
let bucket_name: String = rand::thread_rng() let bucket_name: String = Uuid::new_v4().to_string();
.sample_iter(&rand::distributions::Alphanumeric)
.take(20)
.map(char::from)
.collect::<String>()
.to_lowercase();
let express = match BucketExists::new(self.clone(), bucket_name).send().await { let express = match BucketExists::new(self.clone(), bucket_name).send().await {
Ok(v) => { Ok(v) => {

View File

@ -24,7 +24,7 @@ use crate::s3::response::{
}; };
use crate::s3::types::{S3Api, ToStream}; use crate::s3::types::{S3Api, ToStream};
use bytes::Bytes; use bytes::Bytes;
use futures::StreamExt; use futures_util::StreamExt;
impl Client { impl Client {
/// Creates a [`DeleteBucket`] request builder. /// Creates a [`DeleteBucket`] request builder.

View File

@ -19,9 +19,8 @@ use super::utils::urlencode_object_key;
use crate::s3::client::DEFAULT_REGION; use crate::s3::client::DEFAULT_REGION;
use crate::s3::error::ValidationErr; use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::utils::match_hostname; use crate::s3::utils::match_hostname;
use derivative::Derivative;
use hyper::Uri; use hyper::Uri;
use hyper::http::Method; use hyper::http::Method;
use lazy_static::lazy_static; use lazy_static::lazy_static;
@ -37,11 +36,9 @@ lazy_static! {
static ref AWS_S3_PREFIX_REGEX: Regex = Regex::new(AWS_S3_PREFIX).unwrap(); static ref AWS_S3_PREFIX_REGEX: Regex = Regex::new(AWS_S3_PREFIX).unwrap();
} }
#[derive(Derivative)] #[derive(Clone, Debug)]
#[derivative(Clone, Debug, Default)]
/// Represents HTTP URL /// Represents HTTP URL
pub struct Url { pub struct Url {
#[derivative(Default(value = "true"))]
pub https: bool, pub https: bool,
pub host: String, pub host: String,
pub port: u16, pub port: u16,
@ -58,6 +55,18 @@ impl Url {
} }
} }
impl Default for Url {
fn default() -> Self {
Self {
https: true,
host: String::default(),
port: u16::default(),
path: String::default(),
query: Multimap::default(),
}
}
}
impl fmt::Display for Url { impl fmt::Display for Url {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.host.is_empty() { if self.host.is_empty() {
@ -208,11 +217,9 @@ fn get_aws_info(
Ok(()) Ok(())
} }
#[derive(Derivative)] #[derive(Clone, Debug)]
#[derivative(Clone, Debug, Default)]
/// Represents Base URL of S3 endpoint /// Represents Base URL of S3 endpoint
pub struct BaseUrl { pub struct BaseUrl {
#[derivative(Default(value = "true"))]
pub https: bool, pub https: bool,
host: String, host: String,
port: u16, port: u16,
@ -223,6 +230,21 @@ pub struct BaseUrl {
pub virtual_style: bool, pub virtual_style: bool,
} }
impl Default for BaseUrl {
fn default() -> Self {
Self {
https: true,
host: "127.0.0.1".to_string(),
port: 9000,
region: "".to_string(),
aws_s3_prefix: "".to_string(),
aws_domain_suffix: "".to_string(),
dualstack: false,
virtual_style: false,
}
}
}
impl FromStr for BaseUrl { impl FromStr for BaseUrl {
type Err = ValidationErr; type Err = ValidationErr;

View File

@ -23,7 +23,7 @@ pub mod header_constants;
pub mod http; pub mod http;
pub mod lifecycle_config; pub mod lifecycle_config;
pub mod minio_error_response; pub mod minio_error_response;
pub mod multimap; pub mod multimap_ext;
mod object_content; mod object_content;
pub mod response; pub mod response;
pub mod segmented_bytes; pub mod segmented_bytes;

View File

@ -49,9 +49,7 @@ impl MultimapExt for Multimap {
} }
fn add_multimap(&mut self, other: Multimap) { fn add_multimap(&mut self, other: Multimap) {
for (key, values) in other.into_iter() { for (key, values) in other.into_iter() {
for value in values { self.insert_many(key.clone(), values);
self.insert(key.clone(), value);
}
} }
} }
fn add_version(&mut self, version: Option<String>) { fn add_version(&mut self, version: Option<String>) {

View File

@ -13,14 +13,14 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::s3::segmented_bytes::SegmentedBytes;
use async_std::io::{ReadExt, WriteExt}; use async_std::io::{ReadExt, WriteExt};
use bytes::Bytes; use bytes::Bytes;
use futures::stream::{self, Stream, StreamExt}; use futures_util::stream::{self, Stream, StreamExt};
use rand::prelude::random;
use std::path::PathBuf; use std::path::PathBuf;
use std::{ffi::OsString, fs, path::Path, pin::Pin}; use std::{fs, path::Path, pin::Pin};
use uuid::Uuid;
use crate::s3::segmented_bytes::SegmentedBytes;
#[cfg(test)] #[cfg(test)]
use quickcheck::Arbitrary; use quickcheck::Arbitrary;
@ -216,13 +216,11 @@ impl ObjectContent {
async_std::fs::create_dir_all(parent_dir).await?; async_std::fs::create_dir_all(parent_dir).await?;
} }
let file_name = file_path.file_name().ok_or(std::io::Error::other( let file_name = file_path.file_name().ok_or(std::io::Error::other(
"could not get filename component of path", "could not get filename-component of path",
))?; ))?;
let mut tmp_file_name: OsString = file_name.to_os_string(); let mut tmp_file_name = file_name.to_os_string();
tmp_file_name.push(format!("_{}", random::<u64>())); tmp_file_name.push(format!("_{}", Uuid::new_v4().to_string().replace('-', "_")));
let tmp_file_path = parent_dir let tmp_file_path = parent_dir.join(tmp_file_name);
.to_path_buf()
.join(Path::new(tmp_file_name.as_os_str()));
let mut total_bytes_written = 0; let mut total_bytes_written = 0;
let mut fp = async_std::fs::OpenOptions::new() let mut fp = async_std::fs::OpenOptions::new()

View File

@ -15,7 +15,7 @@
use crate::impl_has_s3fields; use crate::impl_has_s3fields;
use crate::s3::error::{Error, ValidationErr}; use crate::s3::error::{Error, ValidationErr};
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::{HasBucket, HasObject, HasRegion, HasS3Fields}; use crate::s3::response::a_response_traits::{HasBucket, HasObject, HasRegion, HasS3Fields};
use crate::s3::types::{FromS3Response, S3Request, SelectProgress}; use crate::s3::types::{FromS3Response, S3Request, SelectProgress};
use crate::s3::utils::{copy_slice, crc32, get_text_result, uint32}; use crate::s3::utils::{copy_slice, crc32, get_text_result, uint32};

View File

@ -16,9 +16,8 @@
//! Signature V4 for S3 API //! Signature V4 for S3 API
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::utils::{UtcTime, sha256_hash, to_amz_date, to_signer_date}; use crate::s3::utils::{UtcTime, hex_encode, sha256_hash, to_amz_date, to_signer_date};
use hex::encode as hexencode;
#[cfg(not(feature = "ring"))] #[cfg(not(feature = "ring"))]
use hmac::{Hmac, Mac}; use hmac::{Hmac, Mac};
use hyper::http::Method; use hyper::http::Method;
@ -45,16 +44,14 @@ fn hmac_hash(key: &[u8], data: &[u8]) -> Vec<u8> {
/// Returns hex encoded HMAC hash for given key and data /// Returns hex encoded HMAC hash for given key and data
fn hmac_hash_hex(key: &[u8], data: &[u8]) -> String { fn hmac_hash_hex(key: &[u8], data: &[u8]) -> String {
hexencode(hmac_hash(key, data)) hex_encode(hmac_hash(key, data).as_slice())
} }
/// Returns scope value of given date, region and service name /// Returns scope value of given date, region and service name
fn get_scope(date: UtcTime, region: &str, service_name: &str) -> String { fn get_scope(date: UtcTime, region: &str, service_name: &str) -> String {
format!( format!(
"{}/{}/{}/aws4_request", "{}/{region}/{service_name}/aws4_request",
to_signer_date(date), to_signer_date(date)
region,
service_name
) )
} }
@ -76,10 +73,8 @@ fn get_canonical_request_hash(
/// Returns string-to-sign value of given date, scope and canonical request hash /// Returns string-to-sign value of given date, scope and canonical request hash
fn get_string_to_sign(date: UtcTime, scope: &str, canonical_request_hash: &str) -> String { fn get_string_to_sign(date: UtcTime, scope: &str, canonical_request_hash: &str) -> String {
format!( format!(
"AWS4-HMAC-SHA256\n{}\n{}\n{}", "AWS4-HMAC-SHA256\n{}\n{scope}\n{canonical_request_hash}",
to_amz_date(date), to_amz_date(date)
scope,
canonical_request_hash
) )
} }

View File

@ -16,8 +16,8 @@
//! Server side encryption definitions //! Server side encryption definitions
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt}; use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::utils::{b64encode, md5sum_hash}; use crate::s3::utils::{b64_encode, md5sum_hash};
use std::any::Any; use std::any::Any;
/// Base server side encryption /// Base server side encryption
@ -39,7 +39,7 @@ pub struct SseCustomerKey {
impl SseCustomerKey { impl SseCustomerKey {
pub fn new(key: &str) -> Self { pub fn new(key: &str) -> Self {
let b64key: String = b64encode(key); let b64key: String = b64_encode(key);
let md5key: String = md5sum_hash(key.as_bytes()); let md5key: String = md5sum_hash(key.as_bytes());
let mut headers = Multimap::with_capacity(3); let mut headers = Multimap::with_capacity(3);
@ -102,7 +102,7 @@ impl SseKms {
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key); headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key);
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION, "aws:kms"); headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION, "aws:kms");
if let Some(v) = context { if let Some(v) = context {
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64encode(v)); headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64_encode(v));
} }
SseKms { headers } SseKms { headers }

View File

@ -18,7 +18,7 @@
use super::client::{Client, DEFAULT_REGION}; use super::client::{Client, DEFAULT_REGION};
use crate::s3::error::{Error, ValidationErr}; use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*; use crate::s3::header_constants::*;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::utils::{UtcTime, get_text_option, get_text_result}; use crate::s3::utils::{UtcTime, get_text_option, get_text_result};
use async_trait::async_trait; use async_trait::async_trait;

View File

@ -13,18 +13,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
//! Various utility and helper functions use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap; use crate::s3::multimap_ext::Multimap;
use crate::s3::segmented_bytes::SegmentedBytes; use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::sse::{Sse, SseCustomerKey};
use base64::engine::Engine as _; use base64::engine::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use byteorder::{BigEndian, ReadBytesExt};
use chrono::{DateTime, Datelike, NaiveDateTime, Utc}; use chrono::{DateTime, Datelike, NaiveDateTime, Utc};
use crc::{CRC_32_ISO_HDLC, Crc}; use crc::{CRC_32_ISO_HDLC, Crc};
use hex::ToHex;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use md5::compute as md5compute;
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode}; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
use regex::Regex; use regex::Regex;
#[cfg(feature = "ring")] #[cfg(feature = "ring")]
@ -34,12 +31,9 @@ use sha2::{Digest, Sha256};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use xmltree::Element; use xmltree::Element;
/// Date and time with UTC timezone /// Date and time with UTC timezone
pub type UtcTime = DateTime<Utc>; pub type UtcTime = DateTime<Utc>;
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::sse::{Sse, SseCustomerKey};
use url::form_urlencoded;
// Great stuff to get confused about. // Great stuff to get confused about.
// String "a b+c" in Percent-Encoding (RFC 3986) becomes "a%20b%2Bc". // String "a b+c" in Percent-Encoding (RFC 3986) becomes "a%20b%2Bc".
@ -50,7 +44,7 @@ use url::form_urlencoded;
/// Decodes a URL-encoded string in the application/x-www-form-urlencoded syntax into a string. /// Decodes a URL-encoded string in the application/x-www-form-urlencoded syntax into a string.
/// Note that "+" is decoded to a space character, and "%2B" is decoded to a plus sign. /// Note that "+" is decoded to a space character, and "%2B" is decoded to a plus sign.
pub fn url_decode(s: &str) -> String { pub fn url_decode(s: &str) -> String {
form_urlencoded::parse(s.as_bytes()) url::form_urlencoded::parse(s.as_bytes())
.map(|(k, _)| k) .map(|(k, _)| k)
.collect() .collect()
} }
@ -62,8 +56,8 @@ pub fn url_encode(s: &str) -> String {
} }
/// Encodes data using base64 algorithm /// Encodes data using base64 algorithm
pub fn b64encode(input: impl AsRef<[u8]>) -> String { pub fn b64_encode(input: impl AsRef<[u8]>) -> String {
BASE64.encode(input) base64::engine::general_purpose::STANDARD.encode(input)
} }
/// Computes CRC32 of given data. /// Computes CRC32 of given data.
@ -73,12 +67,17 @@ pub fn crc32(data: &[u8]) -> u32 {
} }
/// Converts data array into 32 bit BigEndian unsigned int /// Converts data array into 32 bit BigEndian unsigned int
pub fn uint32(mut data: &[u8]) -> Result<u32, ValidationErr> { pub fn uint32(data: &[u8]) -> Result<u32, ValidationErr> {
data.read_u32::<BigEndian>() if data.len() < 4 {
.map_err(|e| ValidationErr::InvalidIntegerValue { return Err(ValidationErr::InvalidIntegerValue {
message: "data is not a valid 32-bit BigEndian unsigned integer".into(), message: "data is not a valid 32-bit BigEndian unsigned integer".into(),
source: Box::new(e), source: Box::new(std::io::Error::new(
}) std::io::ErrorKind::UnexpectedEof,
"not enough bytes",
)),
});
}
Ok(u32::from_be_bytes(data[..4].try_into().unwrap()))
} }
/// sha256 hash of empty data /// sha256 hash of empty data
@ -88,32 +87,84 @@ pub const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934c
pub fn sha256_hash(data: &[u8]) -> String { pub fn sha256_hash(data: &[u8]) -> String {
#[cfg(feature = "ring")] #[cfg(feature = "ring")]
{ {
ring::digest::digest(&SHA256, data).encode_hex() hex_encode(ring::digest::digest(&SHA256, data).as_ref())
} }
#[cfg(not(feature = "ring"))] #[cfg(not(feature = "ring"))]
{ {
Sha256::new_with_prefix(data).finalize().encode_hex() hex_encode(Sha256::new_with_prefix(data).finalize().as_slice())
} }
} }
/// Hex-encode a byte slice into a lowercase ASCII string.
///
/// # Safety
/// This implementation uses `unsafe` code for performance reasons:
/// - We call [`String::as_mut_vec`] to get direct access to the
/// underlying `Vec<u8>` backing the `String`.
/// - We then use [`set_len`] to pre-allocate the final length without
/// initializing the contents first.
/// - Finally, we use [`get_unchecked`] and [`get_unchecked_mut`] to
/// avoid bounds checking inside the tight encoding loop.
///
/// # Why unsafe is needed
/// Normally, writing this function with safe Rust requires:
/// - Pushing each hex digit one-by-one into the string (extra bounds checks).
/// - Or allocating and copying temporary buffers.
///
/// Using `unsafe` avoids redundant checks and makes this implementation
/// significantly faster, especially for large inputs.
///
/// # Why this is correct
/// - `s` is allocated with exactly `len * 2` capacity, and we immediately
/// set its length to that value. Every byte in the string buffer will be
/// initialized before being read or used.
/// - The loop index `i` is always in `0..len`, so `bytes.get_unchecked(i)`
/// is safe.
/// - Each write goes to positions `j` and `j + 1`, where `j = i * 2`.
/// Since `i < len`, the maximum write index is `2*len - 1`, which is
/// within the allocated range.
/// - All written bytes come from the `LUT` table, which has exactly 16
/// elements, and indices are masked into the 015 range.
///
/// Therefore, although `unsafe` is used to skip bounds checking,
/// the logic ensures all memory accesses remain in-bounds and initialized.
pub fn hex_encode(bytes: &[u8]) -> String {
const LUT: &[u8; 16] = b"0123456789abcdef";
let len = bytes.len();
let mut s = String::with_capacity(len * 2);
unsafe {
let v = s.as_mut_vec();
v.set_len(len * 2);
for i in 0..len {
let b = bytes.get_unchecked(i);
let hi = LUT.get_unchecked((b >> 4) as usize);
let lo = LUT.get_unchecked((b & 0xF) as usize);
let j = i * 2;
*v.get_unchecked_mut(j) = *hi;
*v.get_unchecked_mut(j + 1) = *lo;
}
}
s
}
pub fn sha256_hash_sb(sb: Arc<SegmentedBytes>) -> String { pub fn sha256_hash_sb(sb: Arc<SegmentedBytes>) -> String {
#[cfg(feature = "ring")] #[cfg(feature = "ring")]
{ {
let mut context = Context::new(&SHA256); let mut context = Context::new(&SHA256);
for data in sb.iter() { for data in sb.iter() {
// Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap
context.update(data.as_ref()); context.update(data.as_ref());
} }
context.finish().encode_hex() hex_encode(context.finish().as_ref())
} }
#[cfg(not(feature = "ring"))] #[cfg(not(feature = "ring"))]
{ {
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
for data in sb.iter() { for data in sb.iter() {
// Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap
hasher.update(data); hasher.update(data);
} }
hasher.finalize().encode_hex() hex_encode(hasher.finalize().as_slice())
} }
} }
@ -134,7 +185,7 @@ mod tests {
/// Gets bas64 encoded MD5 hash of given data /// Gets bas64 encoded MD5 hash of given data
pub fn md5sum_hash(data: &[u8]) -> String { pub fn md5sum_hash(data: &[u8]) -> String {
b64encode(md5compute(data).as_slice()) b64_encode(md5::compute(data).as_slice())
} }
/// Gets current UTC time /// Gets current UTC time
@ -480,7 +531,7 @@ fn escape(s: &str) -> String {
// //
// Handles escaping same as MinIO server - needed for ensuring compatibility. // Handles escaping same as MinIO server - needed for ensuring compatibility.
pub fn encode_tags(h: &HashMap<String, String>) -> String { pub fn encode_tags(h: &HashMap<String, String>) -> String {
let mut tags = Vec::new(); let mut tags = Vec::with_capacity(h.len());
for (k, v) in h { for (k, v) in h {
tags.push(format!("{}={}", escape(k), escape(v))); tags.push(format!("{}={}", escape(k), escape(v)));
} }

View File

@ -12,3 +12,7 @@ cargo test -- --nocapture
# run one specific test and show stdout # run one specific test and show stdout
# cargo test --test test_bucket_exists -- --nocapture # cargo test --test test_bucket_exists -- --nocapture
# run tests with ring instead of default-crypto
# cargo test --no-default-features --features "default-tls,ring" -- --nocapture

View File

@ -14,11 +14,11 @@
// limitations under the License. // limitations under the License.
use async_std::io::ReadExt; use async_std::io::ReadExt;
use hex::ToHex;
use minio::s3::builders::ObjectContent; use minio::s3::builders::ObjectContent;
use minio::s3::response::a_response_traits::{HasBucket, HasObject}; use minio::s3::response::a_response_traits::{HasBucket, HasObject};
use minio::s3::response::{GetObjectResponse, PutObjectContentResponse}; use minio::s3::response::{GetObjectResponse, PutObjectContentResponse};
use minio::s3::types::S3Api; use minio::s3::types::S3Api;
use minio::s3::utils::hex_encode;
use minio_common::rand_reader::RandReader; use minio_common::rand_reader::RandReader;
use minio_common::test_context::TestContext; use minio_common::test_context::TestContext;
use minio_common::utils::rand_object_name_utf8; use minio_common::utils::rand_object_name_utf8;
@ -36,7 +36,7 @@ async fn get_hash(filename: &str) -> String {
let mut buf = Vec::new(); let mut buf = Vec::new();
file.read_to_end(&mut buf).await.unwrap(); file.read_to_end(&mut buf).await.unwrap();
context.update(&buf); context.update(&buf);
context.finish().encode_hex() hex_encode(context.finish().as_ref())
} }
#[cfg(not(feature = "ring"))] #[cfg(not(feature = "ring"))]
{ {
@ -45,7 +45,7 @@ async fn get_hash(filename: &str) -> String {
let mut buf = Vec::new(); let mut buf = Vec::new();
file.read_to_end(&mut buf).await.unwrap(); file.read_to_end(&mut buf).await.unwrap();
hasher.update(&buf); hasher.update(&buf);
hasher.finalize().encode_hex() hex_encode(hasher.finalize().as_slice())
} }
} }