Fixed clippy issues and cleanup of unnecessary imports (#182)

This commit is contained in:
Henk-Jan Lebbink 2025-08-26 11:04:51 +02:00 committed by GitHub
parent 5080bf9b85
commit 25d424b97f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
58 changed files with 265 additions and 197 deletions

View File

@ -10,11 +10,6 @@ readme = "README.md"
keywords = ["object-storage", "minio", "s3"]
categories = ["api-bindings", "web-programming::http-client"]
[dependencies.reqwest]
version = "0.12.22"
default-features = false
features = ["stream"]
[features]
default = ["default-tls", "default-crypto"]
default-tls = ["reqwest/default-tls"]
@ -23,21 +18,29 @@ rustls-tls = ["reqwest/rustls-tls"]
default-crypto = ["dep:sha2", "dep:hmac"]
ring = ["dep:ring"]
[workspace.dependencies]
uuid = "1.18"
futures-util = "0.3"
reqwest = { version = "0.12", default-features = false }
bytes = "1.10"
async-std = "1.13"
[dependencies]
uuid = { workspace = true, features = ["v4"] }
futures-util = { workspace = true }
bytes = { workspace = true }
async-std = { workspace = true, features = ["attributes"] }
reqwest = { workspace = true, features = ["stream"] }
async-recursion = "1.1.1"
async-std = { version = "1.13.1", features = ["attributes"] }
async-stream = "0.3.6"
async-trait = "0.1.88"
base64 = "0.22.1"
byteorder = "1.5.0"
bytes = "1.10.1"
chrono = "0.4.41"
crc = "3.3.0"
dashmap = "6.1.0"
derivative = "2.2.0"
env_logger = "0.11.8"
futures-util = "0.3.31"
hex = "0.4.3"
hmac = { version = "0.12.1", optional = true }
hyper = { version = "1.6.0", features = ["full"] }
lazy_static = "1.5.0"
@ -46,26 +49,25 @@ md5 = "0.8.0"
multimap = "0.10.1"
percent-encoding = "2.3.1"
url = "2.5.4"
rand = { version = "0.8.5", features = ["small_rng"] }
regex = "1.11.1"
ring = { version = "0.17.14", optional = true, default-features = false, features = ["alloc"] }
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
serde_json = "1.0.142"
sha2 = { version = "0.10.9", optional = true }
urlencoding = "2.1.3"
xmltree = "0.11.0"
futures = "0.3.31"
http = "1.3.1"
thiserror = "2.0.12"
thiserror = "2.0.14"
[dev-dependencies]
minio-common = { path = "./common" }
minio-macros = { path = "./macros" }
tokio = { version = "1.47.1", features = ["full"] }
minio_common = { path = "./common" }
async-std = { version = "1.13.1", features = ["attributes", "tokio1"] }
clap = { version = "4.5.44", features = ["derive"] }
rand = { version = "0.9.2", features = ["small_rng"] }
quickcheck = "1.0.3"
criterion = "0.7.0"
minio-macros = { path = "./macros" }
[lib]
name = "minio"

View File

@ -1,20 +1,23 @@
[package]
name = "minio_common"
name = "minio-common"
version = "0.1.0"
edition = "2024"
[dependencies]
minio = {path = ".." }
uuid = { workspace = true, features = ["v4"] }
reqwest = { workspace = true }
bytes = { workspace = true }
async-std = { workspace = true }
futures-io = "0.3.31"
tokio = { version = "1.47.1", features = ["full"] }
async-std = "1.13.1"
rand = { version = "0.8.5", features = ["small_rng"] }
bytes = "1.10.1"
rand = { version = "0.9.2", features = ["small_rng"] }
log = "0.4.27"
chrono = "0.4.41"
reqwest = "0.12.22"
http = "1.3.1"
futures = "0.3.31"
uuid = { version = "1.18.0", features = ["v4"] }
[lib]
name = "minio_common"

View File

@ -38,7 +38,7 @@ impl CleanupGuard {
pub async fn cleanup(client: Client, bucket_name: &str) {
tokio::select!(
_ = tokio::time::sleep(std::time::Duration::from_secs(60)) => {
eprintln!("Cleanup timeout after 60s while removing bucket {}", bucket_name);
eprintln!("Cleanup timeout after 60s while removing bucket {bucket_name}");
},
outcome = client.delete_and_purge_bucket(bucket_name) => {
match outcome {
@ -46,7 +46,7 @@ pub async fn cleanup(client: Client, bucket_name: &str) {
//eprintln!("Bucket {} removed successfully", bucket_name);
}
Err(e) => {
eprintln!("Error removing bucket '{}':\n{}", bucket_name, e);
eprintln!("Error removing bucket '{bucket_name}':\n{e}");
}
}
}

View File

@ -140,7 +140,7 @@ pub fn create_bucket_replication_config_example(dst_bucket: &str) -> Replication
rules: vec![ReplicationRule {
id: Some(String::from("rule1")),
destination: Destination {
bucket_arn: String::from(&format!("arn:aws:s3:::{}", dst_bucket)),
bucket_arn: String::from(&format!("arn:aws:s3:::{dst_bucket}")),
..Default::default()
},
filter: Some(Filter {
@ -170,8 +170,8 @@ pub fn create_object_lock_config_example() -> ObjectLockConfig {
pub fn create_post_policy_example(bucket_name: &str, object_name: &str) -> PostPolicy {
let expiration: DateTime<Utc> = utc_now() + chrono::Duration::days(5);
let mut policy = PostPolicy::new(&bucket_name, expiration).unwrap();
policy.add_equals_condition("key", &object_name).unwrap();
let mut policy = PostPolicy::new(bucket_name, expiration).unwrap();
policy.add_equals_condition("key", object_name).unwrap();
policy
.add_content_length_range_condition(1024 * 1024, 4 * 1024 * 1024)
.unwrap();
@ -189,7 +189,7 @@ pub fn create_select_content_data() -> (String, String) {
(body, data)
}
pub fn create_select_content_request() -> SelectRequest {
let request = SelectRequest::new_csv_input_output(
SelectRequest::new_csv_input_output(
"select * from S3Object",
CsvInputSerialization {
compression_type: None,
@ -209,6 +209,5 @@ pub fn create_select_content_request() -> SelectRequest {
record_delimiter: None,
},
)
.unwrap();
request
.unwrap()
}

View File

@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use futures::AsyncRead;
use futures_io::AsyncRead;
use std::io;
use std::pin::Pin;
use std::task::{Context, Poll};
@ -34,7 +34,7 @@ impl io::Read for RandReader {
let bytes_read = buf.len().min(self.size as usize);
if bytes_read > 0 {
let random: &mut dyn rand::RngCore = &mut rand::thread_rng();
let random: &mut dyn rand::RngCore = &mut rand::rng();
random.fill_bytes(&mut buf[0..bytes_read]);
}
@ -53,7 +53,7 @@ impl AsyncRead for RandReader {
let bytes_read = buf.len().min(self.size as usize);
if bytes_read > 0 {
let random: &mut dyn rand::RngCore = &mut rand::thread_rng();
let random: &mut dyn rand::RngCore = &mut rand::rng();
random.fill_bytes(&mut buf[0..bytes_read]);
}

View File

@ -15,7 +15,7 @@
use async_std::stream::Stream;
use bytes::Bytes;
use futures::io::AsyncRead;
use futures_io::AsyncRead;
use rand::prelude::SmallRng;
use rand::{RngCore, SeedableRng};
use std::io;
@ -30,7 +30,7 @@ pub struct RandSrc {
impl RandSrc {
#[allow(dead_code)]
pub fn new(size: u64) -> RandSrc {
let rng = SmallRng::from_entropy();
let rng: SmallRng = SmallRng::from_os_rng();
RandSrc { size, rng }
}
}

View File

@ -84,10 +84,10 @@ impl TestContext {
let host: String =
std::env::var("SERVER_ENDPOINT").unwrap_or(DEFAULT_SERVER_ENDPOINT.to_string());
log::debug!("SERVER_ENDPOINT={}", host);
log::debug!("SERVER_ENDPOINT={host}");
let access_key: String =
std::env::var("ACCESS_KEY").unwrap_or(DEFAULT_ACCESS_KEY.to_string());
log::debug!("ACCESS_KEY={}", access_key);
log::debug!("ACCESS_KEY={access_key}");
let secret_key: String =
std::env::var("SECRET_KEY").unwrap_or(DEFAULT_SECRET_KEY.to_string());
log::debug!("SECRET_KEY=*****");
@ -95,19 +95,19 @@ impl TestContext {
.unwrap_or(DEFAULT_ENABLE_HTTPS.to_string())
.parse()
.unwrap_or(false);
log::debug!("ENABLE_HTTPS={}", secure);
log::debug!("ENABLE_HTTPS={secure}");
let ssl_cert: String =
std::env::var("MINIO_SSL_CERT_FILE").unwrap_or(DEFAULT_SSL_CERT_FILE.to_string());
log::debug!("MINIO_SSL_CERT_FILE={}", ssl_cert);
log::debug!("MINIO_SSL_CERT_FILE={ssl_cert}");
let ssl_cert_file: PathBuf = ssl_cert.into();
let ignore_cert_check: bool = std::env::var("IGNORE_CERT_CHECK")
.unwrap_or(DEFAULT_IGNORE_CERT_CHECK.to_string())
.parse()
.unwrap_or(true);
log::debug!("IGNORE_CERT_CHECK={}", ignore_cert_check);
log::debug!("IGNORE_CERT_CHECK={ignore_cert_check}");
let region: String =
std::env::var("SERVER_REGION").unwrap_or(DEFAULT_SERVER_REGION.to_string());
log::debug!("SERVER_REGION={:?}", region);
log::debug!("SERVER_REGION={region:?}");
let mut base_url: BaseUrl = host.parse().unwrap();
base_url.https = secure;

View File

@ -15,8 +15,9 @@
use http::{Response as HttpResponse, StatusCode};
use minio::s3::error::Error;
use rand::distributions::Standard;
use rand::{Rng, thread_rng};
use rand::Rng;
use rand::distr::StandardUniform;
use uuid::Uuid;
pub fn rand_bucket_name() -> String {
@ -28,9 +29,9 @@ pub fn rand_object_name() -> String {
}
pub fn rand_object_name_utf8(len: usize) -> String {
let rng = thread_rng();
rng.sample_iter::<char, _>(Standard)
.filter(|c| !c.is_control())
let rng = rand::rng();
rng.sample_iter(StandardUniform)
.filter(|c: &char| !c.is_control())
.take(len)
.collect()
}
@ -39,9 +40,9 @@ pub async fn get_bytes_from_response(v: Result<reqwest::Response, Error>) -> byt
match v {
Ok(r) => match r.bytes().await {
Ok(b) => b,
Err(e) => panic!("{:?}", e),
Err(e) => panic!("{e:?}"),
},
Err(e) => panic!("{:?}", e),
Err(e) => panic!("{e:?}"),
}
}
@ -52,5 +53,5 @@ pub fn get_response_from_bytes(bytes: bytes::Bytes) -> reqwest::Response {
.body(bytes)
.expect("Failed to build HTTP response");
reqwest::Response::try_from(http_response).expect("Failed to convert to reqwest::Response")
reqwest::Response::from(http_response)
}

View File

@ -22,7 +22,7 @@ use minio::s3::response::{AppendObjectResponse, StatObjectResponse};
use minio::s3::segmented_bytes::SegmentedBytes;
use minio::s3::types::S3Api;
use rand::Rng;
use rand::distributions::Alphanumeric;
use rand::distr::Alphanumeric;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@ -78,7 +78,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
}
fn random_string(len: usize) -> String {
rand::thread_rng()
rand::rng()
.sample_iter(&Alphanumeric)
.take(len)
.map(char::from)

View File

@ -3,18 +3,19 @@ name = "minio-macros"
version = "0.1.0"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
proc-macro = true
[dependencies]
uuid = { workspace = true, features = ["v4"] }
futures-util = { workspace = true }
syn = "2.0.104"
proc-macro2 = "1.0.95"
proc-macro2 = "1.0.97"
quote = "1.0.40"
darling = "0.21.0"
darling_core = "0.21.0"
uuid = { version = "1.17.0", features = ["v4"] }
[dev-dependencies]
minio_common = { path = "../common" }
minio-common = { path = "../common" }
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
proc-macro = true

View File

@ -104,6 +104,6 @@ pub fn test(
// Expand the macro
match test_attr::expand_test_macro(args, input_fn) {
Ok(expanded) => expanded.into(),
Err(err) => err.into(),
Err(err) => err,
}
}

View File

@ -55,7 +55,7 @@ impl MacroArgs {
}
// Validate that the function has exactly two arguments: ctx and bucket_name
if func.sig.inputs.len() != 2 && !self.no_bucket.is_present() {
if (func.sig.inputs.len() != 2) && !self.no_bucket.is_present() {
let error_msg = "Minio test function must have exactly two arguments: (ctx: TestContext, bucket_name: String)";
return Err(proc_macro::TokenStream::from(
Error::custom(error_msg)
@ -80,9 +80,10 @@ impl MacroArgs {
}
}
// Check second argument (bucket_name: String)
if !self.no_bucket.is_present() {
if let Some(FnArg::Typed(pat_type)) = iter.next() {
// Check the second argument (bucket_name: String)
if !self.no_bucket.is_present()
&& let Some(FnArg::Typed(pat_type)) = iter.next()
{
let type_str = pat_type.ty.to_token_stream().to_string();
if !type_str.contains("String") {
let error_msg = "Second argument must be of type String";
@ -93,7 +94,6 @@ impl MacroArgs {
));
}
}
}
Ok(())
}
@ -124,7 +124,7 @@ pub(crate) fn expand_test_macro(
// Setup common prelude
let prelude = quote!(
use ::futures::FutureExt;
use ::futures_util::FutureExt;
use ::std::panic::AssertUnwindSafe;
use ::minio::s3::types::S3Api;
use ::minio::s3::response::a_response_traits::HasBucket;

View File

@ -20,7 +20,7 @@ use crate::s3::builders::{
use crate::s3::error::ValidationErr;
use crate::s3::error::{Error, IoError};
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::HasObjectSize;
use crate::s3::response::{AppendObjectResponse, StatObjectResponse};
use crate::s3::segmented_bytes::SegmentedBytes;

View File

@ -14,7 +14,7 @@
// limitations under the License.
use crate::s3::client::Client;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use std::marker::PhantomData;
/// Common parameters for bucket operations

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::client::{MAX_MULTIPART_COUNT, MAX_PART_SIZE};
use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::HasEtagFromBody;
use crate::s3::response::{
AbortMultipartUploadResponse, CompleteMultipartUploadResponse, ComposeObjectResponse,

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::client::DEFAULT_REGION;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::CreateBucketResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::DeleteObjectTaggingResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::client::MAX_MULTIPART_COUNT;
use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::{DeleteError, DeleteObjectResponse, DeleteObjectsResponse};
use crate::s3::types::{ListEntry, S3Api, S3Request, ToS3Request, ToStream};
use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetBucketLifecycleResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::GetBucketTaggingResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert};

View File

@ -16,7 +16,7 @@
use crate::s3::client::Client;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectResponse;
use crate::s3::sse::{Sse, SseCustomerKey};
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectLegalHoldResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::client::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectPromptResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::sse::SseCustomerKey;

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectRetentionResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetObjectTaggingResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert};

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::creds::Credentials;
use crate::s3::error::Error;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::GetPresignedObjectUrlResponse;
use crate::s3::signer::presign_v4;
use crate::s3::utils::{UtcTime, check_bucket_name, check_object_name, utc_now};

View File

@ -19,7 +19,7 @@ use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*;
use crate::s3::signer::post_presign_v4;
use crate::s3::utils::{
UtcTime, b64encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now,
UtcTime, b64_encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now,
};
use serde_json::{Value, json};
use std::collections::HashMap;
@ -341,7 +341,7 @@ impl PostPolicy {
"conditions": conditions,
});
let encoded_policy = b64encode(policy.to_string());
let encoded_policy = b64_encode(policy.to_string());
let signature = post_presign_v4(&encoded_policy, &secret_key, date, &region);
let mut data: HashMap<String, String> = HashMap::new();

View File

@ -16,7 +16,7 @@
use crate::s3::Client;
use crate::s3::client::DEFAULT_REGION;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::GetRegionResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::ListBucketsResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use http::Method;

View File

@ -14,7 +14,7 @@
use crate::s3::client::Client;
use crate::s3::error::{Error, ValidationErr};
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::ListObjectsResponse;
use crate::s3::response::list_objects::{
ListObjectVersionsResponse, ListObjectsV1Response, ListObjectsV2Response,

View File

@ -15,7 +15,7 @@
use crate::s3::client::Client;
use crate::s3::error::{Error, ValidationErr};
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::ListenBucketNotificationResponse;
use crate::s3::types::{NotificationRecords, S3Api, S3Request, ToS3Request};
use crate::s3::utils::check_bucket_name;

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketEncryptionResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, SseConfig, ToS3Request};

View File

@ -17,7 +17,7 @@ use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::lifecycle_config::LifecycleConfig;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutBucketLifecycleResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, insert, md5sum_hash};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketNotificationResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{NotificationConfig, S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketPolicyResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketReplicationResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{ReplicationConfig, S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketTaggingResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutBucketVersioningResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -18,7 +18,7 @@ use crate::s3::builders::{ContentStream, Size};
use crate::s3::client::Client;
use crate::s3::error::{Error, IoError, ValidationErr};
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::HasEtagFromHeaders;
use crate::s3::response::{
AbortMultipartUploadResponse, CompleteMultipartUploadResponse, CreateMultipartUploadResponse,

View File

@ -16,7 +16,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutObjectLegalHoldResponse;
use crate::s3::types::{S3Api, S3Request, ToS3Request};
use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash};

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::response::PutObjectLockConfigResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{ObjectLockConfig, S3Api, S3Request, ToS3Request};

View File

@ -16,7 +16,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutObjectRetentionResponse;
use crate::s3::types::{RetentionMode, S3Api, S3Request, ToS3Request};
use crate::s3::utils::{

View File

@ -15,7 +15,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::PutObjectTaggingResponse;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -16,7 +16,7 @@
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::SelectObjectContentResponse;
use crate::s3::sse::SseCustomerKey;
use crate::s3::types::{S3Api, S3Request, SelectRequest, ToS3Request};

View File

@ -16,7 +16,7 @@
use crate::s3::client::Client;
use crate::s3::error::ValidationErr;
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::StatObjectResponse;
use crate::s3::sse::{Sse, SseCustomerKey};
use crate::s3::types::{S3Api, S3Request, ToS3Request};

View File

@ -15,32 +15,31 @@
//! S3 client to perform bucket and object operations
use bytes::Bytes;
use dashmap::DashMap;
use http::HeaderMap;
use hyper::http::Method;
use reqwest::{Body, Response};
use std::fs::File;
use std::io::prelude::*;
use std::mem;
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock};
use uuid::Uuid;
use crate::s3::builders::{BucketExists, ComposeSource};
use crate::s3::creds::Provider;
use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr};
use crate::s3::header_constants::*;
use crate::s3::http::BaseUrl;
use crate::s3::minio_error_response::{MinioErrorCode, MinioErrorResponse};
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::{HasEtagFromHeaders, HasS3Fields};
use crate::s3::response::*;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::signer::sign_v4_s3;
use crate::s3::utils::{EMPTY_SHA256, check_ssec_with_log, sha256_hash_sb, to_amz_date, utc_now};
use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr};
use bytes::Bytes;
use dashmap::DashMap;
use http::HeaderMap;
use hyper::http::Method;
use rand::Rng;
use reqwest::{Body, Response};
mod append_object;
mod bucket_exists;
mod copy_object;
@ -276,12 +275,7 @@ impl Client {
*val
} else {
// Create a random bucket name
let bucket_name: String = rand::thread_rng()
.sample_iter(&rand::distributions::Alphanumeric)
.take(20)
.map(char::from)
.collect::<String>()
.to_lowercase();
let bucket_name: String = Uuid::new_v4().to_string();
let express = match BucketExists::new(self.clone(), bucket_name).send().await {
Ok(v) => {

View File

@ -24,7 +24,7 @@ use crate::s3::response::{
};
use crate::s3::types::{S3Api, ToStream};
use bytes::Bytes;
use futures::StreamExt;
use futures_util::StreamExt;
impl Client {
/// Creates a [`DeleteBucket`] request builder.

View File

@ -19,9 +19,8 @@ use super::utils::urlencode_object_key;
use crate::s3::client::DEFAULT_REGION;
use crate::s3::error::ValidationErr;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::utils::match_hostname;
use derivative::Derivative;
use hyper::Uri;
use hyper::http::Method;
use lazy_static::lazy_static;
@ -37,11 +36,9 @@ lazy_static! {
static ref AWS_S3_PREFIX_REGEX: Regex = Regex::new(AWS_S3_PREFIX).unwrap();
}
#[derive(Derivative)]
#[derivative(Clone, Debug, Default)]
#[derive(Clone, Debug)]
/// Represents HTTP URL
pub struct Url {
#[derivative(Default(value = "true"))]
pub https: bool,
pub host: String,
pub port: u16,
@ -58,6 +55,18 @@ impl Url {
}
}
impl Default for Url {
fn default() -> Self {
Self {
https: true,
host: String::default(),
port: u16::default(),
path: String::default(),
query: Multimap::default(),
}
}
}
impl fmt::Display for Url {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.host.is_empty() {
@ -208,11 +217,9 @@ fn get_aws_info(
Ok(())
}
#[derive(Derivative)]
#[derivative(Clone, Debug, Default)]
#[derive(Clone, Debug)]
/// Represents Base URL of S3 endpoint
pub struct BaseUrl {
#[derivative(Default(value = "true"))]
pub https: bool,
host: String,
port: u16,
@ -223,6 +230,21 @@ pub struct BaseUrl {
pub virtual_style: bool,
}
impl Default for BaseUrl {
fn default() -> Self {
Self {
https: true,
host: "127.0.0.1".to_string(),
port: 9000,
region: "".to_string(),
aws_s3_prefix: "".to_string(),
aws_domain_suffix: "".to_string(),
dualstack: false,
virtual_style: false,
}
}
}
impl FromStr for BaseUrl {
type Err = ValidationErr;

View File

@ -23,7 +23,7 @@ pub mod header_constants;
pub mod http;
pub mod lifecycle_config;
pub mod minio_error_response;
pub mod multimap;
pub mod multimap_ext;
mod object_content;
pub mod response;
pub mod segmented_bytes;

View File

@ -49,9 +49,7 @@ impl MultimapExt for Multimap {
}
fn add_multimap(&mut self, other: Multimap) {
for (key, values) in other.into_iter() {
for value in values {
self.insert(key.clone(), value);
}
self.insert_many(key.clone(), values);
}
}
fn add_version(&mut self, version: Option<String>) {

View File

@ -13,14 +13,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::s3::segmented_bytes::SegmentedBytes;
use async_std::io::{ReadExt, WriteExt};
use bytes::Bytes;
use futures::stream::{self, Stream, StreamExt};
use rand::prelude::random;
use futures_util::stream::{self, Stream, StreamExt};
use std::path::PathBuf;
use std::{ffi::OsString, fs, path::Path, pin::Pin};
use std::{fs, path::Path, pin::Pin};
use uuid::Uuid;
use crate::s3::segmented_bytes::SegmentedBytes;
#[cfg(test)]
use quickcheck::Arbitrary;
@ -216,13 +216,11 @@ impl ObjectContent {
async_std::fs::create_dir_all(parent_dir).await?;
}
let file_name = file_path.file_name().ok_or(std::io::Error::other(
"could not get filename component of path",
"could not get filename-component of path",
))?;
let mut tmp_file_name: OsString = file_name.to_os_string();
tmp_file_name.push(format!("_{}", random::<u64>()));
let tmp_file_path = parent_dir
.to_path_buf()
.join(Path::new(tmp_file_name.as_os_str()));
let mut tmp_file_name = file_name.to_os_string();
tmp_file_name.push(format!("_{}", Uuid::new_v4().to_string().replace('-', "_")));
let tmp_file_path = parent_dir.join(tmp_file_name);
let mut total_bytes_written = 0;
let mut fp = async_std::fs::OpenOptions::new()

View File

@ -15,7 +15,7 @@
use crate::impl_has_s3fields;
use crate::s3::error::{Error, ValidationErr};
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::response::a_response_traits::{HasBucket, HasObject, HasRegion, HasS3Fields};
use crate::s3::types::{FromS3Response, S3Request, SelectProgress};
use crate::s3::utils::{copy_slice, crc32, get_text_result, uint32};

View File

@ -16,9 +16,8 @@
//! Signature V4 for S3 API
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::utils::{UtcTime, sha256_hash, to_amz_date, to_signer_date};
use hex::encode as hexencode;
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::utils::{UtcTime, hex_encode, sha256_hash, to_amz_date, to_signer_date};
#[cfg(not(feature = "ring"))]
use hmac::{Hmac, Mac};
use hyper::http::Method;
@ -45,16 +44,14 @@ fn hmac_hash(key: &[u8], data: &[u8]) -> Vec<u8> {
/// Returns hex encoded HMAC hash for given key and data
fn hmac_hash_hex(key: &[u8], data: &[u8]) -> String {
hexencode(hmac_hash(key, data))
hex_encode(hmac_hash(key, data).as_slice())
}
/// Returns scope value of given date, region and service name
fn get_scope(date: UtcTime, region: &str, service_name: &str) -> String {
format!(
"{}/{}/{}/aws4_request",
to_signer_date(date),
region,
service_name
"{}/{region}/{service_name}/aws4_request",
to_signer_date(date)
)
}
@ -76,10 +73,8 @@ fn get_canonical_request_hash(
/// Returns string-to-sign value of given date, scope and canonical request hash
fn get_string_to_sign(date: UtcTime, scope: &str, canonical_request_hash: &str) -> String {
format!(
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
to_amz_date(date),
scope,
canonical_request_hash
"AWS4-HMAC-SHA256\n{}\n{scope}\n{canonical_request_hash}",
to_amz_date(date)
)
}

View File

@ -16,8 +16,8 @@
//! Server side encryption definitions
use crate::s3::header_constants::*;
use crate::s3::multimap::{Multimap, MultimapExt};
use crate::s3::utils::{b64encode, md5sum_hash};
use crate::s3::multimap_ext::{Multimap, MultimapExt};
use crate::s3::utils::{b64_encode, md5sum_hash};
use std::any::Any;
/// Base server side encryption
@ -39,7 +39,7 @@ pub struct SseCustomerKey {
impl SseCustomerKey {
pub fn new(key: &str) -> Self {
let b64key: String = b64encode(key);
let b64key: String = b64_encode(key);
let md5key: String = md5sum_hash(key.as_bytes());
let mut headers = Multimap::with_capacity(3);
@ -102,7 +102,7 @@ impl SseKms {
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key);
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION, "aws:kms");
if let Some(v) = context {
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64encode(v));
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64_encode(v));
}
SseKms { headers }

View File

@ -18,7 +18,7 @@
use super::client::{Client, DEFAULT_REGION};
use crate::s3::error::{Error, ValidationErr};
use crate::s3::header_constants::*;
use crate::s3::multimap::Multimap;
use crate::s3::multimap_ext::Multimap;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::utils::{UtcTime, get_text_option, get_text_result};
use async_trait::async_trait;

View File

@ -13,18 +13,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Various utility and helper functions
use crate::s3::multimap::Multimap;
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::multimap_ext::Multimap;
use crate::s3::segmented_bytes::SegmentedBytes;
use crate::s3::sse::{Sse, SseCustomerKey};
use base64::engine::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64;
use byteorder::{BigEndian, ReadBytesExt};
use chrono::{DateTime, Datelike, NaiveDateTime, Utc};
use crc::{CRC_32_ISO_HDLC, Crc};
use hex::ToHex;
use lazy_static::lazy_static;
use md5::compute as md5compute;
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
use regex::Regex;
#[cfg(feature = "ring")]
@ -34,12 +31,9 @@ use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::sync::Arc;
use xmltree::Element;
/// Date and time with UTC timezone
pub type UtcTime = DateTime<Utc>;
use crate::s3::Client;
use crate::s3::error::ValidationErr;
use crate::s3::sse::{Sse, SseCustomerKey};
use url::form_urlencoded;
// Great stuff to get confused about.
// String "a b+c" in Percent-Encoding (RFC 3986) becomes "a%20b%2Bc".
@ -50,7 +44,7 @@ use url::form_urlencoded;
/// Decodes a URL-encoded string in the application/x-www-form-urlencoded syntax into a string.
/// Note that "+" is decoded to a space character, and "%2B" is decoded to a plus sign.
pub fn url_decode(s: &str) -> String {
form_urlencoded::parse(s.as_bytes())
url::form_urlencoded::parse(s.as_bytes())
.map(|(k, _)| k)
.collect()
}
@ -62,8 +56,8 @@ pub fn url_encode(s: &str) -> String {
}
/// Encodes data using base64 algorithm
pub fn b64encode(input: impl AsRef<[u8]>) -> String {
BASE64.encode(input)
pub fn b64_encode(input: impl AsRef<[u8]>) -> String {
base64::engine::general_purpose::STANDARD.encode(input)
}
/// Computes CRC32 of given data.
@ -73,12 +67,17 @@ pub fn crc32(data: &[u8]) -> u32 {
}
/// Converts data array into 32 bit BigEndian unsigned int
pub fn uint32(mut data: &[u8]) -> Result<u32, ValidationErr> {
data.read_u32::<BigEndian>()
.map_err(|e| ValidationErr::InvalidIntegerValue {
pub fn uint32(data: &[u8]) -> Result<u32, ValidationErr> {
if data.len() < 4 {
return Err(ValidationErr::InvalidIntegerValue {
message: "data is not a valid 32-bit BigEndian unsigned integer".into(),
source: Box::new(e),
})
source: Box::new(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"not enough bytes",
)),
});
}
Ok(u32::from_be_bytes(data[..4].try_into().unwrap()))
}
/// sha256 hash of empty data
@ -88,32 +87,84 @@ pub const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934c
pub fn sha256_hash(data: &[u8]) -> String {
#[cfg(feature = "ring")]
{
ring::digest::digest(&SHA256, data).encode_hex()
hex_encode(ring::digest::digest(&SHA256, data).as_ref())
}
#[cfg(not(feature = "ring"))]
{
Sha256::new_with_prefix(data).finalize().encode_hex()
hex_encode(Sha256::new_with_prefix(data).finalize().as_slice())
}
}
/// Hex-encode a byte slice into a lowercase ASCII string.
///
/// # Safety
/// This implementation uses `unsafe` code for performance reasons:
/// - We call [`String::as_mut_vec`] to get direct access to the
/// underlying `Vec<u8>` backing the `String`.
/// - We then use [`set_len`] to pre-allocate the final length without
/// initializing the contents first.
/// - Finally, we use [`get_unchecked`] and [`get_unchecked_mut`] to
/// avoid bounds checking inside the tight encoding loop.
///
/// # Why unsafe is needed
/// Normally, writing this function with safe Rust requires:
/// - Pushing each hex digit one-by-one into the string (extra bounds checks).
/// - Or allocating and copying temporary buffers.
///
/// Using `unsafe` avoids redundant checks and makes this implementation
/// significantly faster, especially for large inputs.
///
/// # Why this is correct
/// - `s` is allocated with exactly `len * 2` capacity, and we immediately
/// set its length to that value. Every byte in the string buffer will be
/// initialized before being read or used.
/// - The loop index `i` is always in `0..len`, so `bytes.get_unchecked(i)`
/// is safe.
/// - Each write goes to positions `j` and `j + 1`, where `j = i * 2`.
/// Since `i < len`, the maximum write index is `2*len - 1`, which is
/// within the allocated range.
/// - All written bytes come from the `LUT` table, which has exactly 16
/// elements, and indices are masked into the 015 range.
///
/// Therefore, although `unsafe` is used to skip bounds checking,
/// the logic ensures all memory accesses remain in-bounds and initialized.
pub fn hex_encode(bytes: &[u8]) -> String {
const LUT: &[u8; 16] = b"0123456789abcdef";
let len = bytes.len();
let mut s = String::with_capacity(len * 2);
unsafe {
let v = s.as_mut_vec();
v.set_len(len * 2);
for i in 0..len {
let b = bytes.get_unchecked(i);
let hi = LUT.get_unchecked((b >> 4) as usize);
let lo = LUT.get_unchecked((b & 0xF) as usize);
let j = i * 2;
*v.get_unchecked_mut(j) = *hi;
*v.get_unchecked_mut(j + 1) = *lo;
}
}
s
}
pub fn sha256_hash_sb(sb: Arc<SegmentedBytes>) -> String {
#[cfg(feature = "ring")]
{
let mut context = Context::new(&SHA256);
for data in sb.iter() {
// Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap
context.update(data.as_ref());
}
context.finish().encode_hex()
hex_encode(context.finish().as_ref())
}
#[cfg(not(feature = "ring"))]
{
let mut hasher = Sha256::new();
for data in sb.iter() {
// Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap
hasher.update(data);
}
hasher.finalize().encode_hex()
hex_encode(hasher.finalize().as_slice())
}
}
@ -134,7 +185,7 @@ mod tests {
/// Gets bas64 encoded MD5 hash of given data
pub fn md5sum_hash(data: &[u8]) -> String {
b64encode(md5compute(data).as_slice())
b64_encode(md5::compute(data).as_slice())
}
/// Gets current UTC time
@ -480,7 +531,7 @@ fn escape(s: &str) -> String {
//
// Handles escaping same as MinIO server - needed for ensuring compatibility.
pub fn encode_tags(h: &HashMap<String, String>) -> String {
let mut tags = Vec::new();
let mut tags = Vec::with_capacity(h.len());
for (k, v) in h {
tags.push(format!("{}={}", escape(k), escape(v)));
}

View File

@ -12,3 +12,7 @@ cargo test -- --nocapture
# run one specific test and show stdout
# cargo test --test test_bucket_exists -- --nocapture
# run tests with ring instead of default-crypto
# cargo test --no-default-features --features "default-tls,ring" -- --nocapture

View File

@ -14,11 +14,11 @@
// limitations under the License.
use async_std::io::ReadExt;
use hex::ToHex;
use minio::s3::builders::ObjectContent;
use minio::s3::response::a_response_traits::{HasBucket, HasObject};
use minio::s3::response::{GetObjectResponse, PutObjectContentResponse};
use minio::s3::types::S3Api;
use minio::s3::utils::hex_encode;
use minio_common::rand_reader::RandReader;
use minio_common::test_context::TestContext;
use minio_common::utils::rand_object_name_utf8;
@ -36,7 +36,7 @@ async fn get_hash(filename: &str) -> String {
let mut buf = Vec::new();
file.read_to_end(&mut buf).await.unwrap();
context.update(&buf);
context.finish().encode_hex()
hex_encode(context.finish().as_ref())
}
#[cfg(not(feature = "ring"))]
{
@ -45,7 +45,7 @@ async fn get_hash(filename: &str) -> String {
let mut buf = Vec::new();
file.read_to_end(&mut buf).await.unwrap();
hasher.update(&buf);
hasher.finalize().encode_hex()
hex_encode(hasher.finalize().as_slice())
}
}