mirror of
https://github.com/minio/minio-rs.git
synced 2025-12-06 15:26:51 +08:00
Fixed clippy issues and cleanup of unnecessary imports (#182)
This commit is contained in:
parent
5080bf9b85
commit
25d424b97f
36
Cargo.toml
36
Cargo.toml
@ -10,11 +10,6 @@ readme = "README.md"
|
||||
keywords = ["object-storage", "minio", "s3"]
|
||||
categories = ["api-bindings", "web-programming::http-client"]
|
||||
|
||||
[dependencies.reqwest]
|
||||
version = "0.12.22"
|
||||
default-features = false
|
||||
features = ["stream"]
|
||||
|
||||
[features]
|
||||
default = ["default-tls", "default-crypto"]
|
||||
default-tls = ["reqwest/default-tls"]
|
||||
@ -23,21 +18,29 @@ rustls-tls = ["reqwest/rustls-tls"]
|
||||
default-crypto = ["dep:sha2", "dep:hmac"]
|
||||
ring = ["dep:ring"]
|
||||
|
||||
[workspace.dependencies]
|
||||
uuid = "1.18"
|
||||
futures-util = "0.3"
|
||||
reqwest = { version = "0.12", default-features = false }
|
||||
bytes = "1.10"
|
||||
async-std = "1.13"
|
||||
|
||||
|
||||
[dependencies]
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
futures-util = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
async-std = { workspace = true, features = ["attributes"] }
|
||||
reqwest = { workspace = true, features = ["stream"] }
|
||||
|
||||
async-recursion = "1.1.1"
|
||||
async-std = { version = "1.13.1", features = ["attributes"] }
|
||||
async-stream = "0.3.6"
|
||||
async-trait = "0.1.88"
|
||||
base64 = "0.22.1"
|
||||
byteorder = "1.5.0"
|
||||
bytes = "1.10.1"
|
||||
chrono = "0.4.41"
|
||||
crc = "3.3.0"
|
||||
dashmap = "6.1.0"
|
||||
derivative = "2.2.0"
|
||||
env_logger = "0.11.8"
|
||||
futures-util = "0.3.31"
|
||||
hex = "0.4.3"
|
||||
hmac = { version = "0.12.1", optional = true }
|
||||
hyper = { version = "1.6.0", features = ["full"] }
|
||||
lazy_static = "1.5.0"
|
||||
@ -46,26 +49,25 @@ md5 = "0.8.0"
|
||||
multimap = "0.10.1"
|
||||
percent-encoding = "2.3.1"
|
||||
url = "2.5.4"
|
||||
rand = { version = "0.8.5", features = ["small_rng"] }
|
||||
regex = "1.11.1"
|
||||
ring = { version = "0.17.14", optional = true, default-features = false, features = ["alloc"] }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_json = "1.0.140"
|
||||
serde_json = "1.0.142"
|
||||
sha2 = { version = "0.10.9", optional = true }
|
||||
urlencoding = "2.1.3"
|
||||
xmltree = "0.11.0"
|
||||
futures = "0.3.31"
|
||||
http = "1.3.1"
|
||||
thiserror = "2.0.12"
|
||||
thiserror = "2.0.14"
|
||||
|
||||
[dev-dependencies]
|
||||
minio-common = { path = "./common" }
|
||||
minio-macros = { path = "./macros" }
|
||||
tokio = { version = "1.47.1", features = ["full"] }
|
||||
minio_common = { path = "./common" }
|
||||
async-std = { version = "1.13.1", features = ["attributes", "tokio1"] }
|
||||
clap = { version = "4.5.44", features = ["derive"] }
|
||||
rand = { version = "0.9.2", features = ["small_rng"] }
|
||||
quickcheck = "1.0.3"
|
||||
criterion = "0.7.0"
|
||||
minio-macros = { path = "./macros" }
|
||||
|
||||
[lib]
|
||||
name = "minio"
|
||||
|
||||
@ -1,20 +1,23 @@
|
||||
[package]
|
||||
name = "minio_common"
|
||||
name = "minio-common"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
minio = {path = ".." }
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
reqwest = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
async-std = { workspace = true }
|
||||
|
||||
futures-io = "0.3.31"
|
||||
tokio = { version = "1.47.1", features = ["full"] }
|
||||
async-std = "1.13.1"
|
||||
rand = { version = "0.8.5", features = ["small_rng"] }
|
||||
bytes = "1.10.1"
|
||||
rand = { version = "0.9.2", features = ["small_rng"] }
|
||||
|
||||
log = "0.4.27"
|
||||
chrono = "0.4.41"
|
||||
reqwest = "0.12.22"
|
||||
|
||||
http = "1.3.1"
|
||||
futures = "0.3.31"
|
||||
uuid = { version = "1.18.0", features = ["v4"] }
|
||||
|
||||
[lib]
|
||||
name = "minio_common"
|
||||
|
||||
@ -38,7 +38,7 @@ impl CleanupGuard {
|
||||
pub async fn cleanup(client: Client, bucket_name: &str) {
|
||||
tokio::select!(
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(60)) => {
|
||||
eprintln!("Cleanup timeout after 60s while removing bucket {}", bucket_name);
|
||||
eprintln!("Cleanup timeout after 60s while removing bucket {bucket_name}");
|
||||
},
|
||||
outcome = client.delete_and_purge_bucket(bucket_name) => {
|
||||
match outcome {
|
||||
@ -46,7 +46,7 @@ pub async fn cleanup(client: Client, bucket_name: &str) {
|
||||
//eprintln!("Bucket {} removed successfully", bucket_name);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error removing bucket '{}':\n{}", bucket_name, e);
|
||||
eprintln!("Error removing bucket '{bucket_name}':\n{e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -140,7 +140,7 @@ pub fn create_bucket_replication_config_example(dst_bucket: &str) -> Replication
|
||||
rules: vec![ReplicationRule {
|
||||
id: Some(String::from("rule1")),
|
||||
destination: Destination {
|
||||
bucket_arn: String::from(&format!("arn:aws:s3:::{}", dst_bucket)),
|
||||
bucket_arn: String::from(&format!("arn:aws:s3:::{dst_bucket}")),
|
||||
..Default::default()
|
||||
},
|
||||
filter: Some(Filter {
|
||||
@ -170,8 +170,8 @@ pub fn create_object_lock_config_example() -> ObjectLockConfig {
|
||||
pub fn create_post_policy_example(bucket_name: &str, object_name: &str) -> PostPolicy {
|
||||
let expiration: DateTime<Utc> = utc_now() + chrono::Duration::days(5);
|
||||
|
||||
let mut policy = PostPolicy::new(&bucket_name, expiration).unwrap();
|
||||
policy.add_equals_condition("key", &object_name).unwrap();
|
||||
let mut policy = PostPolicy::new(bucket_name, expiration).unwrap();
|
||||
policy.add_equals_condition("key", object_name).unwrap();
|
||||
policy
|
||||
.add_content_length_range_condition(1024 * 1024, 4 * 1024 * 1024)
|
||||
.unwrap();
|
||||
@ -189,7 +189,7 @@ pub fn create_select_content_data() -> (String, String) {
|
||||
(body, data)
|
||||
}
|
||||
pub fn create_select_content_request() -> SelectRequest {
|
||||
let request = SelectRequest::new_csv_input_output(
|
||||
SelectRequest::new_csv_input_output(
|
||||
"select * from S3Object",
|
||||
CsvInputSerialization {
|
||||
compression_type: None,
|
||||
@ -209,6 +209,5 @@ pub fn create_select_content_request() -> SelectRequest {
|
||||
record_delimiter: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
request
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use futures::AsyncRead;
|
||||
use futures_io::AsyncRead;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
@ -34,7 +34,7 @@ impl io::Read for RandReader {
|
||||
let bytes_read = buf.len().min(self.size as usize);
|
||||
|
||||
if bytes_read > 0 {
|
||||
let random: &mut dyn rand::RngCore = &mut rand::thread_rng();
|
||||
let random: &mut dyn rand::RngCore = &mut rand::rng();
|
||||
random.fill_bytes(&mut buf[0..bytes_read]);
|
||||
}
|
||||
|
||||
@ -53,7 +53,7 @@ impl AsyncRead for RandReader {
|
||||
let bytes_read = buf.len().min(self.size as usize);
|
||||
|
||||
if bytes_read > 0 {
|
||||
let random: &mut dyn rand::RngCore = &mut rand::thread_rng();
|
||||
let random: &mut dyn rand::RngCore = &mut rand::rng();
|
||||
random.fill_bytes(&mut buf[0..bytes_read]);
|
||||
}
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use async_std::stream::Stream;
|
||||
use bytes::Bytes;
|
||||
use futures::io::AsyncRead;
|
||||
use futures_io::AsyncRead;
|
||||
use rand::prelude::SmallRng;
|
||||
use rand::{RngCore, SeedableRng};
|
||||
use std::io;
|
||||
@ -30,7 +30,7 @@ pub struct RandSrc {
|
||||
impl RandSrc {
|
||||
#[allow(dead_code)]
|
||||
pub fn new(size: u64) -> RandSrc {
|
||||
let rng = SmallRng::from_entropy();
|
||||
let rng: SmallRng = SmallRng::from_os_rng();
|
||||
RandSrc { size, rng }
|
||||
}
|
||||
}
|
||||
|
||||
@ -84,10 +84,10 @@ impl TestContext {
|
||||
|
||||
let host: String =
|
||||
std::env::var("SERVER_ENDPOINT").unwrap_or(DEFAULT_SERVER_ENDPOINT.to_string());
|
||||
log::debug!("SERVER_ENDPOINT={}", host);
|
||||
log::debug!("SERVER_ENDPOINT={host}");
|
||||
let access_key: String =
|
||||
std::env::var("ACCESS_KEY").unwrap_or(DEFAULT_ACCESS_KEY.to_string());
|
||||
log::debug!("ACCESS_KEY={}", access_key);
|
||||
log::debug!("ACCESS_KEY={access_key}");
|
||||
let secret_key: String =
|
||||
std::env::var("SECRET_KEY").unwrap_or(DEFAULT_SECRET_KEY.to_string());
|
||||
log::debug!("SECRET_KEY=*****");
|
||||
@ -95,19 +95,19 @@ impl TestContext {
|
||||
.unwrap_or(DEFAULT_ENABLE_HTTPS.to_string())
|
||||
.parse()
|
||||
.unwrap_or(false);
|
||||
log::debug!("ENABLE_HTTPS={}", secure);
|
||||
log::debug!("ENABLE_HTTPS={secure}");
|
||||
let ssl_cert: String =
|
||||
std::env::var("MINIO_SSL_CERT_FILE").unwrap_or(DEFAULT_SSL_CERT_FILE.to_string());
|
||||
log::debug!("MINIO_SSL_CERT_FILE={}", ssl_cert);
|
||||
log::debug!("MINIO_SSL_CERT_FILE={ssl_cert}");
|
||||
let ssl_cert_file: PathBuf = ssl_cert.into();
|
||||
let ignore_cert_check: bool = std::env::var("IGNORE_CERT_CHECK")
|
||||
.unwrap_or(DEFAULT_IGNORE_CERT_CHECK.to_string())
|
||||
.parse()
|
||||
.unwrap_or(true);
|
||||
log::debug!("IGNORE_CERT_CHECK={}", ignore_cert_check);
|
||||
log::debug!("IGNORE_CERT_CHECK={ignore_cert_check}");
|
||||
let region: String =
|
||||
std::env::var("SERVER_REGION").unwrap_or(DEFAULT_SERVER_REGION.to_string());
|
||||
log::debug!("SERVER_REGION={:?}", region);
|
||||
log::debug!("SERVER_REGION={region:?}");
|
||||
|
||||
let mut base_url: BaseUrl = host.parse().unwrap();
|
||||
base_url.https = secure;
|
||||
|
||||
@ -15,8 +15,9 @@
|
||||
|
||||
use http::{Response as HttpResponse, StatusCode};
|
||||
use minio::s3::error::Error;
|
||||
use rand::distributions::Standard;
|
||||
use rand::{Rng, thread_rng};
|
||||
|
||||
use rand::Rng;
|
||||
use rand::distr::StandardUniform;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn rand_bucket_name() -> String {
|
||||
@ -28,9 +29,9 @@ pub fn rand_object_name() -> String {
|
||||
}
|
||||
|
||||
pub fn rand_object_name_utf8(len: usize) -> String {
|
||||
let rng = thread_rng();
|
||||
rng.sample_iter::<char, _>(Standard)
|
||||
.filter(|c| !c.is_control())
|
||||
let rng = rand::rng();
|
||||
rng.sample_iter(StandardUniform)
|
||||
.filter(|c: &char| !c.is_control())
|
||||
.take(len)
|
||||
.collect()
|
||||
}
|
||||
@ -39,9 +40,9 @@ pub async fn get_bytes_from_response(v: Result<reqwest::Response, Error>) -> byt
|
||||
match v {
|
||||
Ok(r) => match r.bytes().await {
|
||||
Ok(b) => b,
|
||||
Err(e) => panic!("{:?}", e),
|
||||
Err(e) => panic!("{e:?}"),
|
||||
},
|
||||
Err(e) => panic!("{:?}", e),
|
||||
Err(e) => panic!("{e:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@ -52,5 +53,5 @@ pub fn get_response_from_bytes(bytes: bytes::Bytes) -> reqwest::Response {
|
||||
.body(bytes)
|
||||
.expect("Failed to build HTTP response");
|
||||
|
||||
reqwest::Response::try_from(http_response).expect("Failed to convert to reqwest::Response")
|
||||
reqwest::Response::from(http_response)
|
||||
}
|
||||
|
||||
@ -22,7 +22,7 @@ use minio::s3::response::{AppendObjectResponse, StatObjectResponse};
|
||||
use minio::s3::segmented_bytes::SegmentedBytes;
|
||||
use minio::s3::types::S3Api;
|
||||
use rand::Rng;
|
||||
use rand::distributions::Alphanumeric;
|
||||
use rand::distr::Alphanumeric;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@ -78,7 +78,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
}
|
||||
|
||||
fn random_string(len: usize) -> String {
|
||||
rand::thread_rng()
|
||||
rand::rng()
|
||||
.sample_iter(&Alphanumeric)
|
||||
.take(len)
|
||||
.map(char::from)
|
||||
|
||||
@ -3,18 +3,19 @@ name = "minio-macros"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[lib]
|
||||
proc-macro = true
|
||||
|
||||
|
||||
[dependencies]
|
||||
uuid = { workspace = true, features = ["v4"] }
|
||||
futures-util = { workspace = true }
|
||||
|
||||
syn = "2.0.104"
|
||||
proc-macro2 = "1.0.95"
|
||||
proc-macro2 = "1.0.97"
|
||||
quote = "1.0.40"
|
||||
darling = "0.21.0"
|
||||
darling_core = "0.21.0"
|
||||
uuid = { version = "1.17.0", features = ["v4"] }
|
||||
|
||||
[dev-dependencies]
|
||||
minio_common = { path = "../common" }
|
||||
minio-common = { path = "../common" }
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[lib]
|
||||
proc-macro = true
|
||||
|
||||
@ -104,6 +104,6 @@ pub fn test(
|
||||
// Expand the macro
|
||||
match test_attr::expand_test_macro(args, input_fn) {
|
||||
Ok(expanded) => expanded.into(),
|
||||
Err(err) => err.into(),
|
||||
Err(err) => err,
|
||||
}
|
||||
}
|
||||
|
||||
@ -55,7 +55,7 @@ impl MacroArgs {
|
||||
}
|
||||
|
||||
// Validate that the function has exactly two arguments: ctx and bucket_name
|
||||
if func.sig.inputs.len() != 2 && !self.no_bucket.is_present() {
|
||||
if (func.sig.inputs.len() != 2) && !self.no_bucket.is_present() {
|
||||
let error_msg = "Minio test function must have exactly two arguments: (ctx: TestContext, bucket_name: String)";
|
||||
return Err(proc_macro::TokenStream::from(
|
||||
Error::custom(error_msg)
|
||||
@ -80,9 +80,10 @@ impl MacroArgs {
|
||||
}
|
||||
}
|
||||
|
||||
// Check second argument (bucket_name: String)
|
||||
if !self.no_bucket.is_present() {
|
||||
if let Some(FnArg::Typed(pat_type)) = iter.next() {
|
||||
// Check the second argument (bucket_name: String)
|
||||
if !self.no_bucket.is_present()
|
||||
&& let Some(FnArg::Typed(pat_type)) = iter.next()
|
||||
{
|
||||
let type_str = pat_type.ty.to_token_stream().to_string();
|
||||
if !type_str.contains("String") {
|
||||
let error_msg = "Second argument must be of type String";
|
||||
@ -93,7 +94,6 @@ impl MacroArgs {
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@ -124,7 +124,7 @@ pub(crate) fn expand_test_macro(
|
||||
|
||||
// Setup common prelude
|
||||
let prelude = quote!(
|
||||
use ::futures::FutureExt;
|
||||
use ::futures_util::FutureExt;
|
||||
use ::std::panic::AssertUnwindSafe;
|
||||
use ::minio::s3::types::S3Api;
|
||||
use ::minio::s3::response::a_response_traits::HasBucket;
|
||||
|
||||
@ -20,7 +20,7 @@ use crate::s3::builders::{
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::error::{Error, IoError};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::a_response_traits::HasObjectSize;
|
||||
use crate::s3::response::{AppendObjectResponse, StatObjectResponse};
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use std::marker::PhantomData;
|
||||
|
||||
/// Common parameters for bucket operations
|
||||
|
||||
@ -17,7 +17,7 @@ use crate::s3::Client;
|
||||
use crate::s3::client::{MAX_MULTIPART_COUNT, MAX_PART_SIZE};
|
||||
use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::a_response_traits::HasEtagFromBody;
|
||||
use crate::s3::response::{
|
||||
AbortMultipartUploadResponse, CompleteMultipartUploadResponse, ComposeObjectResponse,
|
||||
|
||||
@ -17,7 +17,7 @@ use crate::s3::Client;
|
||||
use crate::s3::client::DEFAULT_REGION;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::CreateBucketResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::DeleteObjectTaggingResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, check_object_name, insert};
|
||||
|
||||
@ -17,7 +17,7 @@ use crate::s3::Client;
|
||||
use crate::s3::client::MAX_MULTIPART_COUNT;
|
||||
use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::{DeleteError, DeleteObjectResponse, DeleteObjectsResponse};
|
||||
use crate::s3::types::{ListEntry, S3Api, S3Request, ToS3Request, ToStream};
|
||||
use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetBucketLifecycleResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, insert};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::GetBucketTaggingResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, insert};
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetObjectResponse;
|
||||
use crate::s3::sse::{Sse, SseCustomerKey};
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetObjectLegalHoldResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, check_object_name, insert};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetObjectPromptResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::sse::SseCustomerKey;
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetObjectRetentionResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, check_object_name, insert};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetObjectTaggingResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, check_object_name, insert};
|
||||
|
||||
@ -17,7 +17,7 @@ use crate::s3::Client;
|
||||
use crate::s3::creds::Credentials;
|
||||
use crate::s3::error::Error;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::GetPresignedObjectUrlResponse;
|
||||
use crate::s3::signer::presign_v4;
|
||||
use crate::s3::utils::{UtcTime, check_bucket_name, check_object_name, utc_now};
|
||||
|
||||
@ -19,7 +19,7 @@ use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::signer::post_presign_v4;
|
||||
use crate::s3::utils::{
|
||||
UtcTime, b64encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now,
|
||||
UtcTime, b64_encode, check_bucket_name, to_amz_date, to_iso8601utc, to_signer_date, utc_now,
|
||||
};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
@ -341,7 +341,7 @@ impl PostPolicy {
|
||||
"conditions": conditions,
|
||||
});
|
||||
|
||||
let encoded_policy = b64encode(policy.to_string());
|
||||
let encoded_policy = b64_encode(policy.to_string());
|
||||
let signature = post_presign_v4(&encoded_policy, &secret_key, date, ®ion);
|
||||
|
||||
let mut data: HashMap<String, String> = HashMap::new();
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
use crate::s3::Client;
|
||||
use crate::s3::client::DEFAULT_REGION;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::GetRegionResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, insert};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::ListBucketsResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use http::Method;
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::ListObjectsResponse;
|
||||
use crate::s3::response::list_objects::{
|
||||
ListObjectVersionsResponse, ListObjectsV1Response, ListObjectsV2Response,
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::ListenBucketNotificationResponse;
|
||||
use crate::s3::types::{NotificationRecords, S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::check_bucket_name;
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutBucketEncryptionResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{S3Api, S3Request, SseConfig, ToS3Request};
|
||||
|
||||
@ -17,7 +17,7 @@ use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::lifecycle_config::LifecycleConfig;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::PutBucketLifecycleResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, insert, md5sum_hash};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutBucketNotificationResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{NotificationConfig, S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutBucketPolicyResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutBucketReplicationResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{ReplicationConfig, S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutBucketTaggingResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutBucketVersioningResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -18,7 +18,7 @@ use crate::s3::builders::{ContentStream, Size};
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::error::{Error, IoError, ValidationErr};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::a_response_traits::HasEtagFromHeaders;
|
||||
use crate::s3::response::{
|
||||
AbortMultipartUploadResponse, CompleteMultipartUploadResponse, CreateMultipartUploadResponse,
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::PutObjectLegalHoldResponse;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{check_bucket_name, check_object_name, insert, md5sum_hash};
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::response::PutObjectLockConfigResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{ObjectLockConfig, S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::PutObjectRetentionResponse;
|
||||
use crate::s3::types::{RetentionMode, S3Api, S3Request, ToS3Request};
|
||||
use crate::s3::utils::{
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::PutObjectTaggingResponse;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::SelectObjectContentResponse;
|
||||
use crate::s3::sse::SseCustomerKey;
|
||||
use crate::s3::types::{S3Api, S3Request, SelectRequest, ToS3Request};
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
use crate::s3::client::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::StatObjectResponse;
|
||||
use crate::s3::sse::{Sse, SseCustomerKey};
|
||||
use crate::s3::types::{S3Api, S3Request, ToS3Request};
|
||||
|
||||
@ -15,32 +15,31 @@
|
||||
|
||||
//! S3 client to perform bucket and object operations
|
||||
|
||||
use bytes::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use http::HeaderMap;
|
||||
use hyper::http::Method;
|
||||
use reqwest::{Body, Response};
|
||||
use std::fs::File;
|
||||
use std::io::prelude::*;
|
||||
use std::mem;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::s3::builders::{BucketExists, ComposeSource};
|
||||
use crate::s3::creds::Provider;
|
||||
use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::http::BaseUrl;
|
||||
use crate::s3::minio_error_response::{MinioErrorCode, MinioErrorResponse};
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::a_response_traits::{HasEtagFromHeaders, HasS3Fields};
|
||||
use crate::s3::response::*;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::signer::sign_v4_s3;
|
||||
use crate::s3::utils::{EMPTY_SHA256, check_ssec_with_log, sha256_hash_sb, to_amz_date, utc_now};
|
||||
|
||||
use crate::s3::error::{Error, IoError, NetworkError, S3ServerError, ValidationErr};
|
||||
use bytes::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use http::HeaderMap;
|
||||
use hyper::http::Method;
|
||||
use rand::Rng;
|
||||
use reqwest::{Body, Response};
|
||||
|
||||
mod append_object;
|
||||
mod bucket_exists;
|
||||
mod copy_object;
|
||||
@ -276,12 +275,7 @@ impl Client {
|
||||
*val
|
||||
} else {
|
||||
// Create a random bucket name
|
||||
let bucket_name: String = rand::thread_rng()
|
||||
.sample_iter(&rand::distributions::Alphanumeric)
|
||||
.take(20)
|
||||
.map(char::from)
|
||||
.collect::<String>()
|
||||
.to_lowercase();
|
||||
let bucket_name: String = Uuid::new_v4().to_string();
|
||||
|
||||
let express = match BucketExists::new(self.clone(), bucket_name).send().await {
|
||||
Ok(v) => {
|
||||
|
||||
@ -24,7 +24,7 @@ use crate::s3::response::{
|
||||
};
|
||||
use crate::s3::types::{S3Api, ToStream};
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use futures_util::StreamExt;
|
||||
|
||||
impl Client {
|
||||
/// Creates a [`DeleteBucket`] request builder.
|
||||
|
||||
@ -19,9 +19,8 @@ use super::utils::urlencode_object_key;
|
||||
use crate::s3::client::DEFAULT_REGION;
|
||||
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::utils::match_hostname;
|
||||
use derivative::Derivative;
|
||||
use hyper::Uri;
|
||||
use hyper::http::Method;
|
||||
use lazy_static::lazy_static;
|
||||
@ -37,11 +36,9 @@ lazy_static! {
|
||||
static ref AWS_S3_PREFIX_REGEX: Regex = Regex::new(AWS_S3_PREFIX).unwrap();
|
||||
}
|
||||
|
||||
#[derive(Derivative)]
|
||||
#[derivative(Clone, Debug, Default)]
|
||||
#[derive(Clone, Debug)]
|
||||
/// Represents HTTP URL
|
||||
pub struct Url {
|
||||
#[derivative(Default(value = "true"))]
|
||||
pub https: bool,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
@ -58,6 +55,18 @@ impl Url {
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Url {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
https: true,
|
||||
host: String::default(),
|
||||
port: u16::default(),
|
||||
path: String::default(),
|
||||
query: Multimap::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Url {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
if self.host.is_empty() {
|
||||
@ -208,11 +217,9 @@ fn get_aws_info(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Derivative)]
|
||||
#[derivative(Clone, Debug, Default)]
|
||||
#[derive(Clone, Debug)]
|
||||
/// Represents Base URL of S3 endpoint
|
||||
pub struct BaseUrl {
|
||||
#[derivative(Default(value = "true"))]
|
||||
pub https: bool,
|
||||
host: String,
|
||||
port: u16,
|
||||
@ -223,6 +230,21 @@ pub struct BaseUrl {
|
||||
pub virtual_style: bool,
|
||||
}
|
||||
|
||||
impl Default for BaseUrl {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
https: true,
|
||||
host: "127.0.0.1".to_string(),
|
||||
port: 9000,
|
||||
region: "".to_string(),
|
||||
aws_s3_prefix: "".to_string(),
|
||||
aws_domain_suffix: "".to_string(),
|
||||
dualstack: false,
|
||||
virtual_style: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for BaseUrl {
|
||||
type Err = ValidationErr;
|
||||
|
||||
|
||||
@ -23,7 +23,7 @@ pub mod header_constants;
|
||||
pub mod http;
|
||||
pub mod lifecycle_config;
|
||||
pub mod minio_error_response;
|
||||
pub mod multimap;
|
||||
pub mod multimap_ext;
|
||||
mod object_content;
|
||||
pub mod response;
|
||||
pub mod segmented_bytes;
|
||||
|
||||
@ -49,9 +49,7 @@ impl MultimapExt for Multimap {
|
||||
}
|
||||
fn add_multimap(&mut self, other: Multimap) {
|
||||
for (key, values) in other.into_iter() {
|
||||
for value in values {
|
||||
self.insert(key.clone(), value);
|
||||
}
|
||||
self.insert_many(key.clone(), values);
|
||||
}
|
||||
}
|
||||
fn add_version(&mut self, version: Option<String>) {
|
||||
@ -13,14 +13,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use async_std::io::{ReadExt, WriteExt};
|
||||
use bytes::Bytes;
|
||||
use futures::stream::{self, Stream, StreamExt};
|
||||
use rand::prelude::random;
|
||||
use futures_util::stream::{self, Stream, StreamExt};
|
||||
use std::path::PathBuf;
|
||||
use std::{ffi::OsString, fs, path::Path, pin::Pin};
|
||||
use std::{fs, path::Path, pin::Pin};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
#[cfg(test)]
|
||||
use quickcheck::Arbitrary;
|
||||
|
||||
@ -216,13 +216,11 @@ impl ObjectContent {
|
||||
async_std::fs::create_dir_all(parent_dir).await?;
|
||||
}
|
||||
let file_name = file_path.file_name().ok_or(std::io::Error::other(
|
||||
"could not get filename component of path",
|
||||
"could not get filename-component of path",
|
||||
))?;
|
||||
let mut tmp_file_name: OsString = file_name.to_os_string();
|
||||
tmp_file_name.push(format!("_{}", random::<u64>()));
|
||||
let tmp_file_path = parent_dir
|
||||
.to_path_buf()
|
||||
.join(Path::new(tmp_file_name.as_os_str()));
|
||||
let mut tmp_file_name = file_name.to_os_string();
|
||||
tmp_file_name.push(format!("_{}", Uuid::new_v4().to_string().replace('-', "_")));
|
||||
let tmp_file_path = parent_dir.join(tmp_file_name);
|
||||
|
||||
let mut total_bytes_written = 0;
|
||||
let mut fp = async_std::fs::OpenOptions::new()
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
use crate::impl_has_s3fields;
|
||||
use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::response::a_response_traits::{HasBucket, HasObject, HasRegion, HasS3Fields};
|
||||
use crate::s3::types::{FromS3Response, S3Request, SelectProgress};
|
||||
use crate::s3::utils::{copy_slice, crc32, get_text_result, uint32};
|
||||
|
||||
@ -16,9 +16,8 @@
|
||||
//! Signature V4 for S3 API
|
||||
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::utils::{UtcTime, sha256_hash, to_amz_date, to_signer_date};
|
||||
use hex::encode as hexencode;
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::utils::{UtcTime, hex_encode, sha256_hash, to_amz_date, to_signer_date};
|
||||
#[cfg(not(feature = "ring"))]
|
||||
use hmac::{Hmac, Mac};
|
||||
use hyper::http::Method;
|
||||
@ -45,16 +44,14 @@ fn hmac_hash(key: &[u8], data: &[u8]) -> Vec<u8> {
|
||||
|
||||
/// Returns hex encoded HMAC hash for given key and data
|
||||
fn hmac_hash_hex(key: &[u8], data: &[u8]) -> String {
|
||||
hexencode(hmac_hash(key, data))
|
||||
hex_encode(hmac_hash(key, data).as_slice())
|
||||
}
|
||||
|
||||
/// Returns scope value of given date, region and service name
|
||||
fn get_scope(date: UtcTime, region: &str, service_name: &str) -> String {
|
||||
format!(
|
||||
"{}/{}/{}/aws4_request",
|
||||
to_signer_date(date),
|
||||
region,
|
||||
service_name
|
||||
"{}/{region}/{service_name}/aws4_request",
|
||||
to_signer_date(date)
|
||||
)
|
||||
}
|
||||
|
||||
@ -76,10 +73,8 @@ fn get_canonical_request_hash(
|
||||
/// Returns string-to-sign value of given date, scope and canonical request hash
|
||||
fn get_string_to_sign(date: UtcTime, scope: &str, canonical_request_hash: &str) -> String {
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
|
||||
to_amz_date(date),
|
||||
scope,
|
||||
canonical_request_hash
|
||||
"AWS4-HMAC-SHA256\n{}\n{scope}\n{canonical_request_hash}",
|
||||
to_amz_date(date)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@ -16,8 +16,8 @@
|
||||
//! Server side encryption definitions
|
||||
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::{Multimap, MultimapExt};
|
||||
use crate::s3::utils::{b64encode, md5sum_hash};
|
||||
use crate::s3::multimap_ext::{Multimap, MultimapExt};
|
||||
use crate::s3::utils::{b64_encode, md5sum_hash};
|
||||
use std::any::Any;
|
||||
|
||||
/// Base server side encryption
|
||||
@ -39,7 +39,7 @@ pub struct SseCustomerKey {
|
||||
|
||||
impl SseCustomerKey {
|
||||
pub fn new(key: &str) -> Self {
|
||||
let b64key: String = b64encode(key);
|
||||
let b64key: String = b64_encode(key);
|
||||
let md5key: String = md5sum_hash(key.as_bytes());
|
||||
|
||||
let mut headers = Multimap::with_capacity(3);
|
||||
@ -102,7 +102,7 @@ impl SseKms {
|
||||
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_AWS_KMS_KEY_ID, key);
|
||||
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION, "aws:kms");
|
||||
if let Some(v) = context {
|
||||
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64encode(v));
|
||||
headers.add(X_AMZ_SERVER_SIDE_ENCRYPTION_CONTEXT, b64_encode(v));
|
||||
}
|
||||
|
||||
SseKms { headers }
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
use super::client::{Client, DEFAULT_REGION};
|
||||
use crate::s3::error::{Error, ValidationErr};
|
||||
use crate::s3::header_constants::*;
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::utils::{UtcTime, get_text_option, get_text_result};
|
||||
use async_trait::async_trait;
|
||||
|
||||
105
src/s3/utils.rs
105
src/s3/utils.rs
@ -13,18 +13,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Various utility and helper functions
|
||||
|
||||
use crate::s3::multimap::Multimap;
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::multimap_ext::Multimap;
|
||||
use crate::s3::segmented_bytes::SegmentedBytes;
|
||||
use crate::s3::sse::{Sse, SseCustomerKey};
|
||||
use base64::engine::Engine as _;
|
||||
use base64::engine::general_purpose::STANDARD as BASE64;
|
||||
use byteorder::{BigEndian, ReadBytesExt};
|
||||
use chrono::{DateTime, Datelike, NaiveDateTime, Utc};
|
||||
use crc::{CRC_32_ISO_HDLC, Crc};
|
||||
use hex::ToHex;
|
||||
use lazy_static::lazy_static;
|
||||
use md5::compute as md5compute;
|
||||
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_decode_str, utf8_percent_encode};
|
||||
use regex::Regex;
|
||||
#[cfg(feature = "ring")]
|
||||
@ -34,12 +31,9 @@ use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use xmltree::Element;
|
||||
|
||||
/// Date and time with UTC timezone
|
||||
pub type UtcTime = DateTime<Utc>;
|
||||
use crate::s3::Client;
|
||||
use crate::s3::error::ValidationErr;
|
||||
use crate::s3::sse::{Sse, SseCustomerKey};
|
||||
use url::form_urlencoded;
|
||||
|
||||
// Great stuff to get confused about.
|
||||
// String "a b+c" in Percent-Encoding (RFC 3986) becomes "a%20b%2Bc".
|
||||
@ -50,7 +44,7 @@ use url::form_urlencoded;
|
||||
/// Decodes a URL-encoded string in the application/x-www-form-urlencoded syntax into a string.
|
||||
/// Note that "+" is decoded to a space character, and "%2B" is decoded to a plus sign.
|
||||
pub fn url_decode(s: &str) -> String {
|
||||
form_urlencoded::parse(s.as_bytes())
|
||||
url::form_urlencoded::parse(s.as_bytes())
|
||||
.map(|(k, _)| k)
|
||||
.collect()
|
||||
}
|
||||
@ -62,8 +56,8 @@ pub fn url_encode(s: &str) -> String {
|
||||
}
|
||||
|
||||
/// Encodes data using base64 algorithm
|
||||
pub fn b64encode(input: impl AsRef<[u8]>) -> String {
|
||||
BASE64.encode(input)
|
||||
pub fn b64_encode(input: impl AsRef<[u8]>) -> String {
|
||||
base64::engine::general_purpose::STANDARD.encode(input)
|
||||
}
|
||||
|
||||
/// Computes CRC32 of given data.
|
||||
@ -73,12 +67,17 @@ pub fn crc32(data: &[u8]) -> u32 {
|
||||
}
|
||||
|
||||
/// Converts data array into 32 bit BigEndian unsigned int
|
||||
pub fn uint32(mut data: &[u8]) -> Result<u32, ValidationErr> {
|
||||
data.read_u32::<BigEndian>()
|
||||
.map_err(|e| ValidationErr::InvalidIntegerValue {
|
||||
pub fn uint32(data: &[u8]) -> Result<u32, ValidationErr> {
|
||||
if data.len() < 4 {
|
||||
return Err(ValidationErr::InvalidIntegerValue {
|
||||
message: "data is not a valid 32-bit BigEndian unsigned integer".into(),
|
||||
source: Box::new(e),
|
||||
})
|
||||
source: Box::new(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"not enough bytes",
|
||||
)),
|
||||
});
|
||||
}
|
||||
Ok(u32::from_be_bytes(data[..4].try_into().unwrap()))
|
||||
}
|
||||
|
||||
/// sha256 hash of empty data
|
||||
@ -88,32 +87,84 @@ pub const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934c
|
||||
pub fn sha256_hash(data: &[u8]) -> String {
|
||||
#[cfg(feature = "ring")]
|
||||
{
|
||||
ring::digest::digest(&SHA256, data).encode_hex()
|
||||
hex_encode(ring::digest::digest(&SHA256, data).as_ref())
|
||||
}
|
||||
#[cfg(not(feature = "ring"))]
|
||||
{
|
||||
Sha256::new_with_prefix(data).finalize().encode_hex()
|
||||
hex_encode(Sha256::new_with_prefix(data).finalize().as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
/// Hex-encode a byte slice into a lowercase ASCII string.
|
||||
///
|
||||
/// # Safety
|
||||
/// This implementation uses `unsafe` code for performance reasons:
|
||||
/// - We call [`String::as_mut_vec`] to get direct access to the
|
||||
/// underlying `Vec<u8>` backing the `String`.
|
||||
/// - We then use [`set_len`] to pre-allocate the final length without
|
||||
/// initializing the contents first.
|
||||
/// - Finally, we use [`get_unchecked`] and [`get_unchecked_mut`] to
|
||||
/// avoid bounds checking inside the tight encoding loop.
|
||||
///
|
||||
/// # Why unsafe is needed
|
||||
/// Normally, writing this function with safe Rust requires:
|
||||
/// - Pushing each hex digit one-by-one into the string (extra bounds checks).
|
||||
/// - Or allocating and copying temporary buffers.
|
||||
///
|
||||
/// Using `unsafe` avoids redundant checks and makes this implementation
|
||||
/// significantly faster, especially for large inputs.
|
||||
///
|
||||
/// # Why this is correct
|
||||
/// - `s` is allocated with exactly `len * 2` capacity, and we immediately
|
||||
/// set its length to that value. Every byte in the string buffer will be
|
||||
/// initialized before being read or used.
|
||||
/// - The loop index `i` is always in `0..len`, so `bytes.get_unchecked(i)`
|
||||
/// is safe.
|
||||
/// - Each write goes to positions `j` and `j + 1`, where `j = i * 2`.
|
||||
/// Since `i < len`, the maximum write index is `2*len - 1`, which is
|
||||
/// within the allocated range.
|
||||
/// - All written bytes come from the `LUT` table, which has exactly 16
|
||||
/// elements, and indices are masked into the 0–15 range.
|
||||
///
|
||||
/// Therefore, although `unsafe` is used to skip bounds checking,
|
||||
/// the logic ensures all memory accesses remain in-bounds and initialized.
|
||||
pub fn hex_encode(bytes: &[u8]) -> String {
|
||||
const LUT: &[u8; 16] = b"0123456789abcdef";
|
||||
let len = bytes.len();
|
||||
let mut s = String::with_capacity(len * 2);
|
||||
|
||||
unsafe {
|
||||
let v = s.as_mut_vec();
|
||||
v.set_len(len * 2);
|
||||
for i in 0..len {
|
||||
let b = bytes.get_unchecked(i);
|
||||
let hi = LUT.get_unchecked((b >> 4) as usize);
|
||||
let lo = LUT.get_unchecked((b & 0xF) as usize);
|
||||
let j = i * 2;
|
||||
*v.get_unchecked_mut(j) = *hi;
|
||||
*v.get_unchecked_mut(j + 1) = *lo;
|
||||
}
|
||||
}
|
||||
|
||||
s
|
||||
}
|
||||
|
||||
pub fn sha256_hash_sb(sb: Arc<SegmentedBytes>) -> String {
|
||||
#[cfg(feature = "ring")]
|
||||
{
|
||||
let mut context = Context::new(&SHA256);
|
||||
for data in sb.iter() {
|
||||
// Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap
|
||||
context.update(data.as_ref());
|
||||
}
|
||||
context.finish().encode_hex()
|
||||
hex_encode(context.finish().as_ref())
|
||||
}
|
||||
#[cfg(not(feature = "ring"))]
|
||||
{
|
||||
let mut hasher = Sha256::new();
|
||||
for data in sb.iter() {
|
||||
// Note: &SegmentedBytes.iter yields clones of Bytes, but those clones are cheap
|
||||
hasher.update(data);
|
||||
}
|
||||
hasher.finalize().encode_hex()
|
||||
hex_encode(hasher.finalize().as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
@ -134,7 +185,7 @@ mod tests {
|
||||
|
||||
/// Gets bas64 encoded MD5 hash of given data
|
||||
pub fn md5sum_hash(data: &[u8]) -> String {
|
||||
b64encode(md5compute(data).as_slice())
|
||||
b64_encode(md5::compute(data).as_slice())
|
||||
}
|
||||
|
||||
/// Gets current UTC time
|
||||
@ -480,7 +531,7 @@ fn escape(s: &str) -> String {
|
||||
//
|
||||
// Handles escaping same as MinIO server - needed for ensuring compatibility.
|
||||
pub fn encode_tags(h: &HashMap<String, String>) -> String {
|
||||
let mut tags = Vec::new();
|
||||
let mut tags = Vec::with_capacity(h.len());
|
||||
for (k, v) in h {
|
||||
tags.push(format!("{}={}", escape(k), escape(v)));
|
||||
}
|
||||
|
||||
@ -12,3 +12,7 @@ cargo test -- --nocapture
|
||||
|
||||
# run one specific test and show stdout
|
||||
# cargo test --test test_bucket_exists -- --nocapture
|
||||
|
||||
# run tests with ring instead of default-crypto
|
||||
# cargo test --no-default-features --features "default-tls,ring" -- --nocapture
|
||||
|
||||
|
||||
@ -14,11 +14,11 @@
|
||||
// limitations under the License.
|
||||
|
||||
use async_std::io::ReadExt;
|
||||
use hex::ToHex;
|
||||
use minio::s3::builders::ObjectContent;
|
||||
use minio::s3::response::a_response_traits::{HasBucket, HasObject};
|
||||
use minio::s3::response::{GetObjectResponse, PutObjectContentResponse};
|
||||
use minio::s3::types::S3Api;
|
||||
use minio::s3::utils::hex_encode;
|
||||
use minio_common::rand_reader::RandReader;
|
||||
use minio_common::test_context::TestContext;
|
||||
use minio_common::utils::rand_object_name_utf8;
|
||||
@ -36,7 +36,7 @@ async fn get_hash(filename: &str) -> String {
|
||||
let mut buf = Vec::new();
|
||||
file.read_to_end(&mut buf).await.unwrap();
|
||||
context.update(&buf);
|
||||
context.finish().encode_hex()
|
||||
hex_encode(context.finish().as_ref())
|
||||
}
|
||||
#[cfg(not(feature = "ring"))]
|
||||
{
|
||||
@ -45,7 +45,7 @@ async fn get_hash(filename: &str) -> String {
|
||||
let mut buf = Vec::new();
|
||||
file.read_to_end(&mut buf).await.unwrap();
|
||||
hasher.update(&buf);
|
||||
hasher.finalize().encode_hex()
|
||||
hex_encode(hasher.finalize().as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user