From 0fb80e1456dcb9f73a7b31437c35647966f247fe Mon Sep 17 00:00:00 2001 From: Bala FA Date: Sun, 21 Aug 2022 03:40:11 +0530 Subject: [PATCH] Refactor S3 client implementation (#13) Added below S3 APIs * abort_multipart_upload() * bucket_exists() * complete_multipart_upload() * create_multipart_upload() * get_object() * list_buckets() * list_objects_v1() * list_objects_v2() * list_object_versions() * list_objects() * make_bucket() * put_object() * put_object_api() * remove_bucket() * remove_object() * remove_objects_api() * remove_objects() * select_object_content() * stat_object() * upload_part() Signed-off-by: Bala.FA --- .github/workflow/ci.yml | 39 + Cargo.toml | 46 +- src/lib.rs | 98 +- src/minio.rs | 534 ---------- src/minio/api.rs | 45 - src/minio/api_notification.rs | 177 ---- src/minio/net.rs | 113 -- src/minio/sign.rs | 238 ----- src/minio/types.rs | 190 ---- src/minio/woxml.rs | 91 -- src/minio/xml.rs | 265 ----- src/s3/args.rs | 981 +++++++++++++++++ src/s3/client.rs | 1888 +++++++++++++++++++++++++++++++++ src/s3/creds.rs | 48 + src/s3/error.rs | 187 ++++ src/s3/http.rs | 279 +++++ src/s3/mod.rs | 25 + src/s3/response.rs | 596 +++++++++++ src/s3/signer.rs | 241 +++++ src/s3/sse.rs | 168 +++ src/s3/types.rs | 459 ++++++++ src/s3/utils.rs | 282 +++++ tests/tests.rs | 453 ++++++++ 23 files changed, 5686 insertions(+), 1757 deletions(-) create mode 100644 .github/workflow/ci.yml delete mode 100644 src/minio.rs delete mode 100644 src/minio/api.rs delete mode 100644 src/minio/api_notification.rs delete mode 100644 src/minio/net.rs delete mode 100644 src/minio/sign.rs delete mode 100644 src/minio/types.rs delete mode 100644 src/minio/woxml.rs delete mode 100644 src/minio/xml.rs create mode 100644 src/s3/args.rs create mode 100644 src/s3/client.rs create mode 100644 src/s3/creds.rs create mode 100644 src/s3/error.rs create mode 100644 src/s3/http.rs create mode 100644 src/s3/mod.rs create mode 100644 src/s3/response.rs create mode 100644 src/s3/signer.rs create mode 100644 src/s3/sse.rs create mode 100644 src/s3/types.rs create mode 100644 src/s3/utils.rs create mode 100644 tests/tests.rs diff --git a/.github/workflow/ci.yml b/.github/workflow/ci.yml new file mode 100644 index 0000000..f5f9439 --- /dev/null +++ b/.github/workflow/ci.yml @@ -0,0 +1,39 @@ +name: MinIO Rust Library + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + # - uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + + - name: Check style + run: | + cargo fmt --all -- --check + + - name: Start MinIO server + run: | + wget --quiet https://dl.min.io/server/minio/release/linux-amd64/minio + chmod +x minio + mkdir -p ~/.minio/certs + cp ./tests/public.crt ./tests/private.key ~/.minio/certs/ + sudo cp ./tests/public.crt /usr/local/share/ca-certificates/ + sudo update-ca-certificates + MINIO_CI_CD=true ./minio server /tmp/test-xl/{1...4}/ & + sleep 10 + + - name: Run tests + run: | + SERVER_ENDPOINT=https://localhost:9000/ ACCESS_KEY=minioadmin SECRET_KEY=minioadmin cargo test --verbose -- --nocapture diff --git a/Cargo.toml b/Cargo.toml index 0c2b1ea..c2bf013 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,26 +1,30 @@ [package] -name = "minio-rs" +name = "minio" version = "0.1.0" -authors = ["MinIO Dev Team "] -edition = "2018" +edition = "2021" [dependencies] -bytes = "0.4.12" -futures = "0.1.27" -http = "0.1.17" -hyper = "0.12.28" -hyper-tls = "0.3.2" -log = "0.4.6" -pretty_env_logger = "0.3.0" -ring = "0.14.6" -roxmltree = "0.6.0" -serde = "1.0.92" -serde_derive = "1.0.124" -serde_json = "1.0.39" -time = "0.1.42" +hyper = { version = "0.14", features = ["full"] } tokio = { version = "1", features = ["full"] } -xml-rs = "0.8.3" -quick-xml = { version = "0.22", features = [ "serialize" ] } -thiserror = "1.0.24" - - +derivative = "2.2.0" +multimap = "0.8.3" +urlencoding = "2.1.0" +lazy_static = "1.4.0" +regex = "1.5.6" +chrono = "0.4.19" +chrono_locale = "0.1.1" +sha2 = "0.10.2" +base64 = "0.13.0" +md5 = "0.7.0" +crc = "3.0.0" +byteorder = "1.4.3" +hmac = "0.12.1" +hex = "0.4.3" +reqwest = { version = "0.11.11", features = ["stream"] } +futures-core = "0.3.21" +bytes = "1.2.0" +futures-util = "0.3.21" +xmltree = "0.10.3" +http = "0.2.8" +dashmap = "5.3.4" +rand = "0.8.5" diff --git a/src/lib.rs b/src/lib.rs index 39e19b1..b0e6bae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,84 +1,16 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -pub mod minio; - -#[cfg(test)] -mod tests { - use futures::{future::Future, stream::Stream}; - use hyper::rt; - use log::debug; - - - use super::*; - - fn get_local_default_server() -> minio::Client { - match minio::Client::new("http://localhost:9000") { - Ok(mut c) => { - c.set_credentials(minio::Credentials::new("minio", "minio123")); - c - } - Err(_) => panic!("could not make local client"), - } - } - - #[test] - fn test_lib_functions() { - println!("test func"); - rt::run(rt::lazy(|| { - let c = minio::Client::get_play_client(); - let bucket_name = "aaaa"; - - c.put_object_req(bucket_name, "hhhhhhhhhh", vec![], "object content".as_bytes().to_vec()) - .and_then(|g| { - print!("object: {} {} {:?}", g.object_size, g.etag, g.content_type); - g.get_object_stream().concat2() - }) - .map(|c| { - println!("{:?}", c); - }) - .map_err(|c| { - println!("{:?}", c); - }) - .map(|_| {}) - })); - - rt::run(rt::lazy(|| { - let c = minio::Client::get_play_client(); - let bucket = "aaaa"; - - c.get_object_req(bucket, "hhhhhhhhhh", vec![]) - .and_then(|g| { - debug!("object: {} {} {:?}", g.object_size, g.etag, g.content_type); - g.get_object_stream().concat2() - }) - .map(|c| debug!("get obj res: {:?}", c)) - .map_err(|c| debug!("err res: {:?}", c)) - .map(|_| {}) - })); - - rt::run(rt::lazy(|| { - let c = minio::Client::get_play_client(); - let bucket = "aaaa"; - - c.delete_bucket(bucket) - .map(|_| debug!("Deleted!")) - .map_err(|err| debug!("del err: {:?}", err)) - .map(|_| {}) - })); - } -} +pub mod s3; diff --git a/src/minio.rs b/src/minio.rs deleted file mode 100644 index 60e9ab5..0000000 --- a/src/minio.rs +++ /dev/null @@ -1,534 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -use std::env; -use std::str; -use std::string::String; - -use futures::future::{self, Future}; -use futures::Stream; -use http; -use hyper::{body::Body, client, header, header::HeaderMap, Method, Request, Response, Uri}; -use hyper::header::{HeaderName, HeaderValue}; -use hyper_tls::HttpsConnector; -use log::debug; -use time; -use time::Tm; - -use types::{Err, GetObjectResp, ListObjectsResp, Region}; -pub use types::BucketInfo; - - -use crate::minio::net::{Values, ValuesAccess}; -use crate::minio::xml::{parse_s3_error, S3GenericError}; -use bytes::Buf; - -mod api; -mod api_notification; -mod net; -mod sign; -mod types; -mod xml; -mod woxml; - -pub const SPACE_BYTE: &[u8; 1] = b" "; - -#[derive(Debug, Clone)] -pub struct Credentials { - access_key: String, - secret_key: String, -} - -impl Credentials { - pub fn new(ak: &str, sk: &str) -> Credentials { - Credentials { - access_key: ak.to_string(), - secret_key: sk.to_string(), - } - } - - pub fn from_env() -> Result { - let (ak, sk) = (env::var("MINIO_ACCESS_KEY"), env::var("MINIO_SECRET_KEY")); - match (ak, sk) { - (Ok(ak), Ok(sk)) => Ok(Credentials::new(ak.as_str(), sk.as_str())), - _ => Err(Err::InvalidEnv( - "Missing MINIO_ACCESS_KEY or MINIO_SECRET_KEY environment variables".to_string(), - )), - } - } -} - -#[derive(Clone)] -enum ConnClient { - HttpCC(client::Client), - HttpsCC(client::Client, Body>), -} - -impl ConnClient { - fn make_req(&self, req: http::Request) -> client::ResponseFuture { - match self { - ConnClient::HttpCC(c) => c.request(req), - ConnClient::HttpsCC(c) => c.request(req), - } - } -} - -pub struct Client { - server: Uri, - region: Region, - conn_client: ConnClient, - pub credentials: Option, -} - -impl Client { - pub fn new(server: &str) -> Result { - let valid = server.parse::(); - match valid { - Ok(server_uri) => { - if server_uri.host().is_none() { - Err(Err::InvalidUrl("no host specified!".to_string())) - } else if server_uri.scheme_str() != Some("http") - && server_uri.scheme_str() != Some("https") - { - Err(Err::InvalidUrl("invalid scheme!".to_string())) - } else { - Ok(Client { - server: server_uri.clone(), - region: Region::empty(), - conn_client: if server_uri.scheme_str() == Some("http") { - ConnClient::HttpCC(client::Client::new()) - } else { - let https = HttpsConnector::new(4).unwrap(); - ConnClient::HttpsCC( - client::Client::builder().build::<_, hyper::Body>(https), - ) - }, - credentials: None, - }) - } - } - Err(err) => Err(Err::InvalidUrl(err.to_string())), - } - } - - pub fn set_credentials(&mut self, credentials: Credentials) { - self.credentials = Some(credentials); - } - - pub fn set_region(&mut self, r: Region) { - self.region = r; - } - - fn add_host_header(&self, header_map: &mut HeaderMap) { - let host_val = match self.server.port_part() { - Some(port) => format!("{}:{}", self.server.host().unwrap_or(""), port), - None => self.server.host().unwrap_or("").to_string(), - }; - match header::HeaderValue::from_str(&host_val) { - Ok(v) => { - header_map.insert(header::HOST, v); - } - _ => {} - } - } - - pub fn get_play_client() -> Client { - Client { - server: "https://play.min.io:9000".parse::().unwrap(), - region: Region::new("us-east-1"), - conn_client: { - let https = HttpsConnector::new(4).unwrap(); - ConnClient::HttpsCC(client::Client::builder().build::<_, hyper::Body>(https)) - }, - credentials: Some(Credentials::new( - "Q3AM3UQ867SPQQA43P2F", - "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", - )), - } - } - - fn signed_req_future( - &self, - mut s3_req: S3Req, - body_res: Result, - ) -> impl Future, Error=Err> { - let hmap = &mut s3_req.headers; - self.add_host_header(hmap); - - let body_hash_hdr = ( - HeaderName::from_static("x-amz-content-sha256"), - HeaderValue::from_static("UNSIGNED-PAYLOAD"), - ); - hmap.insert(body_hash_hdr.0.clone(), body_hash_hdr.1.clone()); - let creds = self.credentials.clone(); - let region = self.region.clone(); - let server_addr = self.server.to_string(); - let conn_client = self.conn_client.clone(); - - future::result(body_res) - .and_then(move |body| { - s3_req.body = body; - let sign_hdrs = sign::sign_v4(&s3_req, creds, region); - debug!("signout: {:?}", sign_hdrs); - api::mk_request(s3_req, &server_addr, &sign_hdrs) - }) - .and_then(move |req| { - debug!("{:?}", req); - conn_client.make_req(req).map_err(|e| Err::HyperErr(e)) - }) - .and_then(|resp| { - let st = resp.status(); - if st.is_success() { - Ok(resp) - } else { - Err(Err::RawSvcErr(st, resp)) - } - }) - .or_else(|err| { - future::err(err) - .or_else(|x| match x { - Err::RawSvcErr(st, resp) => Ok((st, resp)), - other_err => Err(other_err), - }) - .and_then(|(st, resp)| { - resp.into_body() - .concat2() - .map_err(|err| Err::HyperErr(err)) - .and_then(move |chunk| { - match st.as_str() { - "404" => { - let x = str::from_utf8(&chunk.bytes()); - let s3_err = parse_s3_error(x.unwrap()); - Err(Err::S3Error(s3_err)) - } - _ => { - Err(Err::FailStatusCodeErr(st, chunk.into_bytes())) - } - } - }) - }) - }) - } - - /// get_bucket_location - Get location for the bucket_name. - pub fn get_bucket_location( - &self, - bucket_name: &str, - ) -> impl Future { - let mut qp = Values::new(); - qp.set_value("location", None); - - let s3_req = S3Req { - method: Method::GET, - bucket: Some(bucket_name.to_string()), - object: None, - headers: HeaderMap::new(), - query: qp, - body: Body::empty(), - ts: time::now_utc(), - }; - self.signed_req_future(s3_req, Ok(Body::empty())) - .and_then(|resp| { - // Read the whole body for bucket location response. - resp.into_body() - .concat2() - .map_err(|err| Err::HyperErr(err)) - .and_then(move |chunk| chunk_to_string(&chunk)) - .and_then(|s| xml::parse_bucket_location(s)) - }) - } - - pub fn delete_bucket(&self, bucket_name: &str) -> impl Future { - let s3_req = S3Req { - method: Method::DELETE, - bucket: Some(bucket_name.to_string()), - object: None, - headers: HeaderMap::new(), - query: Values::new(), - body: Body::empty(), - ts: time::now_utc(), - }; - self.signed_req_future(s3_req, Ok(Body::empty())) - .and_then(|_| Ok(())) - } - - pub fn bucket_exists(&self, bucket_name: &str) -> impl Future { - let s3_req = S3Req { - method: Method::HEAD, - bucket: Some(bucket_name.to_string()), - object: None, - headers: HeaderMap::new(), - query: Values::new(), - body: Body::empty(), - ts: time::now_utc(), - }; - self.signed_req_future(s3_req, Ok(Body::empty())) - .then(|res| match res { - Ok(_) => Ok(true), - Err(Err::FailStatusCodeErr(st, b)) => { - let code = st.as_u16(); - if code == 404 { - Ok(false) - } else { - Err(Err::FailStatusCodeErr(st, b)) - } - } - Err(err) => Err(err), - }) - } - - pub fn get_object_req( - &self, - bucket_name: &str, - key: &str, - get_obj_opts: Vec<(HeaderName, HeaderValue)>, - ) -> impl Future { - let mut h = HeaderMap::new(); - get_obj_opts - .iter() - .map(|(x, y)| (x.clone(), y.clone())) - .for_each(|(k, v)| { - h.insert(k, v); - }); - - let s3_req = S3Req { - method: Method::GET, - bucket: Some(bucket_name.to_string()), - object: Some(key.to_string()), - headers: h, - query: Values::new(), - body: Body::empty(), - ts: time::now_utc(), - }; - - self.signed_req_future(s3_req, Ok(Body::empty())) - .and_then(GetObjectResp::new) - } - - pub fn put_object_req( - &self, - bucket_name: &str, - key: &str, - get_obj_opts: Vec<(HeaderName, HeaderValue)>, - data: Vec, - ) -> impl Future { - let mut h = HeaderMap::new(); - get_obj_opts - .iter() - .map(|(x, y)| (x.clone(), y.clone())) - .for_each(|(k, v)| { - h.insert(k, v); - }); - - let s3_req = S3Req { - method: Method::PUT, - bucket: Some(bucket_name.to_string()), - object: Some(key.to_string()), - headers: h, - query: Values::new(), - body: Body::from(data.clone()), - ts: time::now_utc(), - }; - - self.signed_req_future(s3_req, Ok(Body::from(data))) - .and_then(GetObjectResp::new) - } - - pub fn make_bucket(&self, bucket_name: &str) -> impl Future { - let xml_body_res = xml::get_mk_bucket_body(); - let bucket = bucket_name.clone().to_string(); - let s3_req = S3Req { - method: Method::PUT, - bucket: Some(bucket), - object: None, - query: Values::new(), - headers: HeaderMap::new(), - body: Body::empty(), - ts: time::now_utc(), - }; - self.signed_req_future(s3_req, xml_body_res) - .and_then(|_| future::ok(())) - } - - pub fn list_buckets(&self) -> impl Future, Error=Err> { - let s3_req = S3Req { - method: Method::GET, - bucket: None, - object: None, - query: Values::new(), - headers: HeaderMap::new(), - body: Body::empty(), - ts: time::now_utc(), - }; - self.signed_req_future(s3_req, Ok(Body::empty())) - .and_then(|resp| { - // Read the whole body for list buckets response. - resp.into_body() - .concat2() - .map_err(|err| Err::HyperErr(err)) - .and_then(move |chunk| chunk_to_string(&chunk)) - .and_then(|s| xml::parse_bucket_list(s)) - }) - } - - pub fn list_objects( - &self, - b: &str, - prefix: Option<&str>, - marker: Option<&str>, - delimiter: Option<&str>, - max_keys: Option, - ) -> impl Future { - let mut qparams: Values = Values::new(); - qparams.set_value("list-type", Some("2".to_string())); - if let Some(d) = delimiter { - qparams.set_value("delimiter", Some(d.to_string())); - } - if let Some(m) = marker { - qparams.set_value("marker", Some(m.to_string())); - } - - if let Some(p) = prefix { - qparams.set_value("prefix", Some(p.to_string())); - } - - if let Some(mkeys) = max_keys { - qparams.set_value("max-keys", Some(mkeys.to_string())); - } - - let s3_req = S3Req { - method: Method::GET, - bucket: Some(b.to_string()), - object: None, - query: qparams, - headers: HeaderMap::new(), - body: Body::empty(), - ts: time::now_utc(), - }; - self.signed_req_future(s3_req, Ok(Body::empty())) - .and_then(|resp| { - resp.into_body() - .concat2() - .map_err(|err| Err::HyperErr(err)) - .and_then(move |chunk| chunk_to_string(&chunk)) - .and_then(|s| xml::parse_list_objects(s)) - }) - } -} - -fn run_req_future( - req_result: Result, Err>, - c: ConnClient, -) -> impl Future, Error=Err> { - future::result(req_result) - //.map_err(|e| Err::HttpErr(e)) - .and_then(move |req| c.make_req(req).map_err(|e| Err::HyperErr(e))) - .and_then(|resp| { - let st = resp.status(); - if st.is_success() { - Ok(resp) - } else { - Err(Err::RawSvcErr(st, resp)) - } - }) -} - -/// Converts a `hyper::Chunk` into a string. -fn chunk_to_string(chunk: &hyper::Chunk) -> Result { - match String::from_utf8(chunk.to_vec()) { - Err(e) => Err(Err::Utf8DecodingErr(e)), - Ok(s) => Ok(s.to_string()), - } -} - -pub struct S3Req { - method: Method, - bucket: Option, - object: Option, - headers: HeaderMap, - query: Values, - body: Body, - ts: Tm, -} - -impl S3Req { - fn mk_path(&self) -> String { - let mut res: String = String::from("/"); - if let Some(s) = &self.bucket { - res.push_str(&s); - res.push_str("/"); - if let Some(o) = &self.object { - res.push_str(&o); - } - }; - res - } - - /// Takes the query_parameters and turn them into a valid query string for example: - /// {"key1":["val1","val2"],"key2":["val1","val2"]} - /// will be returned as: - /// "key1=val1&key1=val2&key2=val3&key2=val4" - fn mk_query(&self) -> String { - self.query - .iter() - .map(|(key, values)| { - values.iter().map(move |value| match value { - Some(v) => format!("{}={}", &key, v), - None => format!("{}=", &key, ), - }) - }) - .flatten() - .collect::>() - .join("&") - } -} - -#[cfg(test)] -mod minio_tests { - use std::collections::HashMap; - - use super::*; - - #[test] - fn serialize_query_parameters() { - let mut query_params: HashMap>> = HashMap::new(); - query_params.insert( - "key1".to_string(), - vec![Some("val1".to_string()), Some("val2".to_string())], - ); - query_params.insert( - "key2".to_string(), - vec![Some("val3".to_string()), Some("val4".to_string())], - ); - - let s3_req = S3Req { - method: Method::GET, - bucket: None, - object: None, - headers: HeaderMap::new(), - query: query_params, - body: Body::empty(), - ts: time::now_utc(), - }; - let result = s3_req.mk_query(); - assert!(result.contains("key1=val1")); - assert!(result.contains("key1=val2")); - assert!(result.contains("key2=val3")); - assert!(result.contains("key2=val4")); - } -} diff --git a/src/minio/api.rs b/src/minio/api.rs deleted file mode 100644 index 4e440f1..0000000 --- a/src/minio/api.rs +++ /dev/null @@ -1,45 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use crate::minio; -use hyper::{header::HeaderName, header::HeaderValue, Body, Request}; -use log::debug; - -pub fn mk_request( - r: minio::S3Req, - svr_str: &str, - sign_hdrs: &Vec<(HeaderName, HeaderValue)>, -) -> Result, minio::Err> { - let mut request = Request::builder(); - let uri_str = svr_str.trim_end_matches('/'); - debug!("uri_str: {}", uri_str); - let upd_uri = format!("{}{}?{}", uri_str, &r.mk_path(), &r.mk_query()); - debug!("upd_uri: {}", upd_uri); - - request.uri(&upd_uri).method(&r.method); - for hdr in r - .headers - .iter() - .map(|(x, y)| (x.clone(), y.clone())) - .chain(sign_hdrs.iter().map(|x| x.clone())) - { - request.header(hdr.0, hdr.1); - } - request - .body(r.body) - .map_err(|err| minio::Err::HttpErr(err)) -} diff --git a/src/minio/api_notification.rs b/src/minio/api_notification.rs deleted file mode 100644 index ebc7e52..0000000 --- a/src/minio/api_notification.rs +++ /dev/null @@ -1,177 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use std::collections::HashMap; - -use crate::minio::net::{Values, ValuesAccess}; -use crate::minio::{Client, Err, S3Req, SPACE_BYTE}; -use futures::future::Future; -use futures::{stream, Stream}; -use hyper::{Body, HeaderMap, Method}; -use serde_derive::Deserialize; - -/// Notification event object metadata. -#[derive(Deserialize, Debug)] -pub struct ObjectMeta { - #[serde(rename(deserialize = "key"))] - pub key: String, - #[serde(rename(deserialize = "size"))] - pub size: Option, - #[serde(rename(deserialize = "eTag"))] - pub e_tag: Option, - #[serde(rename(deserialize = "versionId"))] - pub version_id: Option, - #[serde(rename(deserialize = "sequencer"))] - pub sequencer: String, -} - -/// Notification event bucket metadata. -#[derive(Deserialize, Debug)] -pub struct BucketMeta { - #[serde(rename(deserialize = "name"))] - pub name: String, - #[serde(rename(deserialize = "ownerIdentity"))] - pub owner_identity: Identity, - #[serde(rename(deserialize = "arn"))] - pub arn: String, -} - -/// Indentity represents the user id, this is a compliance field. -#[derive(Deserialize, Debug)] -pub struct Identity { - #[serde(rename(deserialize = "principalId"))] - pub principal_id: String, -} - -//// sourceInfo represents information on the client that -//// triggered the event notification. -#[derive(Deserialize, Debug)] -pub struct SourceInfo { - #[serde(rename(deserialize = "host"))] - pub host: String, - #[serde(rename(deserialize = "port"))] - pub port: String, - #[serde(rename(deserialize = "userAgent"))] - pub user_agent: String, -} - -/// Notification event server specific metadata. -#[derive(Deserialize, Debug)] -pub struct EventMeta { - #[serde(rename(deserialize = "s3SchemaVersion"))] - pub schema_version: String, - #[serde(rename(deserialize = "configurationId"))] - pub configuration_id: String, - #[serde(rename(deserialize = "bucket"))] - pub bucket: BucketMeta, - #[serde(rename(deserialize = "object"))] - pub object: ObjectMeta, -} - -/// NotificationEvent represents an Amazon an S3 bucket notification event. -#[derive(Deserialize, Debug)] -pub struct NotificationEvent { - #[serde(rename(deserialize = "eventVersion"))] - pub event_version: String, - #[serde(rename(deserialize = "eventSource"))] - pub event_source: String, - #[serde(rename(deserialize = "awsRegion"))] - pub aws_region: String, - #[serde(rename(deserialize = "eventTime"))] - pub event_time: String, - #[serde(rename(deserialize = "eventName"))] - pub event_name: String, - #[serde(rename(deserialize = "source"))] - pub source: SourceInfo, - #[serde(rename(deserialize = "userIdentity"))] - pub user_identity: Identity, - #[serde(rename(deserialize = "requestParameters"))] - pub request_parameters: HashMap, - #[serde(rename(deserialize = "responseElements"))] - pub response_elements: HashMap, - #[serde(rename(deserialize = "s3"))] - pub s3: EventMeta, -} - -/// NotificationInfo - represents the collection of notification events, additionally -/// also reports errors if any while listening on bucket notifications. -#[derive(Deserialize, Debug)] -pub struct NotificationInfo { - #[serde(rename(deserialize = "Records"), default = "Vec::new")] - pub records: Vec, - pub err: Option, -} - -impl Client { - /// listen_bucket_notificaion - Get bucket notifications for the bucket_name. - pub fn listen_bucket_notification( - &self, - bucket_name: &str, - prefix: Option, - suffix: Option, - events: Vec, - ) -> impl Stream { - // Prepare request query parameters - let mut query_params: Values = Values::new(); - query_params.set_value("prefix", prefix); - query_params.set_value("suffix", suffix); - let opt_events: Vec> = events.into_iter().map(|evt| Some(evt)).collect(); - query_params.insert("events".to_string(), opt_events); - - // build signed request - let s3_req = S3Req { - method: Method::GET, - bucket: Some(bucket_name.to_string()), - object: None, - headers: HeaderMap::new(), - query: query_params, - body: Body::empty(), - ts: time::now_utc(), - }; - - self.signed_req_future(s3_req, Ok(Body::empty())) - .map(|resp| { - // Read the whole body for bucket location response. - resp.into_body() - .map_err(|e| Err::HyperErr(e)) - .filter(|c| { - // filter out white spaces sent by the server to indicate it's still alive - c[0] != SPACE_BYTE[0] - }) - .map(|chunk| { - // Split the chunk by lines and process. - // TODO: Handle case when partial lines are present in the chunk - let chunk_lines = String::from_utf8(chunk.to_vec()) - .map(|p| { - let lines = - p.lines().map(|s| s.to_string()).collect::>(); - stream::iter_ok(lines.into_iter()) - }) - .map_err(|e| Err::Utf8DecodingErr(e)); - futures::future::result(chunk_lines).flatten_stream() - }) - .flatten() - .map(|line| { - // Deserialize the notification - let notification_info: NotificationInfo = - serde_json::from_str(&line).unwrap(); - notification_info - }) - }) - .flatten_stream() - } -} diff --git a/src/minio/net.rs b/src/minio/net.rs deleted file mode 100644 index 9ba37a9..0000000 --- a/src/minio/net.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use std::collections::HashMap; - -pub type Values = HashMap>>; - -pub trait ValuesAccess { - fn get_value(&self, key: &str) -> Option; - fn set_value(&mut self, key: &str, value: Option); - fn add_value(&mut self, key: &str, value: Option); - fn del_value(&mut self, key: &str); -} - -impl ValuesAccess for Values { - /// Gets the first item for a given key. If the key is invalid it returns `None` - /// To get multiple values use the `Values` instance as map. - fn get_value(&self, key: &str) -> Option { - let value_vec = match self.get(key) { - Some(v) => v, - None => return None, - }; - if value_vec.len() == 0 { - return None; - } - return value_vec.get(0).unwrap().clone(); - } - - /// Sets the key to value. It replaces any existing values. - fn set_value(&mut self, key: &str, value: Option) { - self.insert(key.to_string(), vec![value]); - } - - /// Add adds the value to key. It appends to any existing values associated with key. - fn add_value(&mut self, key: &str, value: Option) { - match self.get_mut(key) { - Some(value_vec) => value_vec.push(value), - None => (), - } - } - - // Del deletes the values associated with key. - fn del_value(&mut self, key: &str) { - self.remove(key); - } -} - -#[cfg(test)] -mod net_tests { - use super::*; - - #[test] - fn values_set() { - let mut values = Values::new(); - values.set_value("key", Some("value".to_string())); - assert_eq!(values.len(), 1); - assert_eq!(values.get("key").unwrap().len(), 1); - - values.set_value("key", None); - assert_eq!(values.len(), 1); - assert_eq!(values.get("key").unwrap().len(), 1); - } - - #[test] - fn values_add() { - let mut values = Values::new(); - values.set_value("key", Some("value".to_string())); - assert_eq!(values.get("key").unwrap().len(), 1); - - values.add_value("key", None); - assert_eq!(values.get("key").unwrap().len(), 2); - } - - #[test] - fn values_del() { - let mut values = Values::new(); - values.set_value("key", Some("value".to_string())); - values.add_value("key", None); - values.del_value("key"); - assert_eq!(values.len(), 0); - - let mut values2 = Values::new(); - values2.set_value("key", Some("value".to_string())); - values2.add_value("key", None); - values2.set_value("key2", Some("value".to_string())); - values2.add_value("key2", None); - - values2.del_value("key"); - assert_eq!(values2.len(), 1); - } - - #[test] - fn values_get() { - let mut values = Values::new(); - values.set_value("key", Some("value".to_string())); - values.add_value("key", None); - assert_eq!(values.get_value("key"), Some("value".to_string())); - } -} diff --git a/src/minio/sign.rs b/src/minio/sign.rs deleted file mode 100644 index 7ac1f7e..0000000 --- a/src/minio/sign.rs +++ /dev/null @@ -1,238 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use std::collections::{HashMap, HashSet}; - -use hyper::header::{ - AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT, -}; -use log::debug; -use ring::{digest, hmac}; -use time::Tm; - -use crate::minio; -use crate::minio::types::Region; - -fn aws_format_time(t: &Tm) -> String { - t.strftime("%Y%m%dT%H%M%SZ").unwrap().to_string() -} - -fn aws_format_date(t: &Tm) -> String { - t.strftime("%Y%m%d").unwrap().to_string() -} - -fn mk_scope(t: &Tm, r: &minio::Region) -> String { - let scope_time = t.strftime("%Y%m%d").unwrap().to_string(); - format!("{}/{}/s3/aws4_request", scope_time, r.to_string()) -} - -// Returns list of SORTED headers that will be signed. -// TODO: verify that input headermap contains only ASCII valued headers -fn get_headers_to_sign(h: HeaderMap) -> Vec<(String, String)> { - let ignored_hdrs: HashSet = vec![ - AUTHORIZATION, - CONTENT_LENGTH, - CONTENT_TYPE, - USER_AGENT].into_iter().collect(); - - let mut res: Vec<(String, String)> = h - .iter() - .filter(|(x, _)| !ignored_hdrs.contains(*x)) - .map(|(x, y)| { - ( - x.as_str().to_string(), - y.to_str() - .expect("Unexpected non-ASCII header value!") - .to_string(), - ) - }).collect(); - res.sort(); - res -} - -fn uri_encode(c: char, encode_slash: bool) -> String { - if c == '/' { - if encode_slash { - "%2F".to_string() - } else { - "/".to_string() - } - } else if c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.' || c == '~' { - c.to_string() - } else { - let mut b = [0; 8]; - let cs = c.encode_utf8(&mut b).as_bytes(); - cs.iter().map(|x| format!("%{:02X}", x)).collect() - } -} - -fn uri_encode_str(s: &str, encode_slash: bool) -> String { - s.chars().map(|x| uri_encode(x, encode_slash)).collect() -} - -fn get_canonical_querystr(q: &HashMap>>) -> String { - let mut hs: Vec<(String, Vec>)> = q.clone().drain().collect(); - // sort keys - hs.sort(); - // Build canonical query string - hs.iter() - .map(|(key, values)| { - values.iter().map(move |value| match value { - Some(v) => format!("{}={}", &key, uri_encode_str(&v, true)), - None => format!("{}=", &key), - }) - }) - .flatten() - .collect::>() - .join("&") -} - -fn get_canonical_request( - r: &minio::S3Req, - hdrs_to_use: &Vec<(String, String)>, - signed_hdrs_str: &str, -) -> String { - let path_str = r.mk_path(); - let canonical_qstr = get_canonical_querystr(&r.query); - let canonical_hdrs: String = hdrs_to_use - .iter() - .map(|(x, y)| format!("{}:{}\n", x.clone(), y.clone())) - .collect(); - - // FIXME: using only unsigned payload for now - need to add - // hashing of payload. - let payload_hash_str = String::from("UNSIGNED-PAYLOAD"); - let res = vec![ - r.method.to_string(), - uri_encode_str(&path_str, false), - canonical_qstr, - canonical_hdrs, - signed_hdrs_str.to_string(), - payload_hash_str, - ]; - res.join("\n") -} - -fn string_to_sign(ts: &Tm, scope: &str, canonical_request: &str) -> String { - let sha256_digest: String = digest::digest(&digest::SHA256, canonical_request.as_bytes()) - .as_ref() - .iter() - .map(|x| format!("{:02x}", x)) - .collect(); - vec![ - "AWS4-HMAC-SHA256", - &aws_format_time(&ts), - scope, - &sha256_digest, - ] - .join("\n") -} - -fn hmac_sha256(msg: &str, key: &[u8]) -> hmac::Signature { - let key = hmac::SigningKey::new(&digest::SHA256, key); - hmac::sign(&key, msg.as_bytes()) -} - -fn get_signing_key(ts: &Tm, region: &str, secret_key: &str) -> Vec { - let kstr = format!("AWS4{}", secret_key); - let s1 = hmac_sha256(&aws_format_date(&ts), kstr.as_bytes()); - let s2 = hmac_sha256(®ion, s1.as_ref()); - let s3 = hmac_sha256("s3", s2.as_ref()); - let s4 = hmac_sha256("aws4_request", s3.as_ref()); - // FIXME: can this be done better? - s4.as_ref().iter().map(|x| x.clone()).collect() -} - -fn compute_sign(str_to_sign: &str, key: &Vec) -> String { - let s1 = hmac_sha256(&str_to_sign, key.as_slice()); - s1.as_ref().iter().map(|x| format!("{:02x}", x)).collect() -} - -pub fn sign_v4( - request: &minio::S3Req, - credentials: Option, - region: Region, -) -> Vec<(HeaderName, HeaderValue)> { - credentials.map_or(Vec::new(), |creds| { - let scope = mk_scope(&request.ts, ®ion); - let date_hdr = ( - HeaderName::from_static("x-amz-date"), - HeaderValue::from_str(&aws_format_time(&request.ts)).unwrap(), - ); - let mut hmap = request.headers.clone(); - hmap.insert(date_hdr.0.clone(), date_hdr.1.clone()); - - let headers = get_headers_to_sign(hmap); - let signed_hdrs_str: String = headers - .iter() - .map(|(x, _)| x.clone()) - .collect::>() - .join(";"); - let cr = get_canonical_request(request, &headers, &signed_hdrs_str); - debug!("canonicalreq: {}", cr); - let s2s = string_to_sign(&request.ts, &scope, &cr); - debug!("s2s: {}", s2s); - let skey = get_signing_key(&request.ts, ®ion.to_string(), &creds.secret_key); - debug!("skey: {:?}", skey); - let signature = compute_sign(&s2s, &skey); - debug!("sign: {}", signature); - - let auth_hdr_val = format!( - "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", - &creds.access_key, &scope, &signed_hdrs_str, &signature, - ); - let auth_hdr = (AUTHORIZATION, HeaderValue::from_str(&auth_hdr_val).unwrap()); - vec![auth_hdr, date_hdr] - }) -} - -#[cfg(test)] -mod sign_tests { - use super::*; - - #[test] - fn canonical_ordered() { - let mut query_params: HashMap>> = HashMap::new(); - - query_params.insert("key2".to_string(), vec![Some("val3".to_string()), None]); - - query_params.insert( - "key1".to_string(), - vec![Some("val1".to_string()), Some("val2".to_string())], - ); - - assert_eq!( - get_canonical_querystr(&query_params), - "key1=val1&key1=val2&key2=val3&key2=" - ); - } - - #[test] - fn headers_to_sign_remove_ignored_and_sort() { - let mut map = HeaderMap::new(); - map.insert(AUTHORIZATION, "hello".parse().unwrap()); - map.insert(CONTENT_LENGTH, "123".parse().unwrap()); - map.insert("second", "123".parse().unwrap()); - map.insert("first", "123".parse().unwrap()); - - assert_eq!( - get_headers_to_sign(map), - vec![("first".parse().unwrap(), "123".parse().unwrap()), - ("second".parse().unwrap(), "123".parse().unwrap())] - ); - } -} diff --git a/src/minio/types.rs b/src/minio/types.rs deleted file mode 100644 index 790a9ce..0000000 --- a/src/minio/types.rs +++ /dev/null @@ -1,190 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use bytes::Bytes; -use futures::stream::Stream; -use hyper::header::{ - HeaderMap, HeaderValue, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, - CONTENT_LENGTH, CONTENT_TYPE, ETAG, EXPIRES, -}; -use hyper::{body::Body, Response}; -use roxmltree; -use std::collections::HashMap; -use std::string; -use time::{strptime, Tm}; -use crate::minio::xml::S3GenericError; - -#[derive(Clone)] -pub struct Region(String); - -impl Region { - pub fn new(s: &str) -> Region { - Region(s.to_string()) - } - - pub fn empty() -> Region { - Region::new("") - } - - pub fn to_string(&self) -> String { - self.0.clone() - } -} - -#[derive(Debug)] -pub enum Err { - InvalidUrl(String), - InvalidEnv(String), - InvalidTmFmt(String), - HttpErr(http::Error), - HyperErr(hyper::Error), - FailStatusCodeErr(hyper::StatusCode, Bytes), - Utf8DecodingErr(string::FromUtf8Error), - XmlDocParseErr(roxmltree::Error), - XmlElemMissing(String), - XmlElemParseErr(String), - InvalidXmlResponseErr(String), - MissingRequiredParams, - RawSvcErr(hyper::StatusCode, Response), - XmlWriteErr(String), - S3Error(S3GenericError), -} - -pub struct GetObjectResp { - pub user_metadata: Vec<(String, String)>, - pub object_size: u64, - pub etag: String, - - // standard headers - pub content_type: Option, - pub content_language: Option, - pub expires: Option, - pub cache_control: Option, - pub content_disposition: Option, - pub content_encoding: Option, - - resp: Response, -} - -impl GetObjectResp { - pub fn new(r: Response) -> Result { - let h = r.headers(); - - let cl_opt = hv2s(h.get(CONTENT_LENGTH)).and_then(|l| l.parse::().ok()); - let etag_opt = hv2s(h.get(ETAG)); - match (cl_opt, etag_opt) { - (Some(cl), Some(etag)) => Ok(GetObjectResp { - user_metadata: extract_user_meta(h), - object_size: cl, - etag: etag, - - content_type: hv2s(h.get(CONTENT_TYPE)), - content_language: hv2s(h.get(CONTENT_LANGUAGE)), - expires: hv2s(h.get(EXPIRES)), - cache_control: hv2s(h.get(CACHE_CONTROL)), - content_disposition: hv2s(h.get(CONTENT_DISPOSITION)), - content_encoding: hv2s(h.get(CONTENT_ENCODING)), - - resp: r, - }), - _ => Err(Err::MissingRequiredParams), - } - } - - // Consumes GetObjectResp - pub fn get_object_stream(self) -> impl Stream { - self.resp.into_body().map_err(|err| Err::HyperErr(err)) - } -} - -fn hv2s(o: Option<&HeaderValue>) -> Option { - o.and_then(|v| v.to_str().ok()).map(|x| x.to_string()) -} - -fn extract_user_meta(h: &HeaderMap) -> Vec<(String, String)> { - h.iter() - .map(|(k, v)| (k.as_str(), v.to_str())) - .filter(|(k, v)| k.to_lowercase().starts_with("x-amz-meta-") && v.is_ok()) - .map(|(k, v)| (k.to_string(), v.unwrap_or("").to_string())) - .collect() -} - -fn parse_aws_time(time_str: &str) -> Result { - strptime(time_str, "%Y-%m-%dT%H:%M:%S.%Z") - .map_err(|err| Err::InvalidTmFmt(format!("{:?}", err))) -} - -#[derive(Debug)] -pub struct BucketInfo { - pub name: String, - pub created_time: Tm, -} - -impl BucketInfo { - pub fn new(name: &str, time_str: &str) -> Result { - parse_aws_time(time_str).and_then(|ctime| { - Ok(BucketInfo { - name: name.to_string(), - created_time: ctime, - }) - }) - } -} - -#[derive(Debug)] -pub struct ObjectInfo { - pub name: String, - pub modified_time: Tm, - pub etag: String, - pub size: i64, - pub storage_class: String, - pub metadata: HashMap, -} - -impl ObjectInfo { - pub fn new( - name: &str, - mtime_str: &str, - etag: &str, - size: i64, - storage_class: &str, - metadata: HashMap, - ) -> Result { - parse_aws_time(mtime_str).and_then(|mtime| { - Ok(ObjectInfo { - name: name.to_string(), - modified_time: mtime, - etag: etag.to_string(), - size: size, - storage_class: storage_class.to_string(), - metadata: metadata, - }) - }) - } -} - -#[derive(Debug)] -pub struct ListObjectsResp { - pub bucket_name: String, - pub prefix: String, - pub max_keys: i32, - pub key_count: i32, - pub is_truncated: bool, - pub object_infos: Vec, - pub common_prefixes: Vec, - pub next_continuation_token: String, -} diff --git a/src/minio/woxml.rs b/src/minio/woxml.rs deleted file mode 100644 index d78e2c8..0000000 --- a/src/minio/woxml.rs +++ /dev/null @@ -1,91 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern crate xml; - -use xml::writer::{EmitterConfig, EventWriter, XmlEvent}; - -pub struct XmlNode { - name: String, - namespace: Option, - text: Option, - children: Vec, -} - -impl XmlNode { - pub fn new(name: &str) -> XmlNode { - XmlNode { - name: name.to_string(), - namespace: None, - text: None, - children: Vec::new(), - } - } - pub fn namespace(mut self, ns: &str) -> XmlNode { - self.namespace = Some(ns.to_string()); - self - } - - pub fn text(mut self, value: &str) -> XmlNode { - self.text = Some(value.to_string()); - self - } - - pub fn children(mut self, kids: Vec) -> XmlNode { - self.children = kids; - self - } - - fn serialize_rec(&self, xml_writer: &mut EventWriter) -> xml::writer::Result<()> - where - W: std::io::Write, - { - let st_elem = XmlEvent::start_element(self.name.as_str()); - let st_elem = match &self.namespace { - Some(ns) => st_elem.ns("", ns.clone()), - None => st_elem, - }; - xml_writer.write(st_elem)?; - - // An xml node would have a text field or child nodes, not both, at least not usually. - match &self.text { - Some(content) => { - let content_node = XmlEvent::characters(content.as_str()); - xml_writer.write(content_node)?; - } - None => { - for child in &self.children { - child.serialize_rec(xml_writer)?; - } - } - } - - let end_elem: XmlEvent = XmlEvent::end_element().name(self.name.as_str()).into(); - xml_writer.write(end_elem)?; - - Ok(()) - } - pub fn serialize(&self, writer: W) -> xml::writer::Result<()> - where - W: std::io::Write, - { - let mut xml_writer = EmitterConfig::new() - .perform_indent(true) - .create_writer(writer); - self.serialize_rec(&mut xml_writer) - } -} diff --git a/src/minio/xml.rs b/src/minio/xml.rs deleted file mode 100644 index 25d6866..0000000 --- a/src/minio/xml.rs +++ /dev/null @@ -1,265 +0,0 @@ -/* - * MinIO Rust Library for Amazon S3 Compatible Cloud Storage - * Copyright 2019 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern crate quick_xml; -extern crate serde; - -use std::collections::HashMap; -use std::str::FromStr; -use std::str; - -use hyper::body::Body; -use roxmltree; -use serde_derive::Deserialize; -use thiserror::Error; - -use crate::minio::types::{BucketInfo, Err, ListObjectsResp, ObjectInfo, Region}; -use crate::minio::woxml; - -pub fn parse_bucket_location(s: String) -> Result { - let res = roxmltree::Document::parse(&s); - match res { - Ok(doc) => { - let region_res = doc.root_element().text(); - if let Some(region) = region_res { - Ok(Region::new(region)) - } else { - Ok(Region::empty()) - } - } - Err(e) => Err(Err::XmlDocParseErr(e)), - } -} - -#[allow(non_snake_case)] -#[derive(Debug, Deserialize, PartialEq)] -pub struct Error { - Param: Option, - Code: String, - Message: String, - BucketName: String, - Key: Option, - RequestId: String, - HostId: String, - // Region where the bucket is located. This header is returned - // only in HEAD bucket and ListObjects response. - Region: Option, -} - - -#[derive(Error, Debug, PartialEq)] -pub enum S3GenericError { - #[error("no such bucket: {error:?}")] - NoSuchBucket { - error: Error, - }, - #[error("unknown error: {error:?}")] - Unknown { - error: Error, - }, -} - - -pub(crate) fn parse_s3_error(response_xml: &str) -> S3GenericError { - println!("{}",response_xml); - let doc: Error = quick_xml::de::from_str(response_xml).unwrap(); - match doc.Code.as_str() { - "NoSuchBucket" => { - return S3GenericError::NoSuchBucket { - error: doc, - }; - } - _ => { - return S3GenericError::Unknown { - error: doc, - }; - } - } -} - -#[cfg(test)] -mod xml_tests { - - use super::*; - - #[test] - fn parse_xml_error() { - let response_xml = r#" - - - NoSuchBucket - The specified bucket does not exist - hhhhhhhhhh - aaaa - /aaaa/hhhhhhhhhh - 166B5E4E3A406CC6 - 129c19c9-4cf6-44ff-9f2d-4cb7611be894 - - "#; - - let s3_error = parse_s3_error(response_xml); - - print!("test! {:?}", s3_error); - assert!(matches!(s3_error, S3GenericError::NoSuchBucket {cerror} )); - } -} - - -pub fn parse_bucket_list(s: String) -> Result, Err> { - let res = roxmltree::Document::parse(&s); - match res { - Ok(doc) => { - let mut bucket_infos: Vec = Vec::new(); - let bucket_nodes = doc - .root_element() - .descendants() - .filter(|node| node.has_tag_name("Bucket")); - for bucket in bucket_nodes { - let bucket_names = bucket.children().filter(|node| node.has_tag_name("Name")); - let bucket_ctimes = bucket - .children() - .filter(|node| node.has_tag_name("CreationDate")); - for (name_node, ctime_node) in bucket_names.zip(bucket_ctimes) { - let name = name_node.text().ok_or(Err::InvalidXmlResponseErr( - "Missing name in list buckets XML response ".to_string(), - ))?; - let ctime = ctime_node.text().ok_or(Err::InvalidXmlResponseErr( - "Missing creation date in list buckets XML response".to_string(), - ))?; - match BucketInfo::new(name, ctime) { - Ok(bucket_info) => bucket_infos.push(bucket_info), - Err(err) => return Err(Err::InvalidTmFmt(format!("{:?}", err))), - } - } - } - Ok(bucket_infos) - } - Err(err) => Err(Err::XmlDocParseErr(err)), - } -} - -pub fn parse_list_objects(s: String) -> Result { - let doc_res = roxmltree::Document::parse(&s); - match doc_res { - Ok(doc) => parse_list_objects_result(doc), - Err(err) => panic!(err), - } -} - -pub fn get_mk_bucket_body() -> Result { - let lc_node = woxml::XmlNode::new("LocationConstraint").text("us-east-1"); - let mk_bucket_xml = woxml::XmlNode::new("CreateBucketConfiguration") - .namespace("http://s3.amazonaws.com/doc/2006-03-01/") - .children(vec![lc_node]); - let mut xml_bytes = Vec::new(); - - mk_bucket_xml - .serialize(&mut xml_bytes) - .or_else(|err| Err(Err::XmlWriteErr(err.to_string())))?; - Ok(Body::from(xml_bytes)) -} - -fn get_child_node<'a>(node: &'a roxmltree::Node, tag_name: &str) -> Option<&'a str> { - node.children() - .find(|node| node.has_tag_name(tag_name)) - .and_then(|node| node.text()) -} - -// gets text value inside given tag or return default -fn get_child_node_or<'a>(node: &'a roxmltree::Node, tag_name: &str, default: &'a str) -> &'a str { - get_child_node(&node, tag_name).unwrap_or(default) -} - -fn parse_child_content(node: &roxmltree::Node, tag: &str) -> Result - where - T: FromStr, -{ - let content = get_child_node(node, tag).ok_or(Err::XmlElemMissing(format!("{:?}", tag)))?; - str::parse::(content).map_err(|_| Err::XmlElemParseErr(format!("{}", tag))) -} - -fn parse_tag_content(node: &roxmltree::Node) -> Result - where - T: FromStr, -{ - let content = must_get_node_text(node)?; - str::parse::(content).map_err(|_| Err::XmlElemParseErr(format!("{:?}", node.tag_name()))) -} - -fn must_get_node_text<'a>(node: &'a roxmltree::Node) -> Result<&'a str, Err> { - node.text() - .ok_or(Err::XmlElemMissing(node.tag_name().name().to_string())) -} - -fn parse_object_infos(node: roxmltree::Node) -> Result, Err> { - let mut object_infos: Vec = Vec::new(); - let contents_nodes = node - .descendants() - .filter(|node| node.has_tag_name("Contents")); - for node in contents_nodes { - let keys = node.children().filter(|node| node.has_tag_name("Key")); - let mtimes = node - .children() - .filter(|node| node.has_tag_name("LastModified")); - let etags = node.children().filter(|node| node.has_tag_name("ETag")); - let sizes = node.children().filter(|node| node.has_tag_name("Size")); - let storage_classes = node - .children() - .filter(|node| node.has_tag_name("StorageClass")); - for (key, (mtime, (etag, (size, storage_class)))) in - keys.zip(mtimes.zip(etags.zip(sizes.zip(storage_classes)))) - { - let sz: i64 = parse_tag_content(&size)?; - let key_text = must_get_node_text(&key)?; - let mtime_text = must_get_node_text(&mtime)?; - let etag_text = must_get_node_text(&etag)?; - let storage_class_text = must_get_node_text(&storage_class)?; - let object_info = ObjectInfo::new( - key_text, - mtime_text, - etag_text, - sz, - storage_class_text, - HashMap::new(), - )?; - object_infos.push(object_info); - } - } - Ok(object_infos) -} - -fn parse_list_objects_result(doc: roxmltree::Document) -> Result { - let root = doc.root_element(); - let bucket_name = - get_child_node(&root, "Name").ok_or(Err::XmlElemMissing("Name".to_string()))?; - let prefix = get_child_node_or(&root, "Prefix", ""); - let key_count: i32 = parse_child_content(&root, "KeyCount")?; - let max_keys: i32 = parse_child_content(&root, "MaxKeys")?; - let is_truncated: bool = parse_child_content(&root, "IsTruncated")?; - let object_infos = parse_object_infos(root)?; - - Ok(ListObjectsResp { - bucket_name: bucket_name.to_string(), - prefix: prefix.to_string(), - max_keys: max_keys, - key_count: key_count, - is_truncated: is_truncated, - next_continuation_token: "".to_string(), - common_prefixes: Vec::new(), - object_infos: object_infos, - }) -} diff --git a/src/s3/args.rs b/src/s3/args.rs new file mode 100644 index 0000000..2297ee7 --- /dev/null +++ b/src/s3/args.rs @@ -0,0 +1,981 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::error::Error; +use crate::s3::sse::{Sse, SseCustomerKey}; +use crate::s3::types::{DeleteObject, Item, Part, Retention, SelectProgress, SelectRequest}; +use crate::s3::utils::{ + check_bucket_name, merge, to_http_header_value, to_iso8601utc, urlencode, Multimap, UtcTime, +}; +use derivative::Derivative; + +const MIN_PART_SIZE: usize = 5_242_880; // 5 MiB +const MAX_PART_SIZE: usize = 5_368_709_120; // 5 GiB +const MAX_OBJECT_SIZE: usize = 5_497_558_138_880; // 5 TiB +const MAX_MULTIPART_COUNT: u16 = 10_000; + +fn object_write_args_headers( + extra_headers: Option<&Multimap>, + headers: Option<&Multimap>, + user_metadata: Option<&Multimap>, + sse: Option<&dyn Sse>, + tags: Option<&std::collections::HashMap>, + retention: Option<&Retention>, + legal_hold: bool, +) -> Multimap { + let mut map = Multimap::new(); + + if let Some(v) = extra_headers { + merge(&mut map, v); + } + + if let Some(v) = headers { + merge(&mut map, v); + } + + if let Some(v) = user_metadata { + merge(&mut map, v); + } + + if let Some(v) = sse { + merge(&mut map, &v.headers()); + } + + if let Some(v) = tags { + let mut tagging = String::new(); + for (key, value) in v.iter() { + if !tagging.is_empty() { + tagging.push_str("&"); + } + tagging.push_str(&urlencode(key)); + tagging.push_str("="); + tagging.push_str(&urlencode(value)); + } + + if !tagging.is_empty() { + map.insert(String::from("x-amz-tagging"), tagging); + } + } + + if let Some(v) = retention { + map.insert(String::from("x-amz-object-lock-mode"), v.mode.to_string()); + map.insert( + String::from("x-amz-object-lock-retain-until-date"), + to_iso8601utc(v.retain_until_date), + ); + } + + if legal_hold { + map.insert( + String::from("x-amz-object-lock-legal-hold"), + String::from("ON"), + ); + } + + return map; +} + +fn calc_part_info( + object_size: Option, + part_size: Option, +) -> Result<(usize, i16), Error> { + if let Some(v) = part_size { + if v < MIN_PART_SIZE { + return Err(Error::InvalidMinPartSize(v)); + } + + if v > MAX_PART_SIZE { + return Err(Error::InvalidMaxPartSize(v)); + } + } + + if let Some(v) = object_size { + if v > MAX_OBJECT_SIZE { + return Err(Error::InvalidObjectSize(v)); + } + } else { + if part_size.is_none() { + return Err(Error::MissingPartSize); + } + + return Ok((part_size.unwrap(), -1)); + } + + let mut psize = 0_usize; + if part_size.is_none() { + psize = (object_size.unwrap() as f64 / MAX_MULTIPART_COUNT as f64).ceil() as usize; + psize = MIN_PART_SIZE * (psize as f64 / MIN_PART_SIZE as f64).ceil() as usize; + } + + if psize > object_size.unwrap() { + psize = object_size.unwrap(); + } + + let mut part_count = 1_i16; + if psize > 0 { + part_count = (object_size.unwrap() as f64 / psize as f64).ceil() as i16; + } + + if part_count as u16 > MAX_MULTIPART_COUNT { + return Err(Error::InvalidPartCount( + object_size.unwrap(), + psize, + MAX_MULTIPART_COUNT, + )); + } + + return Ok((psize, part_count)); +} + +#[derive(Clone, Debug, Default)] +pub struct BucketArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, +} + +impl<'a> BucketArgs<'a> { + pub fn new(bucket_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(BucketArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + }) + } +} + +pub type BucketExistsArgs<'a> = BucketArgs<'a>; + +pub type RemoveBucketArgs<'a> = BucketArgs<'a>; + +#[derive(Clone, Debug, Default)] +pub struct ObjectArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, +} + +impl<'a> ObjectArgs<'a> { + pub fn new(bucket_name: &'a str, object_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + Ok(ObjectArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + }) + } +} + +#[derive(Clone, Debug, Default)] +pub struct ObjectVersionArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub version_id: Option<&'a str>, +} + +impl<'a> ObjectVersionArgs<'a> { + pub fn new(bucket_name: &'a str, object_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + Ok(ObjectVersionArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + version_id: None, + }) + } +} + +pub type RemoveObjectArgs<'a> = ObjectVersionArgs<'a>; + +#[derive(Clone, Debug, Default)] +pub struct MakeBucketArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object_lock: bool, +} + +impl<'a> MakeBucketArgs<'a> { + pub fn new(bucket_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(MakeBucketArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object_lock: false, + }) + } +} + +#[derive(Clone, Debug, Default)] +pub struct ListBucketsArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, +} + +impl<'a> ListBucketsArgs<'a> { + pub fn new() -> ListBucketsArgs<'a> { + ListBucketsArgs::default() + } +} + +#[derive(Clone, Debug, Default)] +pub struct AbortMultipartUploadArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub upload_id: &'a str, +} + +impl<'a> AbortMultipartUploadArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + upload_id: &'a str, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + if upload_id.is_empty() { + return Err(Error::InvalidUploadId(String::from( + "upload ID cannot be empty", + ))); + } + + Ok(AbortMultipartUploadArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + upload_id: upload_id, + }) + } +} + +#[derive(Clone, Debug)] +pub struct CompleteMultipartUploadArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub upload_id: &'a str, + pub parts: &'a Vec, +} + +impl<'a> CompleteMultipartUploadArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + upload_id: &'a str, + parts: &'a Vec, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + if upload_id.is_empty() { + return Err(Error::InvalidUploadId(String::from( + "upload ID cannot be empty", + ))); + } + + if parts.len() == 0 { + return Err(Error::EmptyParts(String::from("parts cannot be empty"))); + } + + Ok(CompleteMultipartUploadArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + upload_id: upload_id, + parts: parts, + }) + } +} + +#[derive(Clone, Debug, Default)] +pub struct CreateMultipartUploadArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub headers: Option<&'a Multimap>, +} + +impl<'a> CreateMultipartUploadArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + Ok(CreateMultipartUploadArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + headers: None, + }) + } +} + +#[derive(Clone, Debug, Default)] +pub struct PutObjectApiArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub headers: Option<&'a Multimap>, + pub user_metadata: Option<&'a Multimap>, + pub sse: Option<&'a dyn Sse>, + pub tags: Option<&'a std::collections::HashMap>, + pub retention: Option<&'a Retention>, + pub legal_hold: bool, + pub data: &'a [u8], + pub query_params: Option<&'a Multimap>, +} + +impl<'a> PutObjectApiArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + data: &'a [u8], + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + Ok(PutObjectApiArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + headers: None, + user_metadata: None, + sse: None, + tags: None, + retention: None, + legal_hold: false, + data: data, + query_params: None, + }) + } + + pub fn get_headers(&self) -> Multimap { + object_write_args_headers( + self.extra_headers, + self.headers, + self.user_metadata, + self.sse, + self.tags, + self.retention, + self.legal_hold, + ) + } +} + +#[derive(Clone, Debug, Default)] +pub struct UploadPartArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub headers: Option<&'a Multimap>, + pub user_metadata: Option<&'a Multimap>, + pub sse: Option<&'a dyn Sse>, + pub tags: Option<&'a std::collections::HashMap>, + pub retention: Option<&'a Retention>, + pub legal_hold: bool, + pub upload_id: &'a str, + pub part_number: u16, + pub data: &'a [u8], +} + +impl<'a> UploadPartArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + upload_id: &'a str, + part_number: u16, + data: &'a [u8], + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + if upload_id.is_empty() { + return Err(Error::InvalidUploadId(String::from( + "upload ID cannot be empty", + ))); + } + + if part_number < 1 || part_number > 10000 { + return Err(Error::InvalidPartNumber(String::from( + "part number must be between 1 and 1000", + ))); + } + + Ok(UploadPartArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + headers: None, + user_metadata: None, + sse: None, + tags: None, + retention: None, + legal_hold: false, + upload_id: upload_id, + part_number: part_number, + data: data, + }) + } + + pub fn get_headers(&self) -> Multimap { + object_write_args_headers( + self.extra_headers, + self.headers, + self.user_metadata, + self.sse, + self.tags, + self.retention, + self.legal_hold, + ) + } +} + +pub struct PutObjectArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub headers: Option<&'a Multimap>, + pub user_metadata: Option<&'a Multimap>, + pub sse: Option<&'a dyn Sse>, + pub tags: Option<&'a std::collections::HashMap>, + pub retention: Option<&'a Retention>, + pub legal_hold: bool, + pub object_size: Option, + pub part_size: usize, + pub part_count: i16, + pub content_type: &'a str, + pub stream: &'a mut dyn std::io::Read, +} + +impl<'a> PutObjectArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + stream: &'a mut dyn std::io::Read, + object_size: Option, + part_size: Option, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + let (psize, part_count) = calc_part_info(object_size, part_size)?; + + Ok(PutObjectArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + headers: None, + user_metadata: None, + sse: None, + tags: None, + retention: None, + legal_hold: false, + object_size: object_size, + part_size: psize, + part_count: part_count, + content_type: "application/octet-stream", + stream: stream, + }) + } + + pub fn get_headers(&self) -> Multimap { + object_write_args_headers( + self.extra_headers, + self.headers, + self.user_metadata, + self.sse, + self.tags, + self.retention, + self.legal_hold, + ) + } +} + +#[derive(Clone, Debug, Default)] +pub struct ObjectConditionalReadArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub version_id: Option<&'a str>, + pub ssec: Option<&'a SseCustomerKey>, + pub offset: Option, + pub length: Option, + pub match_etag: Option<&'a str>, + pub not_match_etag: Option<&'a str>, + pub modified_since: Option, + pub unmodified_since: Option, +} + +impl<'a> ObjectConditionalReadArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + Ok(ObjectConditionalReadArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + version_id: None, + ssec: None, + offset: None, + length: None, + match_etag: None, + not_match_etag: None, + modified_since: None, + unmodified_since: None, + }) + } + + pub fn get_headers(&self) -> Multimap { + let (offset, length) = match self.length { + Some(_) => (Some(self.offset.unwrap_or(0_usize)), self.length), + None => (self.offset, None), + }; + + let mut range = String::new(); + if let Some(o) = offset { + range.push_str("bytes="); + range.push_str(&o.to_string()); + range.push_str("-"); + if let Some(l) = length { + range.push_str(&(o + l - 1).to_string()); + } + } + + let mut headers = Multimap::new(); + if !range.is_empty() { + headers.insert(String::from("Range"), range.clone()); + } + + if let Some(v) = self.match_etag { + headers.insert(String::from("if-match"), v.to_string()); + } + + if let Some(v) = self.not_match_etag { + headers.insert(String::from("if-none-match"), v.to_string()); + } + + if let Some(v) = self.modified_since { + headers.insert(String::from("if-modified-since"), to_http_header_value(v)); + } + + if let Some(v) = self.unmodified_since { + headers.insert(String::from("if-unmodified-since"), to_http_header_value(v)); + } + + if let Some(v) = self.ssec { + merge(&mut headers, &v.headers()); + } + + return headers; + } + + pub fn get_copy_headers(&self) -> Multimap { + let mut headers = Multimap::new(); + + let mut copy_source = String::from("/"); + copy_source.push_str(self.bucket); + copy_source.push_str("/"); + copy_source.push_str(self.object); + if let Some(v) = self.version_id { + copy_source.push_str("?versionId="); + copy_source.push_str(&urlencode(v)); + } + headers.insert(String::from("x-amz-copy-source"), copy_source.to_string()); + + if let Some(v) = self.match_etag { + headers.insert(String::from("x-amz-copy-source-if-match"), v.to_string()); + } + + if let Some(v) = self.not_match_etag { + headers.insert( + String::from("x-amz-copy-source-if-none-match"), + v.to_string(), + ); + } + + if let Some(v) = self.modified_since { + headers.insert( + String::from("x-amz-copy-source-if-modified-since"), + to_http_header_value(v), + ); + } + + if let Some(v) = self.unmodified_since { + headers.insert( + String::from("x-amz-copy-source-if-unmodified-since"), + to_http_header_value(v), + ); + } + + if let Some(v) = self.ssec { + merge(&mut headers, &v.copy_headers()); + } + + return headers; + } +} + +pub type GetObjectArgs<'a> = ObjectConditionalReadArgs<'a>; + +pub type StatObjectArgs<'a> = ObjectConditionalReadArgs<'a>; + +#[derive(Derivative, Clone, Debug, Default)] +pub struct RemoveObjectsApiArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub bypass_governance_mode: bool, + #[derivative(Default(value = "true"))] + pub quiet: bool, + pub objects: &'a [DeleteObject<'a>], +} + +impl<'a> RemoveObjectsApiArgs<'a> { + pub fn new( + bucket_name: &'a str, + objects: &'a [DeleteObject], + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(RemoveObjectsApiArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + bypass_governance_mode: false, + quiet: true, + objects: objects, + }) + } +} + +pub struct RemoveObjectsArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub bypass_governance_mode: bool, + pub objects: &'a mut core::slice::Iter<'a, DeleteObject<'a>>, +} + +impl<'a> RemoveObjectsArgs<'a> { + pub fn new( + bucket_name: &'a str, + objects: &'a mut core::slice::Iter<'a, DeleteObject<'a>>, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(RemoveObjectsArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + bypass_governance_mode: false, + objects: objects, + }) + } +} + +pub struct ListObjectsV1Args<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub delimiter: Option<&'a str>, + pub encoding_type: Option<&'a str>, + pub max_keys: Option, + pub prefix: Option<&'a str>, + pub marker: Option, +} + +impl<'a> ListObjectsV1Args<'a> { + pub fn new(bucket_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(ListObjectsV1Args { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + delimiter: None, + encoding_type: None, + max_keys: None, + prefix: None, + marker: None, + }) + } +} + +pub struct ListObjectsV2Args<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub delimiter: Option<&'a str>, + pub encoding_type: Option<&'a str>, + pub max_keys: Option, + pub prefix: Option<&'a str>, + pub start_after: Option, + pub continuation_token: Option, + pub fetch_owner: bool, + pub include_user_metadata: bool, +} + +impl<'a> ListObjectsV2Args<'a> { + pub fn new(bucket_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(ListObjectsV2Args { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + delimiter: None, + encoding_type: None, + max_keys: None, + prefix: None, + start_after: None, + continuation_token: None, + fetch_owner: false, + include_user_metadata: false, + }) + } +} + +pub struct ListObjectVersionsArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub delimiter: Option<&'a str>, + pub encoding_type: Option<&'a str>, + pub max_keys: Option, + pub prefix: Option<&'a str>, + pub key_marker: Option, + pub version_id_marker: Option, +} + +impl<'a> ListObjectVersionsArgs<'a> { + pub fn new(bucket_name: &'a str) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(ListObjectVersionsArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + delimiter: None, + encoding_type: None, + max_keys: None, + prefix: None, + key_marker: None, + version_id_marker: None, + }) + } +} + +pub struct ListObjectsArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub delimiter: Option<&'a str>, + pub use_url_encoding_type: bool, + pub marker: Option<&'a str>, // only for ListObjectsV1. + pub start_after: Option<&'a str>, // only for ListObjectsV2. + pub key_marker: Option<&'a str>, // only for GetObjectVersions. + pub max_keys: Option, + pub prefix: Option<&'a str>, + pub continuation_token: Option<&'a str>, // only for ListObjectsV2. + pub fetch_owner: bool, // only for ListObjectsV2. + pub version_id_marker: Option<&'a str>, // only for GetObjectVersions. + pub include_user_metadata: bool, // MinIO extension for ListObjectsV2. + pub recursive: bool, + pub use_api_v1: bool, + pub include_versions: bool, + pub result_fn: &'a dyn Fn(Result<&Item, Error>) -> bool, +} + +impl<'a> ListObjectsArgs<'a> { + pub fn new( + bucket_name: &'a str, + result_fn: &'a dyn Fn(Result<&Item, Error>) -> bool, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + Ok(ListObjectsArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + delimiter: None, + use_url_encoding_type: true, + marker: None, + start_after: None, + key_marker: None, + max_keys: None, + prefix: None, + continuation_token: None, + fetch_owner: false, + version_id_marker: None, + include_user_metadata: false, + recursive: false, + use_api_v1: false, + include_versions: false, + result_fn: result_fn, + }) + } +} + +pub struct SelectObjectContentArgs<'a> { + pub extra_headers: Option<&'a Multimap>, + pub extra_query_params: Option<&'a Multimap>, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub object: &'a str, + pub version_id: Option<&'a str>, + pub ssec: Option<&'a SseCustomerKey>, + pub request: &'a SelectRequest<'a>, +} + +impl<'a> SelectObjectContentArgs<'a> { + pub fn new( + bucket_name: &'a str, + object_name: &'a str, + request: &'a SelectRequest, + ) -> Result, Error> { + check_bucket_name(bucket_name, true)?; + + if object_name.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + + Ok(SelectObjectContentArgs { + extra_headers: None, + extra_query_params: None, + region: None, + bucket: bucket_name, + object: object_name, + version_id: None, + ssec: None, + request: request, + }) + } +} diff --git a/src/s3/client.rs b/src/s3/client.rs new file mode 100644 index 0000000..7779f1c --- /dev/null +++ b/src/s3/client.rs @@ -0,0 +1,1888 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::args::*; +use crate::s3::creds::Provider; +use crate::s3::error::{Error, ErrorResponse}; +use crate::s3::http::{BaseUrl, Url}; +use crate::s3::response::*; +use crate::s3::signer::sign_v4_s3; +use crate::s3::sse::SseCustomerKey; +use crate::s3::types::{Bucket, DeleteObject, Item, Part}; +use crate::s3::utils::{ + from_iso8601utc, get_default_text, get_option_text, get_text, md5sum_hash, merge, sha256_hash, + to_amz_date, urldecode, utc_now, Multimap, +}; +use bytes::{Buf, Bytes}; +use dashmap::DashMap; +use hyper::http::Method; +use reqwest::header::HeaderMap; +use std::collections::HashMap; +use xmltree::Element; + +fn url_decode( + encoding_type: &Option, + prefix: Option, +) -> Result, Error> { + if let Some(v) = encoding_type.as_ref() { + if v == "url" { + if let Some(v) = prefix { + return Ok(Some(urldecode(&v)?.to_string())); + } + } + } + + if let Some(v) = prefix.as_ref() { + return Ok(Some(v.to_string())); + } + + return Ok(None); +} + +fn add_common_list_objects_query_params( + query_params: &mut Multimap, + delimiter: Option<&str>, + encoding_type: Option<&str>, + max_keys: Option, + prefix: Option<&str>, +) { + query_params.insert( + String::from("delimiter"), + delimiter.unwrap_or("").to_string(), + ); + query_params.insert( + String::from("max-keys"), + max_keys.unwrap_or(1000).to_string(), + ); + query_params.insert(String::from("prefix"), prefix.unwrap_or("").to_string()); + if let Some(v) = encoding_type { + query_params.insert(String::from("encoding-type"), v.to_string()); + } +} + +fn parse_common_list_objects_response( + root: &Element, +) -> Result< + ( + String, + Option, + Option, + Option, + bool, + Option, + ), + Error, +> { + let encoding_type = get_option_text(&root, "EncodingType")?; + let prefix = url_decode(&encoding_type, Some(get_default_text(&root, "Prefix")))?; + Ok(( + get_text(&root, "Name")?, + encoding_type, + prefix, + get_option_text(&root, "Delimiter")?, + match get_option_text(&root, "IsTruncated")? { + Some(v) => v.to_lowercase() == "true", + None => false, + }, + match get_option_text(&root, "MaxKeys")? { + Some(v) => Some(v.parse::()?), + None => None, + }, + )) +} + +fn parse_list_objects_contents( + contents: &mut Vec, + root: &mut xmltree::Element, + tag: &str, + encoding_type: &Option, + is_delete_marker: bool, +) -> Result<(), Error> { + loop { + let content = match root.take_child(tag) { + Some(v) => v, + None => break, + }; + + let etype = encoding_type.as_ref().map(|v| v.clone()); + let key = url_decode(&etype, Some(get_text(&content, "Key")?))?.unwrap(); + let last_modified = Some(from_iso8601utc(&get_text(&content, "LastModified")?)?); + let etag = get_option_text(&content, "ETag")?; + let v = get_default_text(&content, "Size"); + let size = match v.is_empty() { + true => None, + false => Some(v.parse::()?), + }; + let storage_class = get_option_text(&content, "StorageClass")?; + let is_latest = get_default_text(&content, "IsLatest").to_lowercase() == "true"; + let version_id = get_option_text(&content, "VersionId")?; + let (owner_id, owner_name) = match content.get_child("Owner") { + Some(v) => ( + get_option_text(&v, "ID")?, + get_option_text(&v, "DisplayName")?, + ), + None => (None, None), + }; + let user_metadata = match content.get_child("UserMetadata") { + Some(v) => { + let mut map: HashMap = HashMap::new(); + for node in v.children.iter() { + let e = node.as_element().unwrap(); + map.insert(e.name.clone(), e.get_text().unwrap_or_default().to_string()); + } + Some(map) + } + None => None, + }; + + contents.push(Item { + name: key, + last_modified: last_modified, + etag: etag, + owner_id: owner_id, + owner_name: owner_name, + size: size, + storage_class: storage_class, + is_latest: is_latest, + version_id: version_id, + user_metadata: user_metadata, + is_prefix: false, + is_delete_marker: is_delete_marker, + encoding_type: etype, + }); + } + + Ok(()) +} + +fn parse_list_objects_common_prefixes( + contents: &mut Vec, + root: &mut Element, + encoding_type: &Option, +) -> Result<(), Error> { + loop { + let common_prefix = match root.take_child("CommonPrefixes") { + Some(v) => v, + None => break, + }; + + contents.push(Item { + name: url_decode(&encoding_type, Some(get_text(&common_prefix, "Prefix")?))?.unwrap(), + last_modified: None, + etag: None, + owner_id: None, + owner_name: None, + size: None, + storage_class: None, + is_latest: false, + version_id: None, + user_metadata: None, + is_prefix: true, + is_delete_marker: false, + encoding_type: encoding_type.as_ref().map(|v| v.clone()), + }); + } + + Ok(()) +} + +#[derive(Clone, Debug, Default)] +pub struct Client<'a> { + base_url: BaseUrl, + provider: Option<&'a dyn Provider>, + user_agent: String, + debug: bool, + ignore_cert_check: bool, + ssl_cert_file: String, + region_map: DashMap, +} + +impl<'a> Client<'a> { + pub fn new(base_url: BaseUrl, provider: Option<&dyn Provider>) -> Client { + Client { + base_url: base_url, + provider: provider, + user_agent: String::new(), + debug: false, + ignore_cert_check: false, + ssl_cert_file: String::new(), + region_map: DashMap::new(), + } + } + + fn build_headers( + &self, + headers: &mut Multimap, + query_params: &Multimap, + region: &String, + url: &Url, + method: &Method, + data: &[u8], + ) { + headers.insert(String::from("Host"), url.host_header_value()); + headers.insert( + String::from("User-Agent"), + String::from("MinIO (Linux; x86_64) minio-rs/0.1.0"), + ); + + let mut md5sum = String::new(); + let mut sha256 = String::new(); + match *method { + Method::PUT | Method::POST => { + headers.insert(String::from("Content-Length"), data.len().to_string()); + if !headers.contains_key("Content-Type") { + headers.insert( + String::from("Content-Type"), + String::from("application/octet-stream"), + ); + } + if self.provider.is_some() { + sha256 = sha256_hash(data); + } else if !headers.contains_key("Content-MD5") { + md5sum = md5sum_hash(data); + } + } + _ => { + if self.provider.is_some() { + sha256 = String::from( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ); + } + } + }; + if !md5sum.is_empty() { + headers.insert(String::from("Content-MD5"), md5sum); + } + if !sha256.is_empty() { + headers.insert(String::from("x-amz-content-sha256"), sha256.clone()); + } + let date = utc_now(); + headers.insert(String::from("x-amz-date"), to_amz_date(date)); + + match self.provider { + Some(p) => { + let creds = p.fetch(); + if creds.session_token.is_some() { + headers.insert( + String::from("X-Amz-Security-Token"), + creds.session_token.unwrap(), + ); + } + sign_v4_s3( + &method, + &url.path, + region, + headers, + query_params, + &creds.access_key, + &creds.secret_key, + &sha256, + date, + ); + } + _ => todo!(), // Nothing to do for anonymous request + }; + } + + fn handle_redirect_response( + &self, + status_code: u16, + method: &Method, + header_map: &reqwest::header::HeaderMap, + bucket_name: Option<&str>, + retry: bool, + ) -> Result<(String, String), Error> { + let (mut code, mut message) = match status_code { + 301 => ( + String::from("PermanentRedirect"), + String::from("Moved Permanently"), + ), + 307 => (String::from("Redirect"), String::from("Temporary redirect")), + 400 => (String::from("BadRequest"), String::from("Bad request")), + _ => (String::new(), String::new()), + }; + + let region = match header_map.get("x-amz-bucket-region") { + Some(v) => v.to_str()?, + _ => "", + }; + + if !message.is_empty() && !region.is_empty() { + message.push_str("; use region "); + message.push_str(region); + } + + if retry && !region.is_empty() && method == Method::HEAD { + if let Some(v) = bucket_name { + if self.region_map.contains_key(v) { + code = String::from("RetryHead"); + message = String::new(); + } + } + } + + return Ok((code, message)); + } + + fn get_error_response( + &self, + body: &mut Bytes, + status_code: u16, + header_map: &reqwest::header::HeaderMap, + method: &Method, + resource: &str, + bucket_name: Option<&str>, + object_name: Option<&str>, + retry: bool, + ) -> Error { + if body.len() > 0 { + return match header_map.get("Content-Type") { + Some(v) => match v.to_str() { + Ok(s) => match s.to_lowercase().contains("application/xml") { + true => match ErrorResponse::parse(body) { + Ok(v) => Error::S3Error(v), + Err(e) => e, + }, + false => Error::InvalidResponse(status_code, s.to_string()), + }, + Err(e) => return Error::StrError(e), + }, + _ => Error::InvalidResponse(status_code, String::new()), + }; + } + + let (code, message) = match status_code { + 301 | 307 | 400 => match self.handle_redirect_response( + status_code, + method, + header_map, + bucket_name, + retry, + ) { + Ok(v) => v, + Err(e) => return e, + }, + 403 => (String::from("AccessDenied"), String::from("Access denied")), + 404 => match object_name { + Some(_) => ( + String::from("NoSuchKey"), + String::from("Object does not exist"), + ), + _ => match bucket_name { + Some(_) => ( + String::from("NoSuchBucket"), + String::from("Bucket does not exist"), + ), + _ => ( + String::from("ResourceNotFound"), + String::from("Request resource not found"), + ), + }, + }, + 405 => ( + String::from("MethodNotAllowed"), + String::from("The specified method is not allowed against this resource"), + ), + 409 => match bucket_name { + Some(_) => ( + String::from("NoSuchBucket"), + String::from("Bucket does not exist"), + ), + _ => ( + String::from("ResourceConflict"), + String::from("Request resource conflicts"), + ), + }, + 501 => ( + String::from("MethodNotAllowed"), + String::from("The specified method is not allowed against this resource"), + ), + _ => return Error::ServerError(status_code), + }; + + let request_id = match header_map.get("x-amz-request-id") { + Some(v) => match v.to_str() { + Ok(s) => s.to_string(), + Err(e) => return Error::StrError(e), + }, + _ => String::new(), + }; + + let host_id = match header_map.get("x-amz-id-2") { + Some(v) => match v.to_str() { + Ok(s) => s.to_string(), + Err(e) => return Error::StrError(e), + }, + _ => String::new(), + }; + + Error::S3Error(ErrorResponse { + code: code, + message: message, + resource: resource.to_string(), + request_id: request_id, + host_id: host_id, + bucket_name: bucket_name.unwrap_or_default().to_string(), + object_name: object_name.unwrap_or_default().to_string(), + }) + } + + pub async fn do_execute( + &self, + method: Method, + region: &String, + headers: &mut Multimap, + query_params: &Multimap, + bucket_name: Option<&str>, + object_name: Option<&str>, + data: Option<&[u8]>, + retry: bool, + ) -> Result { + let body = data.unwrap_or_default(); + let url = + self.base_url + .build_url(&method, region, query_params, bucket_name, object_name)?; + self.build_headers(headers, query_params, region, &url, &method, body); + + let client = reqwest::Client::new(); + let mut req = client.request(method.clone(), url.to_string()); + + for (key, values) in headers.iter_all() { + for value in values { + req = req.header(key, value); + } + } + + if method == Method::PUT || method == Method::POST { + req = req.body(body.to_vec()); + } + + let resp = req.send().await?; + if resp.status().is_success() { + return Ok(resp); + } + + let status_code = resp.status().as_u16(); + let header_map = resp.headers().clone(); + let mut body = resp.bytes().await?; + let e = self.get_error_response( + &mut body, + status_code, + &header_map, + &method, + &url.path, + bucket_name, + object_name, + retry, + ); + + match e { + Error::S3Error(ref er) => { + if er.code == "NoSuchBucket" || er.code == "RetryHead" { + if let Some(v) = bucket_name { + self.region_map.remove(v); + } + } + } + _ => todo!(), // Nothing to do. + }; + + return Err(e); + } + + pub async fn execute( + &self, + method: Method, + region: &String, + headers: &mut Multimap, + query_params: &Multimap, + bucket_name: Option<&str>, + object_name: Option<&str>, + data: Option<&[u8]>, + ) -> Result { + let res = self + .do_execute( + method.clone(), + region, + headers, + query_params, + bucket_name, + object_name, + data, + true, + ) + .await; + match res { + Ok(r) => return Ok(r), + Err(e) => match e { + Error::S3Error(ref er) => { + if er.code != "RetryHead" { + return Err(e); + } + } + _ => return Err(e), + }, + }; + + // Retry only once on RetryHead error. + self.do_execute( + method.clone(), + region, + headers, + query_params, + bucket_name, + object_name, + data, + false, + ) + .await + } + + pub async fn get_region( + &self, + bucket_name: &str, + region: Option<&str>, + ) -> Result { + if !region.map_or(true, |v| v.is_empty()) { + if !self.base_url.region.is_empty() && self.base_url.region != *region.unwrap() { + return Err(Error::RegionMismatch( + self.base_url.region.clone(), + region.unwrap().to_string(), + )); + } + + return Ok(region.unwrap().to_string()); + } + + if !self.base_url.region.is_empty() { + return Ok(self.base_url.region.clone()); + } + + if bucket_name.is_empty() || self.provider.is_none() { + return Ok(String::from("us-east-1")); + } + + if let Some(v) = self.region_map.get(bucket_name) { + return Ok((*v).to_string()); + } + + let mut headers = Multimap::new(); + let mut query_params = Multimap::new(); + query_params.insert(String::from("location"), String::new()); + + let resp = self + .execute( + Method::GET, + &String::from("us-east-1"), + &mut headers, + &query_params, + Some(bucket_name), + None, + None, + ) + .await?; + let body = resp.bytes().await?; + let root = Element::parse(body.reader())?; + + let location = root.get_text().unwrap_or_default().to_string(); + self.region_map + .insert(bucket_name.to_string(), location.clone()); + Ok(location) + } + + pub async fn abort_multipart_upload( + &self, + args: &AbortMultipartUploadArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("uploadId"), args.upload_id.to_string()); + + let resp = self + .execute( + Method::DELETE, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + None, + ) + .await?; + + Ok(AbortMultipartUploadResponse { + headers: resp.headers().clone(), + region: region.clone(), + bucket_name: args.bucket.to_string(), + object_name: args.object.to_string(), + upload_id: args.upload_id.to_string(), + }) + } + + pub async fn bucket_exists(&self, args: &BucketExistsArgs<'_>) -> Result { + let region; + match self.get_region(&args.bucket, args.region).await { + Ok(r) => region = r, + Err(e) => match e { + Error::S3Error(ref er) => { + if er.code == "NoSuchBucket" { + return Ok(false); + } + return Err(e); + } + _ => return Err(e), + }, + }; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + let mut query_params = &Multimap::new(); + if let Some(v) = &args.extra_query_params { + query_params = v; + } + + match self + .execute( + Method::HEAD, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + None, + None, + ) + .await + { + Ok(_) => Ok(true), + Err(e) => match e { + Error::S3Error(ref er) => { + if er.code == "NoSuchBucket" { + return Ok(false); + } + return Err(e); + } + _ => return Err(e), + }, + } + } + + pub async fn complete_multipart_upload( + &self, + args: &CompleteMultipartUploadArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut data = String::from(""); + for part in args.parts.iter() { + let s = format!( + "{}{}", + part.number, part.etag + ); + data.push_str(&s); + } + data.push_str(""); + let b = data.as_bytes(); + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + headers.insert( + String::from("Content-Type"), + String::from("application/xml"), + ); + headers.insert(String::from("Content-MD5"), md5sum_hash(b)); + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("uploadId"), args.upload_id.to_string()); + + let resp = self + .execute( + Method::POST, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + Some(&b), + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let root = Element::parse(body.reader())?; + + Ok(CompleteMultipartUploadResponse { + headers: header_map.clone(), + bucket_name: get_text(&root, "Bucket")?, + object_name: get_text(&root, "Key")?, + location: get_text(&root, "Location")?, + etag: get_text(&root, "ETag")?.trim_matches('"').to_string(), + version_id: match header_map.get("x-amz-version-id") { + Some(v) => Some(v.to_str()?.to_string()), + None => None, + }, + }) + } + + pub async fn create_multipart_upload( + &self, + args: &CreateMultipartUploadArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + if !headers.contains_key("Content-Type") { + headers.insert( + String::from("Content-Type"), + String::from("application/octet-stream"), + ); + } + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("uploads"), String::new()); + + let resp = self + .execute( + Method::POST, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + None, + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let root = Element::parse(body.reader())?; + + Ok(CreateMultipartUploadResponse { + headers: header_map.clone(), + region: region.clone(), + bucket_name: args.bucket.to_string(), + object_name: args.object.to_string(), + upload_id: get_text(&root, "UploadId")?, + }) + } + + // DeleteBucketEncryptionResponse DeleteBucketEncryption( + // DeleteBucketEncryptionArgs args); + // DisableObjectLegalHoldResponse DisableObjectLegalHold( + // DisableObjectLegalHoldArgs args); + // DeleteBucketLifecycleResponse DeleteBucketLifecycle( + // DeleteBucketLifecycleArgs args); + // DeleteBucketNotificationResponse DeleteBucketNotification( + // DeleteBucketNotificationArgs args); + // DeleteBucketPolicyResponse DeleteBucketPolicy(DeleteBucketPolicyArgs args); + // DeleteBucketReplicationResponse DeleteBucketReplication( + // DeleteBucketReplicationArgs args); + // DeleteBucketTagsResponse DeleteBucketTags(DeleteBucketTagsArgs args); + // DeleteObjectLockConfigResponse DeleteObjectLockConfig( + // DeleteObjectLockConfigArgs args); + // DeleteObjectTagsResponse DeleteObjectTags(DeleteObjectTagsArgs args); + // EnableObjectLegalHoldResponse EnableObjectLegalHold( + // EnableObjectLegalHoldArgs args); + // GetBucketEncryptionResponse GetBucketEncryption(GetBucketEncryptionArgs args); + // GetBucketLifecycleResponse GetBucketLifecycle(GetBucketLifecycleArgs args); + // GetBucketNotificationResponse GetBucketNotification( + // GetBucketNotificationArgs args); + // GetBucketPolicyResponse GetBucketPolicy(GetBucketPolicyArgs args); + // GetBucketReplicationResponse GetBucketReplication( + // GetBucketReplicationArgs args); + // GetBucketTagsResponse GetBucketTags(GetBucketTagsArgs args); + // GetBucketVersioningResponse GetBucketVersioning(GetBucketVersioningArgs args); + + pub async fn get_object(&self, args: &GetObjectArgs<'_>) -> Result { + if args.ssec.is_some() && !self.base_url.https { + return Err(Error::SseTlsRequired); + } + + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + merge(&mut headers, &args.get_headers()); + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + if let Some(v) = args.version_id { + query_params.insert(String::from("versionId"), v.to_string()); + } + + self.execute( + Method::GET, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + None, + ) + .await + } + + // GetObjectLockConfigResponse GetObjectLockConfig(GetObjectLockConfigArgs args); + // GetObjectRetentionResponse GetObjectRetention(GetObjectRetentionArgs args); + // GetObjectTagsResponse GetObjectTags(GetObjectTagsArgs args); + // GetPresignedObjectUrlResponse GetPresignedObjectUrl( + // GetPresignedObjectUrlArgs args); + // GetPresignedPostFormDataResponse GetPresignedPostFormData(PostPolicy policy); + // IsObjectLegalHoldEnabledResponse IsObjectLegalHoldEnabled( + // IsObjectLegalHoldEnabledArgs args); + + pub async fn list_buckets( + &self, + args: &ListBucketsArgs<'_>, + ) -> Result { + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + let mut query_params = &Multimap::new(); + if let Some(v) = &args.extra_query_params { + query_params = v; + } + let resp = self + .execute( + Method::GET, + &String::from("us-east-1"), + &mut headers, + &query_params, + None, + None, + None, + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let mut root = Element::parse(body.reader())?; + let buckets = root + .get_mut_child("Buckets") + .ok_or(Error::XmlError(String::from(" tag not found")))?; + + let mut bucket_list: Vec = Vec::new(); + loop { + let bucket = match buckets.take_child("Bucket") { + Some(b) => b, + None => break, + }; + + bucket_list.push(Bucket { + name: get_text(&bucket, "Name")?, + creation_date: from_iso8601utc(&get_text(&bucket, "CreationDate")?)?, + }) + } + + Ok(ListBucketsResponse { + headers: header_map.clone(), + buckets: bucket_list, + }) + } + + // ListenBucketNotificationResponse ListenBucketNotification( + // ListenBucketNotificationArgs args); + pub async fn list_objects_v1( + &self, + args: &ListObjectsV1Args<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + add_common_list_objects_query_params( + &mut query_params, + args.delimiter, + args.encoding_type, + args.max_keys, + args.prefix, + ); + if let Some(v) = &args.marker { + query_params.insert(String::from("marker"), v.to_string()); + } + + let resp = self + .execute( + Method::GET, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + None, + None, + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let mut root = Element::parse(body.reader())?; + + let (name, encoding_type, prefix, delimiter, is_truncated, max_keys) = + parse_common_list_objects_response(&root)?; + let marker = url_decode(&encoding_type, get_option_text(&root, "Marker")?)?; + let mut next_marker = url_decode(&encoding_type, get_option_text(&root, "NextMarker")?)?; + let mut contents: Vec = Vec::new(); + parse_list_objects_contents(&mut contents, &mut root, "Contents", &encoding_type, false)?; + if is_truncated && next_marker.is_none() { + next_marker = match contents.last() { + Some(v) => Some(v.name.clone()), + None => None, + } + } + parse_list_objects_common_prefixes(&mut contents, &mut root, &encoding_type)?; + + Ok(ListObjectsV1Response { + headers: header_map, + name: name, + encoding_type: encoding_type, + prefix: prefix, + delimiter: delimiter, + is_truncated: is_truncated, + max_keys: max_keys, + contents: contents, + marker: marker, + next_marker: next_marker, + }) + } + + pub async fn list_objects_v2( + &self, + args: &ListObjectsV2Args<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("list-type"), String::from("2")); + add_common_list_objects_query_params( + &mut query_params, + args.delimiter, + args.encoding_type, + args.max_keys, + args.prefix, + ); + if let Some(v) = &args.continuation_token { + query_params.insert(String::from("continuation-token"), v.to_string()); + } + if args.fetch_owner { + query_params.insert(String::from("fetch-owner"), String::from("true")); + } + if let Some(v) = &args.start_after { + query_params.insert(String::from("start-after"), v.to_string()); + } + if args.include_user_metadata { + query_params.insert(String::from("metadata"), String::from("true")); + } + + let resp = self + .execute( + Method::GET, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + None, + None, + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let mut root = Element::parse(body.reader())?; + + let (name, encoding_type, prefix, delimiter, is_truncated, max_keys) = + parse_common_list_objects_response(&root)?; + let text = get_option_text(&root, "KeyCount")?; + let key_count = match text { + Some(v) => match v.is_empty() { + true => None, + false => Some(v.parse::()?), + }, + None => None, + }; + let start_after = url_decode(&encoding_type, get_option_text(&root, "StartAfter")?)?; + let continuation_token = get_option_text(&root, "ContinuationToken")?; + let next_continuation_token = get_option_text(&root, "NextContinuationToken")?; + let mut contents: Vec = Vec::new(); + parse_list_objects_contents(&mut contents, &mut root, "Contents", &encoding_type, false)?; + parse_list_objects_common_prefixes(&mut contents, &mut root, &encoding_type)?; + + Ok(ListObjectsV2Response { + headers: header_map, + name: name, + encoding_type: encoding_type, + prefix: prefix, + delimiter: delimiter, + is_truncated: is_truncated, + max_keys: max_keys, + contents: contents, + key_count: key_count, + start_after: start_after, + continuation_token: continuation_token, + next_continuation_token: next_continuation_token, + }) + } + + pub async fn list_object_versions( + &self, + args: &ListObjectVersionsArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("versions"), String::new()); + add_common_list_objects_query_params( + &mut query_params, + args.delimiter, + args.encoding_type, + args.max_keys, + args.prefix, + ); + if let Some(v) = &args.key_marker { + query_params.insert(String::from("key-marker"), v.to_string()); + } + if let Some(v) = &args.version_id_marker { + query_params.insert(String::from("version-id-marker"), v.to_string()); + } + + let resp = self + .execute( + Method::GET, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + None, + None, + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let mut root = Element::parse(body.reader())?; + + let (name, encoding_type, prefix, delimiter, is_truncated, max_keys) = + parse_common_list_objects_response(&root)?; + let key_marker = url_decode(&encoding_type, get_option_text(&root, "KeyMarker")?)?; + let next_key_marker = url_decode(&encoding_type, get_option_text(&root, "NextKeyMarker")?)?; + let version_id_marker = get_option_text(&root, "VersionIdMarker")?; + let next_version_id_marker = get_option_text(&root, "NextVersionIdMarker")?; + let mut contents: Vec = Vec::new(); + parse_list_objects_contents(&mut contents, &mut root, "Version", &encoding_type, false)?; + parse_list_objects_common_prefixes(&mut contents, &mut root, &encoding_type)?; + parse_list_objects_contents( + &mut contents, + &mut root, + "DeleteMarker", + &encoding_type, + true, + )?; + + Ok(ListObjectVersionsResponse { + headers: header_map, + name: name, + encoding_type: encoding_type, + prefix: prefix, + delimiter: delimiter, + is_truncated: is_truncated, + max_keys: max_keys, + contents: contents, + key_marker: key_marker, + next_key_marker: next_key_marker, + version_id_marker: version_id_marker, + next_version_id_marker: next_version_id_marker, + }) + } + + pub async fn list_objects(&self, args: &ListObjectsArgs<'_>) -> Result<(), Error> { + let mut lov1_args = ListObjectsV1Args::new(&args.bucket)?; + lov1_args.extra_headers = args.extra_headers; + lov1_args.extra_query_params = args.extra_query_params; + lov1_args.region = args.region; + if args.recursive { + lov1_args.delimiter = None; + } else { + lov1_args.delimiter = Some(args.delimiter.unwrap_or("/")); + } + lov1_args.encoding_type = match args.use_url_encoding_type { + true => Some("url"), + false => None, + }; + lov1_args.max_keys = args.max_keys; + lov1_args.prefix = args.prefix; + lov1_args.marker = args.marker.map(|x| x.to_string()); + + let mut lov2_args = ListObjectsV2Args::new(&args.bucket)?; + lov2_args.extra_headers = args.extra_headers; + lov2_args.extra_query_params = args.extra_query_params; + lov2_args.region = args.region; + if args.recursive { + lov2_args.delimiter = None; + } else { + lov2_args.delimiter = Some(args.delimiter.unwrap_or("/")); + } + lov2_args.encoding_type = match args.use_url_encoding_type { + true => Some("url"), + false => None, + }; + lov2_args.max_keys = args.max_keys; + lov2_args.prefix = args.prefix; + lov2_args.start_after = args.start_after.map(|x| x.to_string()); + lov2_args.continuation_token = args.continuation_token.map(|x| x.to_string()); + lov2_args.fetch_owner = args.fetch_owner; + lov2_args.include_user_metadata = args.include_user_metadata; + + let mut lov_args = ListObjectVersionsArgs::new(&args.bucket)?; + lov_args.extra_headers = args.extra_headers; + lov_args.extra_query_params = args.extra_query_params; + lov_args.region = args.region; + if args.recursive { + lov_args.delimiter = None; + } else { + lov_args.delimiter = Some(args.delimiter.unwrap_or("/")); + } + lov_args.encoding_type = match args.use_url_encoding_type { + true => Some("url"), + false => None, + }; + lov_args.max_keys = args.max_keys; + lov_args.prefix = args.prefix; + lov_args.key_marker = args.key_marker.map(|x| x.to_string()); + lov_args.version_id_marker = args.version_id_marker.map(|x| x.to_string()); + + let mut stop = false; + while !stop { + if args.include_versions { + let resp = self.list_object_versions(&lov_args).await; + match resp { + Ok(v) => { + if v.is_truncated { + lov_args.key_marker = v.next_key_marker; + lov_args.version_id_marker = v.next_version_id_marker; + } else { + stop = true; + } + for item in v.contents.iter() { + if !(args.result_fn)(Ok(item)) { + stop = true; + break; + } + } + } + Err(e) => { + (args.result_fn)(Err(e)); + return Ok(()); + } + }; + } else if args.use_api_v1 { + let resp = self.list_objects_v1(&lov1_args).await; + match resp { + Ok(v) => { + if v.is_truncated { + lov1_args.marker = v.next_marker; + } else { + stop = true; + } + for item in v.contents.iter() { + if !(args.result_fn)(Ok(item)) { + stop = true; + break; + } + } + } + Err(e) => { + (args.result_fn)(Err(e)); + return Ok(()); + } + }; + } else { + let resp = self.list_objects_v2(&lov2_args).await; + match resp { + Ok(v) => { + if v.is_truncated { + lov2_args.start_after = v.start_after; + lov2_args.continuation_token = v.next_continuation_token; + } else { + stop = true; + } + for item in v.contents.iter() { + if !(args.result_fn)(Ok(item)) { + stop = true; + break; + } + } + } + Err(e) => { + (args.result_fn)(Err(e)); + return Ok(()); + } + }; + } + } + + Ok(()) + } + + pub async fn make_bucket( + &self, + args: &MakeBucketArgs<'_>, + ) -> Result { + let mut region = "us-east-1"; + if let Some(r) = &args.region { + if !self.base_url.region.is_empty() { + if self.base_url.region != *r { + return Err(Error::RegionMismatch( + self.base_url.region.clone(), + r.to_string(), + )); + } + region = r; + } + } + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + }; + + if args.object_lock { + headers.insert( + String::from("x-amz-bucket-object-lock-enabled"), + String::from("true"), + ); + } + + let mut query_params = &Multimap::new(); + if let Some(v) = &args.extra_query_params { + query_params = v; + } + + let data = match region { + "us-east-1" => String::new(), + _ => format!("{}", region), + }; + + let body = match data.is_empty() { + true => None, + false => Some(data.as_bytes()), + }; + + let resp = self + .execute( + Method::PUT, + ®ion.to_string(), + &mut headers, + &query_params, + Some(&args.bucket), + None, + body, + ) + .await?; + self.region_map + .insert(args.bucket.to_string(), region.to_string()); + + Ok(MakeBucketResponse { + headers: resp.headers().clone(), + region: region.to_string(), + bucket_name: args.bucket.to_string(), + }) + } + + fn read_part( + reader: &mut dyn std::io::Read, + buf: &mut [u8], + size: usize, + ) -> Result { + let mut bytes_read = 0_usize; + let mut i = 0_usize; + let mut stop = false; + while !stop { + let br = reader.read(&mut buf[i..size])?; + bytes_read += br; + stop = (br == 0) || (br == size - i); + i += br; + } + + Ok(bytes_read) + } + + async fn do_put_object( + &self, + args: &mut PutObjectArgs<'_>, + buf: &mut [u8], + upload_id: &mut String, + ) -> Result { + let mut headers = args.get_headers(); + if !headers.contains_key("Content-Type") { + if args.content_type.is_empty() { + headers.insert( + String::from("Content-Type"), + String::from("application/octet-stream"), + ); + } else { + headers.insert(String::from("Content-Type"), args.content_type.to_string()); + } + } + + let mut uploaded_size = 0_usize; + let mut part_number = 0_i16; + let mut stop = false; + let mut one_byte: Vec = Vec::new(); + let mut parts: Vec = Vec::new(); + let object_size = &args.object_size.unwrap(); + let mut part_size = args.part_size; + let mut part_count = args.part_count; + + while !stop { + part_number += 1; + let mut bytes_read = 0_usize; + if args.part_count > 0 { + if part_number == args.part_count { + part_size = object_size - uploaded_size; + stop = true; + } + + bytes_read = Client::read_part(&mut args.stream, buf, part_size)?; + if bytes_read != part_size { + return Err(Error::InsufficientData(part_size, bytes_read)); + } + } else { + let mut size = part_size + 1; + let mut newbuf = match one_byte.len() == 1 { + true => { + buf[0] = one_byte.pop().unwrap(); + size -= 1; + bytes_read = 1; + &mut buf[1..] + } + false => buf, + }; + + let n = Client::read_part(&mut args.stream, &mut newbuf, size)?; + bytes_read += n; + + // If bytes read is less than or equals to part size, then we have reached last part. + if bytes_read <= part_size { + part_count = part_number; + part_size = bytes_read; + stop = true; + } else { + one_byte.push(buf[part_size + 1]); + } + } + + let data = &buf[0..part_size]; + uploaded_size += part_size; + + if part_count == 1_i16 { + let mut poaargs = PutObjectApiArgs::new(&args.bucket, &args.object, &data)?; + poaargs.extra_query_params = args.extra_query_params; + poaargs.region = args.region; + poaargs.headers = Some(&headers); + + return self.put_object_api(&poaargs).await; + } + + if upload_id.is_empty() { + let mut cmuargs = CreateMultipartUploadArgs::new(&args.bucket, &args.object)?; + cmuargs.extra_query_params = args.extra_query_params; + cmuargs.region = args.region; + cmuargs.headers = Some(&headers); + + let resp = self.create_multipart_upload(&cmuargs).await?; + upload_id.push_str(&resp.upload_id); + } + + let mut upargs = UploadPartArgs::new( + &args.bucket, + &args.object, + &upload_id, + part_number as u16, + &data, + )?; + upargs.region = args.region; + + let ssec_headers = match args.sse { + Some(v) => match v.as_any().downcast_ref::() { + Some(_) => v.headers(), + _ => Multimap::new(), + }, + _ => Multimap::new(), + }; + upargs.headers = Some(&ssec_headers); + + let resp = self.upload_part(&upargs).await?; + parts.push(Part { + number: part_number as u16, + etag: resp.etag.clone(), + }); + } + + let mut cmuargs = + CompleteMultipartUploadArgs::new(&args.bucket, &args.object, &upload_id, &parts)?; + cmuargs.region = args.region; + + return self.complete_multipart_upload(&cmuargs).await; + } + + pub async fn put_object( + &self, + args: &mut PutObjectArgs<'_>, + ) -> Result { + if let Some(v) = &args.sse { + if v.tls_required() && !self.base_url.https { + return Err(Error::SseTlsRequired); + } + } + + let bufsize = match args.part_count > 0 { + true => args.part_size as usize, + false => (args.part_size as usize) + 1, + }; + let mut buf = vec![0_u8; bufsize]; + + let mut upload_id = String::new(); + let res = self.do_put_object(args, &mut buf, &mut upload_id).await; + + std::mem::drop(buf); + + if res.is_err() && !upload_id.is_empty() { + let amuargs = &AbortMultipartUploadArgs::new(&args.bucket, &args.object, &upload_id)?; + self.abort_multipart_upload(&amuargs).await?; + } + + return res; + } + + pub async fn put_object_api( + &self, + args: &PutObjectApiArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = args.get_headers(); + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + if let Some(v) = &args.query_params { + merge(&mut query_params, v); + } + + let resp = self + .execute( + Method::PUT, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + Some(&args.data), + ) + .await?; + let header_map = resp.headers(); + + Ok(PutObjectBaseResponse { + headers: header_map.clone(), + bucket_name: args.bucket.to_string(), + object_name: args.object.to_string(), + location: region.clone(), + etag: match header_map.get("etag") { + Some(v) => v.to_str()?.to_string().trim_matches('"').to_string(), + _ => String::new(), + }, + version_id: match header_map.get("x-amz-version-id") { + Some(v) => Some(v.to_str()?.to_string()), + None => None, + }, + }) + } + + pub async fn remove_bucket( + &self, + args: &RemoveBucketArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + let mut query_params = &Multimap::new(); + if let Some(v) = &args.extra_query_params { + query_params = v; + } + + let resp = self + .execute( + Method::DELETE, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + None, + None, + ) + .await?; + self.region_map.remove(&args.bucket.to_string()); + + Ok(RemoveBucketResponse { + headers: resp.headers().clone(), + region: region.to_string(), + bucket_name: args.bucket.to_string(), + }) + } + + pub async fn remove_object( + &self, + args: &RemoveObjectArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + if let Some(v) = args.version_id { + query_params.insert(String::from("versionId"), v.to_string()); + } + + let resp = self + .execute( + Method::DELETE, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + None, + ) + .await?; + + Ok(RemoveObjectResponse { + headers: resp.headers().clone(), + region: region.to_string(), + bucket_name: args.bucket.to_string(), + object_name: args.object.to_string(), + version_id: match args.version_id { + Some(v) => Some(v.to_string()), + None => None, + }, + }) + } + + pub async fn remove_objects_api( + &self, + args: &RemoveObjectsApiArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + let mut data = String::from(""); + if args.quiet { + data.push_str("true"); + } + for object in args.objects.iter() { + data.push_str(""); + data.push_str(""); + data.push_str(&object.name); + data.push_str(""); + if let Some(v) = object.version_id { + data.push_str(""); + data.push_str(&v); + data.push_str(""); + } + data.push_str(""); + } + data.push_str(""); + let b = data.as_bytes(); + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + if args.bypass_governance_mode { + headers.insert( + String::from("x-amz-bypass-governance-retention"), + String::from("true"), + ); + } + headers.insert( + String::from("Content-Type"), + String::from("application/xml"), + ); + headers.insert(String::from("Content-MD5"), md5sum_hash(b)); + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("delete"), String::new()); + + let resp = self + .execute( + Method::POST, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + None, + Some(&b), + ) + .await?; + let header_map = resp.headers().clone(); + let body = resp.bytes().await?; + let mut root = Element::parse(body.reader())?; + + let mut objects: Vec = Vec::new(); + loop { + let deleted = match root.take_child("Deleted") { + Some(v) => v, + None => break, + }; + + objects.push(DeletedObject { + name: get_text(&deleted, "Key")?, + version_id: get_option_text(&deleted, "VersionId")?, + delete_marker: get_text(&deleted, "DeleteMarker")?.to_lowercase() == "true", + delete_marker_version_id: get_option_text(&deleted, "DeleteMarkerVersionId")?, + }) + } + + let mut errors: Vec = Vec::new(); + loop { + let error = match root.take_child("Error") { + Some(v) => v, + None => break, + }; + + errors.push(DeleteError { + code: get_text(&error, "Code")?, + message: get_text(&error, "Message")?, + object_name: get_text(&error, "Key")?, + version_id: get_option_text(&error, "VersionId")?, + }) + } + + Ok(RemoveObjectsApiResponse { + headers: header_map.clone(), + region: region.clone(), + bucket_name: args.bucket.to_string(), + objects: objects, + errors: errors, + }) + } + + pub async fn remove_objects( + &self, + args: &mut RemoveObjectsArgs<'_>, + ) -> Result { + let region = self.get_region(&args.bucket, args.region).await?; + + loop { + let mut objects: Vec = Vec::new(); + for object in args.objects.take(1000) { + objects.push(*object); + } + if objects.len() == 0 { + break; + } + + let mut roa_args = RemoveObjectsApiArgs::new(&args.bucket, &objects)?; + roa_args.extra_headers = args.extra_headers; + roa_args.extra_query_params = args.extra_query_params; + roa_args.region = args.region; + roa_args.bypass_governance_mode = args.bypass_governance_mode; + roa_args.quiet = true; + let resp = self.remove_objects_api(&roa_args).await?; + if resp.errors.len() > 0 { + return Ok(resp); + } + } + + Ok(RemoveObjectsResponse { + headers: HeaderMap::new(), + region: region.to_string(), + bucket_name: args.bucket.to_string(), + objects: vec![], + errors: vec![], + }) + } + + // SetBucketEncryptionResponse SetBucketEncryption(SetBucketEncryptionArgs args); + // SetBucketLifecycleResponse SetBucketLifecycle(SetBucketLifecycleArgs args); + // SetBucketNotificationResponse SetBucketNotification( + // SetBucketNotificationArgs args); + // SetBucketPolicyResponse SetBucketPolicy(SetBucketPolicyArgs args); + // SetBucketReplicationResponse SetBucketReplication( + // SetBucketReplicationArgs args); + // SetBucketTagsResponse SetBucketTags(SetBucketTagsArgs args); + // SetBucketVersioningResponse SetBucketVersioning(SetBucketVersioningArgs args); + // SetObjectLockConfigResponse SetObjectLockConfig(SetObjectLockConfigArgs args); + // SetObjectRetentionResponse SetObjectRetention(SetObjectRetentionArgs args); + // SetObjectTagsResponse SetObjectTags(SetObjectTagsArgs args); + pub async fn select_object_content( + &self, + args: &SelectObjectContentArgs<'_>, + ) -> Result { + if args.ssec.is_some() && !self.base_url.https { + return Err(Error::SseTlsRequired); + } + + let region = self.get_region(&args.bucket, args.region).await?; + + let data = args.request.to_xml(); + let b = data.as_bytes(); + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + headers.insert(String::from("Content-MD5"), md5sum_hash(&b)); + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + query_params.insert(String::from("select"), String::new()); + query_params.insert(String::from("select-type"), String::from("2")); + + Ok(SelectObjectContentResponse::new( + self.execute( + Method::POST, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + Some(&b), + ) + .await?, + ®ion, + &args.bucket, + &args.object, + )) + } + + pub async fn stat_object( + &self, + args: &StatObjectArgs<'_>, + ) -> Result { + if args.ssec.is_some() && !self.base_url.https { + return Err(Error::SseTlsRequired); + } + + let region = self.get_region(&args.bucket, args.region).await?; + + let mut headers = Multimap::new(); + if let Some(v) = &args.extra_headers { + merge(&mut headers, v); + } + merge(&mut headers, &args.get_headers()); + + let mut query_params = Multimap::new(); + if let Some(v) = &args.extra_query_params { + merge(&mut query_params, v); + } + if let Some(v) = args.version_id { + query_params.insert(String::from("versionId"), v.to_string()); + } + + let resp = self + .execute( + Method::HEAD, + ®ion, + &mut headers, + &query_params, + Some(&args.bucket), + Some(&args.object), + None, + ) + .await?; + + StatObjectResponse::new(&resp.headers(), ®ion, &args.bucket, &args.object) + } + + pub async fn upload_part( + &self, + args: &UploadPartArgs<'_>, + ) -> Result { + let mut query_params = Multimap::new(); + query_params.insert(String::from("partNumber"), args.part_number.to_string()); + query_params.insert(String::from("uploadId"), args.upload_id.to_string()); + + let mut poa_args = PutObjectApiArgs::new(&args.bucket, &args.object, &args.data)?; + poa_args.query_params = Some(&query_params); + + poa_args.extra_headers = args.extra_headers; + poa_args.extra_query_params = args.extra_query_params; + poa_args.region = args.region; + poa_args.headers = args.headers; + poa_args.user_metadata = args.user_metadata; + poa_args.sse = args.sse; + poa_args.tags = args.tags; + poa_args.retention = args.retention; + poa_args.legal_hold = args.legal_hold; + + self.put_object_api(&poa_args).await + } + + // UploadPartCopyResponse UploadPartCopy(UploadPartCopyArgs args); +} diff --git a/src/s3/creds.rs b/src/s3/creds.rs new file mode 100644 index 0000000..8bfe0de --- /dev/null +++ b/src/s3/creds.rs @@ -0,0 +1,48 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[derive(Clone, Debug, Default)] +pub struct Credentials { + pub access_key: String, + pub secret_key: String, + pub session_token: Option, +} + +pub trait Provider: std::fmt::Debug { + fn fetch(&self) -> Credentials; +} + +#[derive(Clone, Debug)] +pub struct StaticProvider { + creds: Credentials, +} + +impl StaticProvider { + pub fn new(access_key: &str, secret_key: &str, session_token: Option<&str>) -> StaticProvider { + StaticProvider { + creds: Credentials { + access_key: access_key.to_string(), + secret_key: secret_key.to_string(), + session_token: session_token.map(|v| v.to_string()), + }, + } + } +} + +impl Provider for StaticProvider { + fn fetch(&self) -> Credentials { + self.creds.clone() + } +} diff --git a/src/s3/error.rs b/src/s3/error.rs new file mode 100644 index 0000000..6dfd94a --- /dev/null +++ b/src/s3/error.rs @@ -0,0 +1,187 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern crate alloc; +use crate::s3::utils::get_default_text; +use bytes::{Buf, Bytes}; +use std::fmt; +use xmltree::Element; + +#[derive(Clone, Debug, Default)] +pub struct ErrorResponse { + pub code: String, + pub message: String, + pub resource: String, + pub request_id: String, + pub host_id: String, + pub bucket_name: String, + pub object_name: String, +} + +impl ErrorResponse { + pub fn parse(body: &mut Bytes) -> Result { + let root = match Element::parse(body.reader()) { + Ok(v) => v, + Err(e) => return Err(Error::XmlParseError(e)), + }; + + Ok(ErrorResponse { + code: get_default_text(&root, "Code"), + message: get_default_text(&root, "Message"), + resource: get_default_text(&root, "Resource"), + request_id: get_default_text(&root, "RequestId"), + host_id: get_default_text(&root, "HostId"), + bucket_name: get_default_text(&root, "bucketName"), + object_name: get_default_text(&root, "Key"), + }) + } +} + +#[derive(Debug)] +pub enum Error { + TimeParseError(chrono::ParseError), + InvalidUrl(http::uri::InvalidUri), + IOError(std::io::Error), + XmlParseError(xmltree::ParseError), + HttpError(reqwest::Error), + StrError(reqwest::header::ToStrError), + IntError(std::num::ParseIntError), + BoolError(std::str::ParseBoolError), + Utf8Error(alloc::string::FromUtf8Error), + XmlError(String), + InvalidBucketName(String), + InvalidBaseUrl(String), + UrlBuildError(String), + RegionMismatch(String, String), + S3Error(ErrorResponse), + InvalidResponse(u16, String), + ServerError(u16), + InvalidObjectName(String), + InvalidUploadId(String), + InvalidPartNumber(String), + EmptyParts(String), + InvalidRetentionMode(String), + InvalidMinPartSize(usize), + InvalidMaxPartSize(usize), + InvalidObjectSize(usize), + MissingPartSize, + InvalidPartCount(usize, usize, u16), + SseTlsRequired, + InsufficientData(usize, usize), + InvalidLegalHold(String), + InvalidSelectExpression(String), + InvalidHeaderValueType(u8), + CrcMismatch(String, u32, u32), + UnknownEventType(String), + SelectError(String, String), +} + +impl std::error::Error for Error {} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::TimeParseError(e) => write!(f, "{}", e), + Error::InvalidUrl(e) => write!(f, "{}", e), + Error::IOError(e) => write!(f, "{}", e), + Error::XmlParseError(e) => write!(f, "{}", e), + Error::HttpError(e) => write!(f, "{}", e), + Error::StrError(e) => write!(f, "{}", e), + Error::IntError(e) => write!(f, "{}", e), + Error::BoolError(e) => write!(f, "{}", e), + Error::Utf8Error(e) => write!(f, "{}", e), + Error::XmlError(m) => write!(f, "{}", m), + Error::InvalidBucketName(m) => write!(f, "{}", m), + Error::InvalidObjectName(m) => write!(f, "{}", m), + Error::InvalidUploadId(m) => write!(f, "{}", m), + Error::InvalidPartNumber(m) => write!(f, "{}", m), + Error::EmptyParts(m) => write!(f, "{}", m), + Error::InvalidRetentionMode(m) => write!(f, "invalid retention mode {}", m), + Error::InvalidMinPartSize(s) => write!(f, "part size {} is not supported; minimum allowed 5MiB", s), + Error::InvalidMaxPartSize(s) => write!(f, "part size {} is not supported; maximum allowed 5GiB", s), + Error::InvalidObjectSize(s) => write!(f, "object size {} is not supported; maximum allowed 5TiB", s), + Error::MissingPartSize => write!(f, "valid part size must be provided when object size is unknown"), + Error::InvalidPartCount(os, ps, pc) => write!(f, "object size {} and part size {} make more than {} parts for upload", os, ps, pc), + Error::SseTlsRequired => write!(f, "SSE operation must be performed over a secure connection"), + Error::InsufficientData(ps, br) => write!(f, "not enough data in the stream; expected: {}, got: {} bytes", ps, br), + Error::InvalidBaseUrl(m) => write!(f, "{}", m), + Error::UrlBuildError(m) => write!(f, "{}", m), + Error::InvalidLegalHold(s) => write!(f, "invalid legal hold {}", s), + Error::RegionMismatch(br, r) => write!(f, "region must be {}, but passed {}", br, r), + Error::S3Error(er) => write!(f, "s3 operation failed; code: {}, message: {}, resource: {}, request_id: {}, host_id: {}, bucket_name: {}, object_name: {}", er.code, er.message, er.resource, er.request_id, er.host_id, er.bucket_name, er.object_name), + Error::InvalidResponse(sc, ct) => write!(f, "invalid response received; status code: {}; content-type: {}", sc, ct), + Error::ServerError(sc) => write!(f, "server failed with HTTP status code {}", sc), + Error::InvalidSelectExpression(m) => write!(f, "{}", m), + Error::InvalidHeaderValueType(v) => write!(f, "invalid header value type {}", v), + Error::CrcMismatch(t, e, g) => write!(f, "{} CRC mismatch; expected: {}, got: {}", t, e, g), + Error::UnknownEventType(et) => write!(f, "unknown event type {}", et), + Error::SelectError(ec, em) => write!(f, "error code: {}, error message: {}", ec, em), + } + } +} + +impl From for Error { + fn from(err: chrono::ParseError) -> Self { + Error::TimeParseError(err) + } +} + +impl From for Error { + fn from(err: http::uri::InvalidUri) -> Self { + Error::InvalidUrl(err) + } +} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Error::IOError(err) + } +} + +impl From for Error { + fn from(err: xmltree::ParseError) -> Self { + Error::XmlParseError(err) + } +} + +impl From for Error { + fn from(err: reqwest::Error) -> Self { + Error::HttpError(err) + } +} + +impl From for Error { + fn from(err: reqwest::header::ToStrError) -> Self { + Error::StrError(err) + } +} + +impl From for Error { + fn from(err: std::num::ParseIntError) -> Self { + Error::IntError(err) + } +} + +impl From for Error { + fn from(err: std::str::ParseBoolError) -> Self { + Error::BoolError(err) + } +} + +impl From for Error { + fn from(err: alloc::string::FromUtf8Error) -> Self { + Error::Utf8Error(err) + } +} diff --git a/src/s3/http.rs b/src/s3/http.rs new file mode 100644 index 0000000..f0c08d5 --- /dev/null +++ b/src/s3/http.rs @@ -0,0 +1,279 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::error::Error; +use crate::s3::utils::{to_query_string, Multimap}; +use derivative::Derivative; +use hyper::http::Method; +use hyper::Uri; +use std::fmt; + +#[derive(Derivative)] +#[derivative(Clone, Debug, Default)] +pub struct Url { + #[derivative(Default(value = "true"))] + pub https: bool, + pub host: String, + pub port: u16, + pub path: String, + pub query: Multimap, +} + +impl Url { + pub fn host_header_value(&self) -> String { + if self.port > 0 { + return format!("{}:{}", self.host, self.port); + } + return self.host.clone(); + } +} + +impl fmt::Display for Url { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.host.is_empty() { + return Err(std::fmt::Error); + } + + if self.https { + f.write_str("https://")?; + } else { + f.write_str("http://")?; + } + + if self.port > 0 { + f.write_str(format!("{}:{}", self.host, self.port).as_str())?; + } else { + f.write_str(&self.host)?; + } + + if !self.path.starts_with("/") { + f.write_str("/")?; + } + f.write_str(&self.path)?; + + if !self.query.is_empty() { + f.write_str("?")?; + f.write_str(&to_query_string(&self.query))?; + } + + Ok(()) + } +} + +fn extract_region(host: &str) -> String { + let tokens: Vec<&str> = host.split('.').collect(); + let region = match tokens.get(1) { + Some(r) => match *r { + "dualstack" => match tokens.get(2) { + Some(t) => t, + _ => "", + }, + "amazonaws" => "", + _ => r, + }, + _ => "", + }; + return region.to_string(); +} + +#[derive(Derivative)] +#[derivative(Clone, Debug, Default)] +pub struct BaseUrl { + #[derivative(Default(value = "true"))] + pub https: bool, + host: String, + port: u16, + pub region: String, + aws_host: bool, + accelerate_host: bool, + dualstack_host: bool, + virtual_style: bool, +} + +impl BaseUrl { + pub fn build_url( + &self, + method: &Method, + region: &String, + query: &Multimap, + bucket_name: Option<&str>, + object_name: Option<&str>, + ) -> Result { + if !object_name.map_or(true, |v| v.is_empty()) { + if bucket_name.map_or(true, |v| v.is_empty()) { + return Err(Error::UrlBuildError(String::from( + "empty bucket name provided for object name", + ))); + } + } + + let mut url = Url::default(); + url.https = self.https; + url.host = self.host.clone(); + url.port = self.port; + url.query = query.clone(); + + if bucket_name.is_none() { + url.path.push_str("/"); + if self.aws_host { + url.host = format!("s3.{}.{}", region, self.host); + } + return Ok(url); + } + + let bucket = bucket_name.unwrap(); + + let enforce_path_style = true && + // CreateBucket API requires path style in Amazon AWS S3. + (method == Method::PUT && object_name.is_none() && query.is_empty()) || + // GetBucketLocation API requires path style in Amazon AWS S3. + query.contains_key("location") || + // Use path style for bucket name containing '.' which causes + // SSL certificate validation error. + (bucket.contains('.') && self.https); + + if self.aws_host { + let mut s3_domain = "s3.".to_string(); + if self.accelerate_host { + if bucket.contains('.') { + return Err(Error::UrlBuildError(String::from( + "bucket name with '.' is not allowed for accelerate endpoint", + ))); + } + + if !enforce_path_style { + s3_domain = "s3-accelerate.".to_string(); + } + } + + if self.dualstack_host { + s3_domain.push_str("dualstack."); + } + if enforce_path_style || !self.accelerate_host { + s3_domain.push_str(region); + s3_domain.push_str("."); + } + url.host = s3_domain + &url.host; + } + + if enforce_path_style || !self.virtual_style { + url.path.push_str("/"); + url.path.push_str(bucket); + } else { + url.host = format!("{}.{}", bucket, url.host); + } + + if object_name.is_some() { + if object_name.unwrap().chars().nth(0) != Some('/') { + url.path.push_str("/"); + } + // FIXME: urlencode path + url.path.push_str(object_name.unwrap()); + } + + return Ok(url); + } + + pub fn from_string(s: String) -> Result { + let url = s.parse::()?; + + let https = match url.scheme() { + None => true, + Some(scheme) => match scheme.as_str() { + "http" => false, + "https" => true, + _ => { + return Err(Error::InvalidBaseUrl(String::from( + "scheme must be http or https", + ))) + } + }, + }; + + let mut host = match url.host() { + Some(h) => h, + _ => { + return Err(Error::InvalidBaseUrl(String::from( + "valid host must be provided", + ))) + } + }; + + let ipv6host = "[".to_string() + host + "]"; + if host.parse::().is_ok() { + host = &ipv6host; + } + + let mut port = match url.port() { + Some(p) => p.as_u16(), + _ => 0u16, + }; + + if (https && port == 443) || (!https && port == 80) { + port = 0u16; + } + + if url.path() != "/" && url.path() != "" { + return Err(Error::InvalidBaseUrl(String::from( + "path must be empty for base URL", + ))); + } + + if !url.query().is_none() { + return Err(Error::InvalidBaseUrl(String::from( + "query must be none for base URL", + ))); + } + + let mut accelerate_host = host.starts_with("s3-accelerate."); + let aws_host = (host.starts_with("s3.") || accelerate_host) + && (host.ends_with(".amazonaws.com") || host.ends_with(".amazonaws.com.cn")); + let virtual_style = aws_host || host.ends_with("aliyuncs.com"); + + let mut region = String::new(); + let mut dualstack_host = false; + + if aws_host { + let mut aws_domain = "amazonaws.com"; + region = extract_region(host); + + let is_aws_china_host = host.ends_with(".cn"); + if is_aws_china_host { + aws_domain = "amazonaws.com.cn"; + if region.is_empty() { + return Err(Error::InvalidBaseUrl(String::from( + "region must be provided in Amazon S3 China endpoint", + ))); + } + } + + dualstack_host = host.contains(".dualstack."); + host = aws_domain; + } else { + accelerate_host = false; + } + + return Ok(BaseUrl { + https: https, + host: host.to_string(), + port: port, + region: region, + aws_host: aws_host, + accelerate_host: accelerate_host, + dualstack_host: dualstack_host, + virtual_style: virtual_style, + }); + } +} diff --git a/src/s3/mod.rs b/src/s3/mod.rs new file mode 100644 index 0000000..7904da4 --- /dev/null +++ b/src/s3/mod.rs @@ -0,0 +1,25 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod args; +pub mod client; +pub mod creds; +pub mod error; +pub mod http; +pub mod response; +pub mod signer; +pub mod sse; +pub mod types; +pub mod utils; diff --git a/src/s3/response.rs b/src/s3/response.rs new file mode 100644 index 0000000..4c4a89e --- /dev/null +++ b/src/s3/response.rs @@ -0,0 +1,596 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::error::Error; +use crate::s3::types::{parse_legal_hold, Bucket, Item, RetentionMode, SelectProgress}; +use crate::s3::utils::{ + copy_slice, crc32, from_http_header_value, from_iso8601utc, get_text, uint32, UtcTime, +}; +use reqwest::header::HeaderMap; +use std::collections::HashMap; +use std::collections::VecDeque; +use std::io::BufReader; +use xmltree::Element; + +#[derive(Debug)] +pub struct ListBucketsResponse { + pub headers: HeaderMap, + pub buckets: Vec, +} + +#[derive(Debug)] +pub struct BucketResponse { + pub headers: HeaderMap, + pub region: String, + pub bucket_name: String, +} + +pub type MakeBucketResponse = BucketResponse; + +pub type RemoveBucketResponse = BucketResponse; + +#[derive(Debug)] +pub struct ObjectResponse { + pub headers: HeaderMap, + pub region: String, + pub bucket_name: String, + pub object_name: String, + pub version_id: Option, +} + +pub type RemoveObjectResponse = ObjectResponse; + +#[derive(Debug)] +pub struct UploadIdResponse { + pub headers: HeaderMap, + pub region: String, + pub bucket_name: String, + pub object_name: String, + pub upload_id: String, +} + +pub type AbortMultipartUploadResponse = UploadIdResponse; + +pub type CreateMultipartUploadResponse = UploadIdResponse; + +#[derive(Debug)] +pub struct PutObjectBaseResponse { + pub headers: HeaderMap, + pub bucket_name: String, + pub object_name: String, + pub location: String, + pub etag: String, + pub version_id: Option, +} + +pub type CompleteMultipartUploadResponse = PutObjectBaseResponse; + +pub type PutObjectApiResponse = PutObjectBaseResponse; + +pub type UploadPartResponse = PutObjectApiResponse; + +pub type PutObjectResponse = PutObjectApiResponse; + +#[derive(Debug)] +pub struct StatObjectResponse { + pub headers: HeaderMap, + pub region: String, + pub bucket_name: String, + pub object_name: String, + pub size: usize, + pub etag: String, + pub version_id: Option, + pub last_modified: Option, + pub retention_mode: Option, + pub retention_retain_until_date: Option, + pub legal_hold: Option, + pub delete_marker: Option, + pub user_metadata: HashMap, +} + +impl StatObjectResponse { + pub fn new( + headers: &HeaderMap, + region: &str, + bucket_name: &str, + object_name: &str, + ) -> Result { + let size = match headers.get("Content-Length") { + Some(v) => v.to_str()?.parse::()?, + None => 0_usize, + }; + + let etag = match headers.get("ETag") { + Some(v) => v.to_str()?.trim_matches('"'), + None => "", + }; + + let version_id = match headers.get("x-amz-version-id") { + Some(v) => Some(v.to_str()?.to_string()), + None => None, + }; + + let last_modified = match headers.get("Last-Modified") { + Some(v) => Some(from_http_header_value(v.to_str()?)?), + None => None, + }; + + let retention_mode = match headers.get("x-amz-object-lock-mode") { + Some(v) => Some(RetentionMode::parse(v.to_str()?)?), + None => None, + }; + + let retention_retain_until_date = match headers.get("x-amz-object-lock-retain-until-date") { + Some(v) => Some(from_iso8601utc(v.to_str()?)?), + None => None, + }; + + let legal_hold = match headers.get("x-amz-object-lock-legal-hold") { + Some(v) => Some(parse_legal_hold(v.to_str()?)?), + None => None, + }; + + let delete_marker = match headers.get("x-amz-delete-marker") { + Some(v) => Some(v.to_str()?.parse::()?), + None => None, + }; + + let mut user_metadata: HashMap = HashMap::new(); + for (key, value) in headers.iter() { + if let Some(v) = key.as_str().strip_prefix("x-amz-meta-") { + user_metadata.insert(v.to_string(), value.to_str()?.to_string()); + } + } + + Ok(StatObjectResponse { + headers: headers.clone(), + region: region.to_string(), + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + size: size, + etag: etag.to_string(), + version_id: version_id, + last_modified: last_modified, + retention_mode: retention_mode, + retention_retain_until_date: retention_retain_until_date, + legal_hold: legal_hold, + delete_marker: delete_marker, + user_metadata: user_metadata, + }) + } +} + +#[derive(Clone, Debug)] +pub struct DeleteError { + pub code: String, + pub message: String, + pub object_name: String, + pub version_id: Option, +} + +#[derive(Clone, Debug)] +pub struct DeletedObject { + pub name: String, + pub version_id: Option, + pub delete_marker: bool, + pub delete_marker_version_id: Option, +} + +#[derive(Clone, Debug)] +pub struct RemoveObjectsApiResponse { + pub headers: HeaderMap, + pub region: String, + pub bucket_name: String, + pub objects: Vec, + pub errors: Vec, +} + +pub type RemoveObjectsResponse = RemoveObjectsApiResponse; + +#[derive(Clone, Debug)] +pub struct ListObjectsV1Response { + pub headers: HeaderMap, + pub name: String, + pub encoding_type: Option, + pub prefix: Option, + pub delimiter: Option, + pub is_truncated: bool, + pub max_keys: Option, + pub contents: Vec, + pub marker: Option, + pub next_marker: Option, +} + +#[derive(Clone, Debug)] +pub struct ListObjectsV2Response { + pub headers: HeaderMap, + pub name: String, + pub encoding_type: Option, + pub prefix: Option, + pub delimiter: Option, + pub is_truncated: bool, + pub max_keys: Option, + pub contents: Vec, + pub key_count: Option, + pub start_after: Option, + pub continuation_token: Option, + pub next_continuation_token: Option, +} + +#[derive(Clone, Debug)] +pub struct ListObjectVersionsResponse { + pub headers: HeaderMap, + pub name: String, + pub encoding_type: Option, + pub prefix: Option, + pub delimiter: Option, + pub is_truncated: bool, + pub max_keys: Option, + pub contents: Vec, + pub key_marker: Option, + pub next_key_marker: Option, + pub version_id_marker: Option, + pub next_version_id_marker: Option, +} + +#[derive(Clone, Debug)] +pub struct ListObjectsResponse { + pub headers: HeaderMap, + pub name: String, + pub encoding_type: Option, + pub prefix: Option, + pub delimiter: Option, + pub is_truncated: bool, + pub max_keys: Option, + pub contents: Vec, + + // ListObjectsV1 + pub marker: String, + pub next_marker: String, + + // ListObjectsV2 + pub key_count: u16, + pub start_after: String, + pub continuation_token: String, + pub next_continuation_token: String, + + // ListObjectVersions + pub key_marker: String, + pub next_key_marker: String, + pub version_id_marker: String, + pub next_version_id_marker: String, +} + +pub struct SelectObjectContentResponse { + pub headers: HeaderMap, + pub region: String, + pub bucket_name: String, + pub object_name: String, + pub progress: SelectProgress, + + resp: reqwest::Response, + + done: bool, + buf: VecDeque, + + prelude: [u8; 8], + prelude_read: bool, + + prelude_crc: [u8; 4], + prelude_crc_read: bool, + + total_length: usize, + + data: Vec, + data_read: bool, + + message_crc: [u8; 4], + message_crc_read: bool, + + payload: Vec, + payload_index: usize, +} + +impl SelectObjectContentResponse { + pub fn new( + resp: reqwest::Response, + region: &str, + bucket_name: &str, + object_name: &str, + ) -> SelectObjectContentResponse { + let headers = resp.headers().clone(); + + SelectObjectContentResponse { + headers: headers, + region: region.to_string(), + bucket_name: bucket_name.to_string(), + object_name: object_name.to_string(), + progress: SelectProgress { + bytes_scanned: 0, + bytes_progressed: 0, + bytes_returned: 0, + }, + resp: resp, + done: false, + buf: VecDeque::::new(), + prelude: [0_u8; 8], + prelude_read: false, + prelude_crc: [0_u8; 4], + prelude_crc_read: false, + total_length: 0_usize, + data: Vec::::new(), + data_read: false, + message_crc: [0_u8; 4], + message_crc_read: false, + payload: Vec::::new(), + payload_index: 0, + } + } + + fn reset(&mut self) { + self.buf.clear(); + + self.data.clear(); + self.data_read = false; + + self.prelude_read = false; + self.prelude_crc_read = false; + self.message_crc_read = false; + } + + fn read_prelude(&mut self) -> Result { + if self.buf.len() < 8 { + return Ok(false); + } + + self.prelude_read = true; + for i in 0..8 { + self.prelude[i] = self.buf.pop_front().ok_or(Error::InsufficientData(8, i))?; + } + + return Ok(true); + } + + fn read_prelude_crc(&mut self) -> Result { + if self.buf.len() < 4 { + return Ok(false); + } + + self.prelude_crc_read = true; + for i in 0..4 { + self.prelude_crc[i] = self.buf.pop_front().ok_or(Error::InsufficientData(4, i))?; + } + + return Ok(true); + } + + fn read_data(&mut self) -> Result { + let data_length = self.total_length - 8 - 4 - 4; + if self.buf.len() < data_length { + return Ok(false); + } + + self.data = Vec::new(); + + self.data_read = true; + for i in 0..data_length { + self.data.push( + self.buf + .pop_front() + .ok_or(Error::InsufficientData(data_length, i))?, + ); + } + + return Ok(true); + } + + fn read_message_crc(&mut self) -> Result { + if self.buf.len() < 4 { + return Ok(false); + } + + self.message_crc_read = true; + for i in 0..4 { + self.message_crc[i] = self.buf.pop_front().ok_or(Error::InsufficientData(4, i))?; + } + + return Ok(true); + } + + fn decode_header(&mut self, header_length: usize) -> Result, Error> { + let mut headers: HashMap = HashMap::new(); + let mut offset = 0_usize; + while offset < header_length { + let mut length = self.data[offset] as usize; + offset += 1; + if length == 0 { + break; + } + + let name = String::from_utf8(self.data[offset..offset + length].to_vec())?; + offset += length; + + if self.data[offset] != 7 { + return Err(Error::InvalidHeaderValueType(self.data[offset])); + } + offset += 1; + + let b0 = self.data[offset] as u16; + offset += 1; + let b1 = self.data[offset] as u16; + offset += 1; + length = (b0 << 8 | b1) as usize; + + let value = String::from_utf8(self.data[offset..offset + length].to_vec())?; + offset += length; + + headers.insert(name, value); + } + + return Ok(headers); + } + + async fn do_read(&mut self) -> Result<(), Error> { + if self.done { + return Ok(()); + } + + loop { + let chunk = match self.resp.chunk().await? { + Some(v) => v, + None => return Ok(()), + }; + + self.buf.extend(chunk.iter().copied()); + + if !self.prelude_read && !self.read_prelude()? { + continue; + } + + if !self.prelude_crc_read { + if !self.read_prelude_crc()? { + continue; + } + + let got = crc32(&self.prelude); + let expected = uint32(&self.prelude_crc)?; + if got != expected { + self.done = true; + return Err(Error::CrcMismatch(String::from("prelude"), expected, got)); + } + + self.total_length = uint32(&self.prelude[0..4])? as usize; + } + + if !self.data_read && !self.read_data()? { + continue; + } + + if !self.message_crc_read { + if !self.read_message_crc()? { + continue; + } + + let mut message: Vec = Vec::new(); + message.extend_from_slice(&self.prelude); + message.extend_from_slice(&self.prelude_crc); + message.extend_from_slice(&self.data); + + let got = crc32(&message); + let expected = uint32(&self.message_crc)?; + if got != expected { + self.done = true; + return Err(Error::CrcMismatch(String::from("message"), expected, got)); + } + } + + let header_length = uint32(&self.prelude[4..])? as usize; + let headers = self.decode_header(header_length)?; + let value = match headers.get(":message-type") { + Some(v) => v.as_str(), + None => "", + }; + if value == "error" { + self.done = true; + return Err(Error::SelectError( + match headers.get(":error-code") { + Some(v) => v.clone(), + None => String::new(), + }, + match headers.get(":error-message") { + Some(v) => v.clone(), + None => String::new(), + }, + )); + } + + let event_type = match headers.get(":event-type") { + Some(v) => v.as_str(), + None => "", + }; + + if event_type == "End" { + self.done = true; + return Ok(()); + } + + let payload_length = self.total_length - header_length - 16; + if event_type == "Cont" || payload_length < 1 { + self.reset(); + continue; + } + + let payload = &self.data[header_length..(header_length + payload_length)]; + if event_type == "Progress" || event_type == "Stats" { + let root = Element::parse(&mut BufReader::new(payload))?; + self.reset(); + self.progress = SelectProgress { + bytes_scanned: get_text(&root, "BytesScanned")?.parse::()?, + bytes_progressed: get_text(&root, "BytesProcessed")?.parse::()?, + bytes_returned: get_text(&root, "BytesReturned")?.parse::()?, + }; + continue; + } + + if event_type == "Records" { + self.payload = payload.to_vec(); + self.payload_index = 0; + self.reset(); + return Ok(()); + } + + self.done = true; + return Err(Error::UnknownEventType(event_type.to_string())); + } + } + + pub async fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + if self.done { + return Ok(0); + } + + if self.payload_index < self.payload.len() { + let n = copy_slice(buf, &self.payload[self.payload_index..]); + + self.payload_index += n; + if self.payload_index > self.payload.len() { + self.payload_index = self.payload.len(); + } + + return Ok(n); + } + + self.payload.clear(); + self.payload_index = 0; + + match self.do_read().await { + Err(e) => { + self.done = true; + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + e.to_string(), + )); + } + Ok(_) => { + if self.payload.len() == 0 { + self.done = true; + return Ok(0); + } + } + } + } + } +} diff --git a/src/s3/signer.rs b/src/s3/signer.rs new file mode 100644 index 0000000..6119994 --- /dev/null +++ b/src/s3/signer.rs @@ -0,0 +1,241 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::utils::{ + get_canonical_headers, get_canonical_query_string, sha256_hash, to_amz_date, to_signer_date, + Multimap, UtcTime, +}; +use hex::encode as hexencode; +use hmac::{Hmac, Mac}; +use hyper::http::Method; +use sha2::Sha256; + +pub fn hmac_hash(key: &[u8], data: &[u8]) -> Vec { + let mut hasher = Hmac::::new_from_slice(key).expect("HMAC can take key of any size"); + hasher.update(data); + return hasher.finalize().into_bytes().to_vec(); +} + +pub fn hmac_hash_hex(key: &[u8], data: &[u8]) -> String { + return hexencode(hmac_hash(key, data)); +} + +pub fn get_scope(date: UtcTime, region: &str, service_name: &str) -> String { + return format!( + "{}/{}/{}/aws4_request", + to_signer_date(date), + region, + service_name + ); +} + +pub fn get_canonical_request_hash( + method: &Method, + uri: &str, + query_string: &str, + headers: &str, + signed_headers: &str, + content_sha256: &str, +) -> String { + // CanonicalRequest = + // HTTPRequestMethod + '\n' + + // CanonicalURI + '\n' + + // CanonicalQueryString + '\n' + + // CanonicalHeaders + '\n\n' + + // SignedHeaders + '\n' + + // HexEncode(Hash(RequestPayload)) + let canonical_request = format!( + "{}\n{}\n{}\n{}\n\n{}\n{}", + method, uri, query_string, headers, signed_headers, content_sha256 + ); + return sha256_hash(canonical_request.as_bytes()); +} + +pub fn get_string_to_sign(date: UtcTime, scope: &str, canonical_request_hash: &str) -> String { + return format!( + "AWS4-HMAC-SHA256\n{}\n{}\n{}", + to_amz_date(date), + scope, + canonical_request_hash + ); +} + +pub fn get_signing_key( + secret_key: &str, + date: UtcTime, + region: &str, + service_name: &str, +) -> Vec { + let mut key: Vec = b"AWS4".to_vec(); + key.extend(secret_key.as_bytes()); + + let date_key = hmac_hash(key.as_slice(), to_signer_date(date).as_bytes()); + let date_region_key = hmac_hash(date_key.as_slice(), region.as_bytes()); + let date_region_service_key = hmac_hash(date_region_key.as_slice(), service_name.as_bytes()); + return hmac_hash(date_region_service_key.as_slice(), b"aws4_request"); +} + +pub fn get_signature(signing_key: &[u8], string_to_sign: &[u8]) -> String { + hmac_hash_hex(signing_key, string_to_sign) +} + +pub fn get_authorization( + access_key: &str, + scope: &str, + signed_headers: &str, + signature: &str, +) -> String { + return format!( + "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}", + access_key, scope, signed_headers, signature + ); +} + +pub fn sign_v4( + service_name: &str, + method: &Method, + uri: &str, + region: &str, + headers: &mut Multimap, + query_params: &Multimap, + access_key: &str, + secret_key: &str, + content_sha256: &str, + date: UtcTime, +) { + let scope = get_scope(date, region, service_name); + let (signed_headers, canonical_headers) = get_canonical_headers(headers); + let canonical_query_string = get_canonical_query_string(query_params); + let canonical_request_hash = get_canonical_request_hash( + method, + uri, + &canonical_query_string, + &canonical_headers, + &signed_headers, + content_sha256, + ); + let string_to_sign = get_string_to_sign(date, &scope, &canonical_request_hash); + let signing_key = get_signing_key(secret_key, date, region, service_name); + let signature = get_signature(signing_key.as_slice(), string_to_sign.as_bytes()); + let authorization = get_authorization(access_key, &scope, &signed_headers, &signature); + + headers.insert("Authorization".to_string(), authorization); +} + +pub fn sign_v4_s3( + method: &Method, + uri: &str, + region: &str, + headers: &mut Multimap, + query_params: &Multimap, + access_key: &str, + secret_key: &str, + content_sha256: &str, + date: UtcTime, +) { + sign_v4( + "s3", + method, + uri, + region, + headers, + query_params, + access_key, + secret_key, + content_sha256, + date, + ) +} + +pub fn sign_v4_sts( + method: &Method, + uri: &str, + region: &str, + headers: &mut Multimap, + query_params: &Multimap, + access_key: &str, + secret_key: &str, + content_sha256: &str, + date: UtcTime, +) { + sign_v4( + "sts", + method, + uri, + region, + headers, + query_params, + access_key, + secret_key, + content_sha256, + date, + ) +} + +pub fn presign_v4( + method: &Method, + host: &str, + uri: &str, + region: &str, + query_params: &mut Multimap, + access_key: &str, + secret_key: &str, + date: UtcTime, + expires: u32, +) { + let scope = get_scope(date, region, "s3"); + let canonical_headers = "host:".to_string() + host; + let signed_headers = "host"; + + query_params.insert( + "X-Amz-Algorithm".to_string(), + "AWS4-HMAC-SHA256".to_string(), + ); + query_params.insert( + "X-Amz-Credential".to_string(), + access_key.to_string() + "/" + &scope, + ); + query_params.insert("X-Amz-Date".to_string(), to_amz_date(date)); + query_params.insert("X-Amz-Expires".to_string(), expires.to_string()); + query_params.insert( + "X-Amz-SignedHeaders".to_string(), + signed_headers.to_string(), + ); + + let canonical_query_string = get_canonical_query_string(query_params); + let canonical_request_hash = get_canonical_request_hash( + method, + uri, + &canonical_query_string, + &canonical_headers, + &signed_headers, + "UNSIGNED-PAYLOAD", + ); + let string_to_sign = get_string_to_sign(date, &scope, &canonical_request_hash); + let signing_key = get_signing_key(secret_key, date, region, "s3"); + let signature = get_signature(signing_key.as_slice(), string_to_sign.as_bytes()); + + query_params.insert("X-Amz-Signature".to_string(), signature); +} + +pub fn post_presign_v4( + string_to_sign: &str, + secret_key: &str, + date: UtcTime, + region: &str, +) -> String { + let signing_key = get_signing_key(secret_key, date, region, "s3"); + return get_signature(signing_key.as_slice(), string_to_sign.as_bytes()); +} diff --git a/src/s3/sse.rs b/src/s3/sse.rs new file mode 100644 index 0000000..4a1ba42 --- /dev/null +++ b/src/s3/sse.rs @@ -0,0 +1,168 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::utils; +use std::any::Any; + +pub trait Sse: std::fmt::Debug { + fn headers(&self) -> utils::Multimap; + fn copy_headers(&self) -> utils::Multimap; + fn tls_required(&self) -> bool; + fn as_any(&self) -> &dyn Any; +} + +#[derive(Clone, Debug)] +pub struct SseCustomerKey { + headers: utils::Multimap, + copy_headers: utils::Multimap, +} + +impl SseCustomerKey { + pub fn new(key: &str) -> SseCustomerKey { + let b64key = utils::b64encode(key); + let md5key = utils::md5sum_hash(key.as_bytes()); + + let mut headers = utils::Multimap::new(); + headers.insert( + String::from("X-Amz-Server-Side-Encryption-Customer-Algorithm"), + String::from("AES256"), + ); + headers.insert( + String::from("X-Amz-Server-Side-Encryption-Customer-Key"), + b64key.clone(), + ); + headers.insert( + String::from("X-Amz-Server-Side-Encryption-Customer-Key-MD5"), + md5key.clone(), + ); + + let mut copy_headers = utils::Multimap::new(); + copy_headers.insert( + String::from("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"), + String::from("AES256"), + ); + copy_headers.insert( + String::from("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"), + b64key.clone(), + ); + copy_headers.insert( + String::from("X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"), + md5key.clone(), + ); + + SseCustomerKey { + headers: headers, + copy_headers: copy_headers, + } + } +} + +impl Sse for SseCustomerKey { + fn headers(&self) -> utils::Multimap { + self.headers.clone() + } + + fn copy_headers(&self) -> utils::Multimap { + self.copy_headers.clone() + } + + fn tls_required(&self) -> bool { + true + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +#[derive(Clone, Debug)] +pub struct SseKms { + headers: utils::Multimap, +} + +impl SseKms { + pub fn new(key: &str, context: Option<&str>) -> SseKms { + let mut headers = utils::Multimap::new(); + headers.insert( + String::from("X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"), + key.to_string(), + ); + headers.insert( + String::from("X-Amz-Server-Side-Encryption"), + String::from("aws:kms"), + ); + if let Some(v) = context { + headers.insert( + String::from("X-Amz-Server-Side-Encryption-Context"), + utils::b64encode(v), + ); + } + + SseKms { headers: headers } + } +} + +impl Sse for SseKms { + fn headers(&self) -> utils::Multimap { + self.headers.clone() + } + + fn copy_headers(&self) -> utils::Multimap { + utils::Multimap::new() + } + + fn tls_required(&self) -> bool { + true + } + + fn as_any(&self) -> &dyn Any { + self + } +} + +#[derive(Clone, Debug)] +pub struct SseS3 { + headers: utils::Multimap, +} + +impl SseS3 { + pub fn new() -> SseS3 { + let mut headers = utils::Multimap::new(); + headers.insert( + String::from("X-Amz-Server-Side-Encryption"), + String::from("AES256"), + ); + + SseS3 { headers: headers } + } +} + +impl Sse for SseS3 { + fn headers(&self) -> utils::Multimap { + self.headers.clone() + } + + fn copy_headers(&self) -> utils::Multimap { + utils::Multimap::new() + } + + fn tls_required(&self) -> bool { + false + } + + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/s3/types.rs b/src/s3/types.rs new file mode 100644 index 0000000..e0b75ef --- /dev/null +++ b/src/s3/types.rs @@ -0,0 +1,459 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::error::Error; +use crate::s3::utils::UtcTime; +use std::collections::HashMap; +use std::fmt; + +#[derive(Clone, Debug, Default)] +pub struct Item { + pub name: String, + pub last_modified: Option, + pub etag: Option, // except DeleteMarker + pub owner_id: Option, + pub owner_name: Option, + pub size: Option, // except DeleteMarker + pub storage_class: Option, + pub is_latest: bool, // except ListObjects V1/V2 + pub version_id: Option, // except ListObjects V1/V2 + pub user_metadata: Option>, + pub is_prefix: bool, + pub is_delete_marker: bool, + pub encoding_type: Option, +} + +#[derive(Clone, Debug)] +pub struct Bucket { + pub name: String, + pub creation_date: UtcTime, +} + +#[derive(Clone, Debug)] +pub struct Part { + pub number: u16, + pub etag: String, +} + +#[derive(Clone, Debug)] +pub enum RetentionMode { + Governance, + Compliance, +} + +impl RetentionMode { + pub fn parse(s: &str) -> Result { + match s { + "GOVERNANCE" => Ok(RetentionMode::Governance), + "COMPLIANCE" => Ok(RetentionMode::Compliance), + _ => Err(Error::InvalidRetentionMode(s.to_string())), + } + } +} + +impl fmt::Display for RetentionMode { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + RetentionMode::Governance => write!(f, "GOVERNANCE"), + RetentionMode::Compliance => write!(f, "COMPLIANCE"), + } + } +} + +#[derive(Clone, Debug)] +pub struct Retention { + pub mode: RetentionMode, + pub retain_until_date: UtcTime, +} + +pub fn parse_legal_hold(s: &str) -> Result { + match s { + "ON" => Ok(true), + "OFF" => Ok(false), + _ => Err(Error::InvalidLegalHold(s.to_string())), + } +} + +#[derive(Clone, Debug, Copy)] +pub struct DeleteObject<'a> { + pub name: &'a str, + pub version_id: Option<&'a str>, +} + +#[derive(Clone, Debug)] +pub enum CompressionType { + NONE, + GZIP, + BZIP2, +} + +impl fmt::Display for CompressionType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + CompressionType::NONE => write!(f, "NONE"), + CompressionType::GZIP => write!(f, "GZIP"), + CompressionType::BZIP2 => write!(f, "BZIP2"), + } + } +} + +#[derive(Clone, Debug)] +pub enum FileHeaderInfo { + USE, + IGNORE, + NONE, +} + +impl fmt::Display for FileHeaderInfo { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + FileHeaderInfo::IGNORE => write!(f, "IGNORE"), + FileHeaderInfo::USE => write!(f, "USE"), + FileHeaderInfo::NONE => write!(f, "NONE"), + } + } +} + +#[derive(Clone, Debug)] +pub enum JsonType { + DOCUMENT, + LINES, +} + +impl fmt::Display for JsonType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + JsonType::DOCUMENT => write!(f, "DOCUMENT"), + JsonType::LINES => write!(f, "LINES"), + } + } +} + +#[derive(Clone, Debug)] +pub enum QuoteFields { + ALWAYS, + ASNEEDED, +} + +impl fmt::Display for QuoteFields { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + QuoteFields::ALWAYS => write!(f, "ALWAYS"), + QuoteFields::ASNEEDED => write!(f, "ASNEEDED"), + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct CsvInputSerialization { + pub compression_type: Option, + pub allow_quoted_record_delimiter: bool, + pub comments: Option, + pub field_delimiter: Option, + pub file_header_info: Option, + pub quote_character: Option, + pub quote_escape_character: Option, + pub record_delimiter: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct JsonInputSerialization { + pub compression_type: Option, + pub json_type: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct ParquetInputSerialization; + +#[derive(Clone, Debug, Default)] +pub struct CsvOutputSerialization { + pub field_delimiter: Option, + pub quote_character: Option, + pub quote_escape_character: Option, + pub quote_fields: Option, + pub record_delimiter: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct JsonOutputSerialization { + pub record_delimiter: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct SelectRequest<'a> { + pub expr: &'a str, + pub csv_input: Option, + pub json_input: Option, + pub parquet_input: Option, + pub csv_output: Option, + pub json_output: Option, + pub request_progress: bool, + pub scan_start_range: Option, + pub scan_end_range: Option, +} + +impl<'a> SelectRequest<'a> { + pub fn new_csv_input_output( + expr: &'a str, + csv_input: CsvInputSerialization, + csv_output: CsvOutputSerialization, + ) -> Result { + if expr.is_empty() { + return Err(Error::InvalidSelectExpression(String::from( + "select expression cannot be empty", + ))); + } + + Ok(SelectRequest { + expr: expr, + csv_input: Some(csv_input), + json_input: None, + parquet_input: None, + csv_output: Some(csv_output), + json_output: None, + request_progress: false, + scan_start_range: None, + scan_end_range: None, + }) + } + + pub fn new_csv_input_json_output( + expr: &'a str, + csv_input: CsvInputSerialization, + json_output: JsonOutputSerialization, + ) -> Result { + if expr.is_empty() { + return Err(Error::InvalidSelectExpression(String::from( + "select expression cannot be empty", + ))); + } + + Ok(SelectRequest { + expr: expr, + csv_input: Some(csv_input), + json_input: None, + parquet_input: None, + csv_output: None, + json_output: Some(json_output), + request_progress: false, + scan_start_range: None, + scan_end_range: None, + }) + } + + pub fn new_json_input_output( + expr: &'a str, + json_input: JsonInputSerialization, + json_output: JsonOutputSerialization, + ) -> Result { + if expr.is_empty() { + return Err(Error::InvalidSelectExpression(String::from( + "select expression cannot be empty", + ))); + } + + Ok(SelectRequest { + expr: expr, + csv_input: None, + json_input: Some(json_input), + parquet_input: None, + csv_output: None, + json_output: Some(json_output), + request_progress: false, + scan_start_range: None, + scan_end_range: None, + }) + } + + pub fn new_parquet_input_csv_output( + expr: &'a str, + parquet_input: ParquetInputSerialization, + csv_output: CsvOutputSerialization, + ) -> Result { + if expr.is_empty() { + return Err(Error::InvalidSelectExpression(String::from( + "select expression cannot be empty", + ))); + } + + Ok(SelectRequest { + expr: expr, + csv_input: None, + json_input: None, + parquet_input: Some(parquet_input), + csv_output: Some(csv_output), + json_output: None, + request_progress: false, + scan_start_range: None, + scan_end_range: None, + }) + } + + pub fn new_parquet_input_json_output( + expr: &'a str, + parquet_input: ParquetInputSerialization, + json_output: JsonOutputSerialization, + ) -> Result { + if expr.is_empty() { + return Err(Error::InvalidSelectExpression(String::from( + "select expression cannot be empty", + ))); + } + + Ok(SelectRequest { + expr: expr, + csv_input: None, + json_input: None, + parquet_input: Some(parquet_input), + csv_output: None, + json_output: Some(json_output), + request_progress: false, + scan_start_range: None, + scan_end_range: None, + }) + } + + pub fn to_xml(&self) -> String { + let mut data = String::from(""); + + data.push_str(""); + data.push_str(self.expr); + data.push_str(""); + data.push_str("SQL"); + + data.push_str(""); + if let Some(c) = &self.csv_input { + if let Some(v) = &c.compression_type { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + + data.push_str(""); + if c.allow_quoted_record_delimiter { + data.push_str("true"); + } + if let Some(v) = c.comments { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = c.field_delimiter { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = &c.file_header_info { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = c.quote_character { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = c.record_delimiter { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + data.push_str(""); + } else if let Some(j) = &self.json_input { + if let Some(v) = &j.compression_type { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + data.push_str(""); + if let Some(v) = &j.json_type { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + data.push_str(""); + } else if let Some(_) = &self.parquet_input { + data.push_str(""); + } + data.push_str(""); + + data.push_str(""); + if let Some(c) = &self.csv_output { + data.push_str(""); + if let Some(v) = c.field_delimiter { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = c.quote_character { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = c.quote_escape_character { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = &c.quote_fields { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + if let Some(v) = c.record_delimiter { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + data.push_str(""); + } else if let Some(j) = &self.json_output { + data.push_str(""); + if let Some(v) = j.record_delimiter { + data.push_str(""); + data.push_str(&v.to_string()); + data.push_str(""); + } + data.push_str(""); + } + data.push_str(""); + + if self.request_progress { + data.push_str("true"); + } + + if let Some(s) = self.scan_start_range { + if let Some(e) = self.scan_end_range { + data.push_str(""); + data.push_str(""); + data.push_str(&s.to_string()); + data.push_str(""); + data.push_str(""); + data.push_str(&e.to_string()); + data.push_str(""); + data.push_str(""); + } + } + + data.push_str(""); + return data; + } +} + +#[derive(Clone, Debug)] +pub struct SelectProgress { + pub bytes_scanned: usize, + pub bytes_progressed: usize, + pub bytes_returned: usize, +} diff --git a/src/s3/utils.rs b/src/s3/utils.rs new file mode 100644 index 0000000..898813b --- /dev/null +++ b/src/s3/utils.rs @@ -0,0 +1,282 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::error::Error; +pub use base64::encode as b64encode; +use byteorder::{BigEndian, ReadBytesExt}; +use chrono::{DateTime, NaiveDateTime, ParseError, Utc}; +use chrono_locale::LocaleDate; +use crc::{Crc, CRC_32_ISO_HDLC}; +use lazy_static::lazy_static; +use md5::compute as md5compute; +use multimap::MultiMap; +use regex::Regex; +use sha2::{Digest, Sha256}; +pub use urlencoding::decode as urldecode; +pub use urlencoding::encode as urlencode; +use xmltree::Element; + +pub type UtcTime = DateTime; + +pub type Multimap = MultiMap; + +pub fn merge(m1: &mut Multimap, m2: &Multimap) { + for (key, values) in m2.iter_all() { + for value in values { + m1.insert(key.to_string(), value.to_string()); + } + } +} + +pub fn crc32(data: &[u8]) -> u32 { + Crc::::new(&CRC_32_ISO_HDLC).checksum(data) +} + +pub fn uint32(mut data: &[u8]) -> Result { + data.read_u32::() +} + +pub fn sha256_hash(data: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(data); + return format!("{:x}", hasher.finalize()); +} + +pub fn md5sum_hash(data: &[u8]) -> String { + b64encode(md5compute(data).as_slice()) +} + +pub fn utc_now() -> UtcTime { + chrono::offset::Utc::now() +} + +pub fn to_signer_date(time: UtcTime) -> String { + time.format("%Y%m%d").to_string() +} + +pub fn to_amz_date(time: UtcTime) -> String { + time.format("%Y%m%dT%H%M%SZ").to_string() +} + +pub fn to_http_header_value(time: UtcTime) -> String { + time.formatl("%a, %d %b %Y %H:%M:%S GMT", "C").to_string() +} + +pub fn to_iso8601utc(time: UtcTime) -> String { + time.format("%Y-%m-%dT%H:%M:%S.%3fZ").to_string() +} + +pub fn from_iso8601utc(s: &str) -> Result { + Ok(DateTime::::from_utc( + match NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ") { + Ok(d) => d, + _ => NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%SZ")?, + }, + Utc, + )) +} + +pub fn from_http_header_value(s: &str) -> Result { + Ok(DateTime::::from_utc( + NaiveDateTime::parse_from_str(s, "%a, %d %b %Y %H:%M:%S GMT")?, + Utc, + )) +} + +pub fn to_http_headers(map: &Multimap) -> Vec { + let mut headers: Vec = Vec::new(); + for (key, values) in map.iter_all() { + for value in values { + let mut s = String::new(); + s.push_str(key); + s.push_str(": "); + s.push_str(value); + headers.push(s); + } + } + return headers; +} + +pub fn to_query_string(map: &Multimap) -> String { + let mut query = String::new(); + for (key, values) in map.iter_all() { + for value in values { + if !query.is_empty() { + query.push_str("&"); + } + query.push_str(&urlencode(key)); + query.push_str("="); + query.push_str(&urlencode(value)); + } + } + return query; +} + +pub fn get_canonical_query_string(map: &Multimap) -> String { + let mut keys: Vec = Vec::new(); + for (key, _) in map.iter() { + keys.push(key.to_string()); + } + keys.sort(); + + let mut query = String::new(); + for key in keys { + match map.get_vec(key.as_str()) { + Some(values) => { + for value in values { + if !query.is_empty() { + query.push_str("&"); + } + query.push_str(&urlencode(key.as_str())); + query.push_str("="); + query.push_str(&urlencode(value)); + } + } + None => todo!(), // This never happens. + }; + } + + return query; +} + +pub fn get_canonical_headers(map: &Multimap) -> (String, String) { + lazy_static! { + static ref MULTI_SPACE_REGEX: Regex = Regex::new("( +)").unwrap(); + } + let mut signed_headers: Vec = Vec::new(); + let mut mmap: MultiMap = MultiMap::new(); + + for (k, values) in map.iter_all() { + let key = k.to_lowercase(); + if "authorization" == key || "user-agent" == key { + continue; + } + if !signed_headers.contains(&key) { + signed_headers.push(key.clone()); + } + + for v in values { + mmap.insert(key.clone(), v.to_string()); + } + } + + let mut canonical_headers: Vec = Vec::new(); + for (key, values) in mmap.iter_all_mut() { + values.sort(); + let mut value = String::new(); + for v in values { + if !value.is_empty() { + value.push_str(","); + } + let s: String = MULTI_SPACE_REGEX.replace_all(v, " ").to_string(); + value.push_str(&s); + } + canonical_headers.push(key.to_string() + ":" + value.as_str()); + } + + signed_headers.sort(); + canonical_headers.sort(); + + return (signed_headers.join(";"), canonical_headers.join("\n")); +} + +pub fn check_bucket_name(bucket_name: &str, strict: bool) -> Result<(), Error> { + if bucket_name.trim().is_empty() { + return Err(Error::InvalidBucketName(String::from( + "bucket name cannot be empty", + ))); + } + + if bucket_name.len() < 3 { + return Err(Error::InvalidBucketName(String::from( + "bucket name cannot be less than 3 characters", + ))); + } + + if bucket_name.len() > 63 { + return Err(Error::InvalidBucketName(String::from( + "Bucket name cannot be greater than 63 characters", + ))); + } + + lazy_static! { + static ref VALID_IP_ADDR_REGEX: Regex = Regex::new("^(\\d+\\.){3}\\d+$").unwrap(); + static ref VALID_BUCKET_NAME_REGEX: Regex = + Regex::new("^[A-Za-z0-9][A-Za-z0-9\\.\\-_:]{1,61}[A-Za-z0-9]$").unwrap(); + static ref VALID_BUCKET_NAME_STRICT_REGEX: Regex = + Regex::new("^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$").unwrap(); + } + + if VALID_IP_ADDR_REGEX.is_match(bucket_name) { + return Err(Error::InvalidBucketName(String::from( + "bucket name cannot be an IP address", + ))); + } + + if bucket_name.contains("..") || bucket_name.contains(".-") || bucket_name.contains("-.") { + return Err(Error::InvalidBucketName(String::from( + "bucket name contains invalid successive characters '..', '.-' or '-.'", + ))); + } + + if strict { + if !VALID_BUCKET_NAME_STRICT_REGEX.is_match(bucket_name) { + return Err(Error::InvalidBucketName(String::from( + "bucket name does not follow S3 standards strictly", + ))); + } + } else if !VALID_BUCKET_NAME_REGEX.is_match(bucket_name) { + return Err(Error::InvalidBucketName(String::from( + "bucket name does not follow S3 standards", + ))); + } + + return Ok(()); +} + +pub fn get_text(element: &Element, tag: &str) -> Result { + Ok(element + .get_child(tag) + .ok_or(Error::XmlError(format!("<{}> tag not found", tag)))? + .get_text() + .ok_or(Error::XmlError(format!("text of <{}> tag not found", tag)))? + .to_string()) +} + +pub fn get_option_text(element: &Element, tag: &str) -> Result, Error> { + Ok(match element.get_child(tag) { + Some(v) => Some( + v.get_text() + .ok_or(Error::XmlError(format!("text of <{}> tag not found", tag)))? + .to_string(), + ), + None => None, + }) +} + +pub fn get_default_text(element: &Element, tag: &str) -> String { + element.get_child(tag).map_or(String::new(), |v| { + v.get_text().unwrap_or_default().to_string() + }) +} + +pub fn copy_slice(dst: &mut [u8], src: &[u8]) -> usize { + let mut c = 0; + for (d, s) in dst.iter_mut().zip(src.iter()) { + *d = *s; + c += 1; + } + c +} diff --git a/tests/tests.rs b/tests/tests.rs new file mode 100644 index 0000000..c5f9da3 --- /dev/null +++ b/tests/tests.rs @@ -0,0 +1,453 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rand::distributions::{Alphanumeric, DistString}; +use std::io::BufReader; + +use minio::s3::args::*; +use minio::s3::client::Client; +use minio::s3::creds::StaticProvider; +use minio::s3::http::BaseUrl; +use minio::s3::types::{ + CsvInputSerialization, CsvOutputSerialization, DeleteObject, FileHeaderInfo, QuoteFields, + SelectRequest, +}; + +struct RandReader { + size: usize, +} + +impl RandReader { + fn new(size: usize) -> RandReader { + RandReader { size: size } + } +} + +impl std::io::Read for RandReader { + fn read(&mut self, buf: &mut [u8]) -> Result { + let bytes_read = match self.size > buf.len() { + true => buf.len(), + false => self.size, + }; + + if bytes_read > 0 { + let random: &mut dyn rand::RngCore = &mut rand::thread_rng(); + random.fill_bytes(&mut buf[0..bytes_read]); + } + + self.size -= bytes_read; + + Ok(bytes_read) + } +} + +fn rand_bucket_name() -> String { + Alphanumeric + .sample_string(&mut rand::thread_rng(), 8) + .to_lowercase() +} + +fn rand_object_name() -> String { + Alphanumeric.sample_string(&mut rand::thread_rng(), 8) +} + +struct ClientTest<'a> { + client: &'a Client<'a>, + test_bucket: String, +} + +impl<'a> ClientTest<'a> { + fn new(client: &'a Client<'_>, test_bucket: &'a str) -> ClientTest<'a> { + ClientTest { + client: client, + test_bucket: test_bucket.to_string(), + } + } + + async fn bucket_exists(&self) { + let bucket_name = rand_bucket_name(); + self.client + .make_bucket(&MakeBucketArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + let exists = self + .client + .bucket_exists(&BucketExistsArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + assert_eq!(exists, true); + self.client + .remove_bucket(&RemoveBucketArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + } + + async fn list_buckets(&self) { + let mut names: Vec = Vec::new(); + for _ in 1..=3 { + names.push(rand_bucket_name()); + } + + for b in names.iter() { + self.client + .make_bucket(&MakeBucketArgs::new(&b).unwrap()) + .await + .unwrap(); + } + + let mut count = 0; + let resp = self + .client + .list_buckets(&ListBucketsArgs::new()) + .await + .unwrap(); + for bucket in resp.buckets.iter() { + if names.contains(&bucket.name) { + count += 1; + } + } + assert_eq!(count, 3); + + for b in names.iter() { + self.client + .remove_bucket(&RemoveBucketArgs::new(&b).unwrap()) + .await + .unwrap(); + } + } + + async fn put_object(&self) { + let object_name = rand_object_name(); + let size = 16_usize; + self.client + .put_object( + &mut PutObjectArgs::new( + &self.test_bucket, + &object_name, + &mut RandReader::new(size), + Some(size), + None, + ) + .unwrap(), + ) + .await + .unwrap(); + let resp = self + .client + .stat_object(&StatObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + assert_eq!(resp.bucket_name, self.test_bucket); + assert_eq!(resp.object_name, object_name); + assert_eq!(resp.size, size); + self.client + .remove_object(&RemoveObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + } + + async fn put_object_multipart(&self) { + let object_name = rand_object_name(); + let size: usize = 16 + 5 * 1024 * 1024; + self.client + .put_object( + &mut PutObjectArgs::new( + &self.test_bucket, + &object_name, + &mut RandReader::new(size), + Some(size), + None, + ) + .unwrap(), + ) + .await + .unwrap(); + let resp = self + .client + .stat_object(&StatObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + assert_eq!(resp.bucket_name, self.test_bucket); + assert_eq!(resp.object_name, object_name); + assert_eq!(resp.size, size); + self.client + .remove_object(&RemoveObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + } + + async fn get_object(&self) { + let object_name = rand_object_name(); + let data = "hello, world"; + self.client + .put_object( + &mut PutObjectArgs::new( + &self.test_bucket, + &object_name, + &mut BufReader::new(data.as_bytes()), + Some(data.len()), + None, + ) + .unwrap(), + ) + .await + .unwrap(); + let resp = self + .client + .get_object(&GetObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + let got = resp.text().await.unwrap(); + assert_eq!(got, data); + self.client + .remove_object(&RemoveObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + } + + async fn remove_objects(&self) { + let bucket_name = rand_bucket_name(); + self.client + .make_bucket(&MakeBucketArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + + let mut names: Vec = Vec::new(); + for _ in 1..=3 { + let object_name = rand_object_name(); + let size = 0_usize; + self.client + .put_object( + &mut PutObjectArgs::new( + &self.test_bucket, + &object_name, + &mut RandReader::new(size), + Some(size), + None, + ) + .unwrap(), + ) + .await + .unwrap(); + names.push(object_name); + } + let mut objects: Vec = Vec::new(); + for name in names.iter() { + objects.push(DeleteObject { + name: &name, + version_id: None, + }); + } + + self.client + .remove_objects( + &mut RemoveObjectsArgs::new(&self.test_bucket, &mut objects.iter()).unwrap(), + ) + .await + .unwrap(); + + self.client + .remove_bucket(&RemoveBucketArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + } + + async fn list_objects(&self) { + let bucket_name = rand_bucket_name(); + self.client + .make_bucket(&MakeBucketArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + + let mut names: Vec = Vec::new(); + for _ in 1..=3 { + let object_name = rand_object_name(); + let size = 0_usize; + self.client + .put_object( + &mut PutObjectArgs::new( + &self.test_bucket, + &object_name, + &mut RandReader::new(size), + Some(size), + None, + ) + .unwrap(), + ) + .await + .unwrap(); + names.push(object_name); + } + + self.client + .list_objects( + &mut ListObjectsArgs::new(&self.test_bucket, &|res| { + let item = res.unwrap(); + assert_eq!(names.contains(&item.name), true); + true + }) + .unwrap(), + ) + .await + .unwrap(); + + let mut objects: Vec = Vec::new(); + for name in names.iter() { + objects.push(DeleteObject { + name: &name, + version_id: None, + }); + } + + self.client + .remove_objects( + &mut RemoveObjectsArgs::new(&self.test_bucket, &mut objects.iter()).unwrap(), + ) + .await + .unwrap(); + + self.client + .remove_bucket(&RemoveBucketArgs::new(&bucket_name).unwrap()) + .await + .unwrap(); + } + + async fn select_object_content(&self) { + let object_name = rand_object_name(); + let mut data = String::new(); + data.push_str("1997,Ford,E350,\"ac, abs, moon\",3000.00\n"); + data.push_str("1999,Chevy,\"Venture \"\"Extended Edition\"\"\",,4900.00\n"); + data.push_str("1999,Chevy,\"Venture \"\"Extended Edition, Very Large\"\"\",,5000.00\n"); + data.push_str("1996,Jeep,Grand Cherokee,\"MUST SELL!\n"); + data.push_str("air, moon roof, loaded\",4799.00\n"); + let body = String::from("Year,Make,Model,Description,Price\n") + &data; + + self.client + .put_object( + &mut PutObjectArgs::new( + &self.test_bucket, + &object_name, + &mut BufReader::new(body.as_bytes()), + Some(body.len()), + None, + ) + .unwrap(), + ) + .await + .unwrap(); + + let request = SelectRequest::new_csv_input_output( + "select * from S3Object", + CsvInputSerialization { + compression_type: None, + allow_quoted_record_delimiter: false, + comments: None, + field_delimiter: None, + file_header_info: Some(FileHeaderInfo::USE), + quote_character: None, + quote_escape_character: None, + record_delimiter: None, + }, + CsvOutputSerialization { + field_delimiter: None, + quote_character: None, + quote_escape_character: None, + quote_fields: Some(QuoteFields::ASNEEDED), + record_delimiter: None, + }, + ) + .unwrap(); + let mut resp = self + .client + .select_object_content( + &SelectObjectContentArgs::new(&self.test_bucket, &object_name, &request).unwrap(), + ) + .await + .unwrap(); + let mut got = String::new(); + let mut buf = [0_u8; 512]; + loop { + let size = resp.read(&mut buf).await.unwrap(); + if size == 0 { + break; + } + got += &String::from_utf8(buf[..size].to_vec()).unwrap(); + } + assert_eq!(got, data); + self.client + .remove_object(&RemoveObjectArgs::new(&self.test_bucket, &object_name).unwrap()) + .await + .unwrap(); + } +} + +#[tokio::main] +#[test] +async fn s3_tests() -> Result<(), Box> { + let host = std::env::var("SERVER_ENDPOINT")?; + let access_key = std::env::var("ACCESS_KEY")?; + let secret_key = std::env::var("SECRET_KEY")?; + let secure = std::env::var("ENABLE_HTTPS").is_ok(); + let ignore_cert_check = std::env::var("IGNORE_CERT_CHECK").is_ok(); + let region = std::env::var("SERVER_REGION").ok(); + + let mut burl = BaseUrl::from_string(host).unwrap(); + burl.https = secure; + if let Some(v) = region { + burl.region = v; + } + + let provider = StaticProvider::new(&access_key, &secret_key, None); + let client = Client::new(burl.clone(), Some(&provider)); + + let test_bucket = rand_bucket_name(); + client + .make_bucket(&MakeBucketArgs::new(&test_bucket).unwrap()) + .await + .unwrap(); + + let ctest = ClientTest::new(&client, &test_bucket); + + println!("make_bucket() + bucket_exists() + remove_bucket()"); + ctest.bucket_exists().await; + + println!("list_buckets()"); + ctest.list_buckets().await; + + println!("put_object() + stat_object() + remove_object()"); + ctest.put_object().await; + + println!("[Multipart] put_object()"); + ctest.put_object_multipart().await; + + println!("get_object()"); + ctest.get_object().await; + + println!("remove_objects()"); + ctest.remove_objects().await; + + println!("list_objects()"); + ctest.list_objects().await; + + println!("select_object_content()"); + ctest.select_object_content().await; + + client + .remove_bucket(&RemoveBucketArgs::new(&test_bucket).unwrap()) + .await + .unwrap(); + + Ok(()) +}