diff --git a/Cargo.toml b/Cargo.toml index d8c870d..39a557b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ futures-util = "0.3.31" hex = "0.4.3" hmac = "0.12.1" home = "0.5.9" -http = "1.1.0" +http = "1.2.0" hyper = { version = "1.5.1", features = ["full"] } lazy_static = "1.5.0" log = "0.4.22" @@ -61,4 +61,7 @@ clap = { version = "4.5.23", features = ["derive"] } quickcheck = "1.0.3" [[example]] -name = "file-uploader" +name = "file_uploader" + +[[example]] +name = "object_prompt" diff --git a/README.md b/README.md index 0fe4a08..06cc884 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,20 @@ MinIO Rust SDK is Simple Storage Service (aka S3) client to perform bucket and o For a complete list of APIs and examples, please take a look at the [MinIO Rust Client API Reference](https://minio-rs.min.io/) -## Example:: file-uploader.rs +## Examples -[Upload a file to MinIO](examples/file-uploader.rs) +Run the examples from the command line with: +`cargo run --example ` + +### file_uploader.rs + +* [Upload a file to MinIO](examples/file_uploader) +* [Upload a file to MinIO with CLI](examples/put_object) + +### object_prompt.rs + +* [Prompt a file on MinIO](examples/object_prompt) ## License This SDK is distributed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](https://github.com/minio/minio-rs/blob/master/LICENSE) for more information. diff --git a/examples/cat.png b/examples/cat.png new file mode 100644 index 0000000..0ce7442 Binary files /dev/null and b/examples/cat.png differ diff --git a/examples/file-uploader.rs b/examples/file_uploader.rs similarity index 54% rename from examples/file-uploader.rs rename to examples/file_uploader.rs index 0b485fc..454c983 100644 --- a/examples/file-uploader.rs +++ b/examples/file_uploader.rs @@ -1,4 +1,18 @@ -use log::info; +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use minio::s3::args::{BucketExistsArgs, MakeBucketArgs}; use minio::s3::builders::ObjectContent; use minio::s3::client::ClientBuilder; @@ -11,8 +25,7 @@ async fn main() -> Result<(), Box> { env_logger::init(); // Note: set environment variable RUST_LOG="INFO" to log info and higher let base_url = "https://play.min.io".parse::()?; - - info!("Trying to connect to MinIO at: `{:?}`", base_url); + log::info!("Trying to connect to MinIO at: `{:?}`", base_url); let static_provider = StaticProvider::new( "Q3AM3UQ867SPQQA43P2F", @@ -24,7 +37,7 @@ async fn main() -> Result<(), Box> { .provider(Some(Box::new(static_provider))) .build()?; - let bucket_name: &str = "asiatrip"; + let bucket_name: &str = "file-upload-rust-bucket"; // Check 'bucket_name' bucket exist or not. let exists: bool = client @@ -41,12 +54,17 @@ async fn main() -> Result<(), Box> { } // File we are going to upload to the bucket - let filename: &Path = Path::new("/tmp/asiaphotos.zip"); + let filename: &Path = Path::new("./examples/cat.png"); // Name of the object that will be stored in the bucket - let object_name: &str = "asiaphotos-2015.zip"; + let object_name: &str = "cat.png"; - info!("filename {}", &filename.to_str().unwrap()); + if filename.exists() { + log::info!("File '{}' exists.", &filename.to_str().unwrap()); + } else { + log::error!("File '{}' does not exist.", &filename.to_str().unwrap()); + return Ok(()); + } let content = ObjectContent::from(filename); client @@ -54,8 +72,8 @@ async fn main() -> Result<(), Box> { .send() .await?; - info!( - "file `{}` is successfully uploaded as object `{object_name}` to bucket `{bucket_name}`.", + log::info!( + "file '{}' is successfully uploaded as object '{object_name}' to bucket '{bucket_name}'.", filename.display() ); Ok(()) diff --git a/examples/object_prompt.rs b/examples/object_prompt.rs new file mode 100644 index 0000000..4210e04 --- /dev/null +++ b/examples/object_prompt.rs @@ -0,0 +1,90 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use minio::s3::args::{BucketExistsArgs, MakeBucketArgs}; +use minio::s3::builders::{ObjectContent, ObjectPrompt}; +use minio::s3::client::ClientBuilder; +use minio::s3::creds::StaticProvider; +use minio::s3::http::BaseUrl; +use minio::s3::types::S3Api; +use std::path::Path; + +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::init(); // Note: set environment variable RUST_LOG="INFO" to log info and higher + + let base_url = "https://play.min.io".parse::()?; + log::info!("Trying to connect to MinIO at: `{:?}`", base_url); + + let static_provider = StaticProvider::new( + "Q3AM3UQ867SPQQA43P2F", + "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + None, + ); + + let client = ClientBuilder::new(base_url.clone()) + .provider(Some(Box::new(static_provider))) + .ignore_cert_check(Some(true)) + .build()?; + + let bucket_name: &str = "object-prompt-rust-bucket"; + + // Check 'bucket_name' bucket exist or not. + let exists: bool = client + .bucket_exists(&BucketExistsArgs::new(bucket_name).unwrap()) + .await + .unwrap(); + + // Make 'bucket_name' bucket if not exist. + if !exists { + client + .make_bucket(&MakeBucketArgs::new(bucket_name).unwrap()) + .await + .unwrap(); + } + + // File we are going to upload to the bucket + let filename: &Path = Path::new("./examples/cat.png"); + + // Name of the object that will be stored in the bucket + let object_name: &str = "cat.png"; + + if filename.exists() { + log::info!("File '{}' exists.", &filename.to_str().unwrap()); + } else { + log::error!("File '{}' does not exist.", &filename.to_str().unwrap()); + return Ok(()); + } + + let content = ObjectContent::from(filename); + client + .put_object_content(bucket_name, object_name, content) + .send() + .await?; + + log::info!( + "File '{}' is successfully uploaded as object '{object_name}' to bucket '{bucket_name}'.", + filename.display() + ); + + let op = ObjectPrompt::new(bucket_name, object_name, "what is it about?") + //.lambda_arn("arn:minio:s3-object-lambda::_:webhook") // this is the default value + .client(&client); + + let res = op.send().await?; + log::info!("Object prompt result: '{}'", res.prompt_response); + + Ok(()) +} diff --git a/examples/put-object.rs b/examples/put_object.rs similarity index 100% rename from examples/put-object.rs rename to examples/put_object.rs diff --git a/src/s3/builders.rs b/src/s3/builders.rs index d114a59..9df7c40 100644 --- a/src/s3/builders.rs +++ b/src/s3/builders.rs @@ -1,3 +1,6 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2024 MinIO, Inc. +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -17,6 +20,7 @@ mod get_object; mod list_objects; mod listen_bucket_notification; mod object_content; +mod object_prompt; mod put_object; mod remove_objects; @@ -25,5 +29,6 @@ pub use get_object::*; pub use list_objects::*; pub use listen_bucket_notification::*; pub use object_content::*; +pub use object_prompt::*; pub use put_object::*; pub use remove_objects::*; diff --git a/src/s3/builders/get_object.rs b/src/s3/builders/get_object.rs index b8da547..69f2030 100644 --- a/src/s3/builders/get_object.rs +++ b/src/s3/builders/get_object.rs @@ -178,8 +178,7 @@ impl ToS3Request for GetObject { "object name cannot be empty", ))); } - - let client = self.client.clone().ok_or(Error::NoClientProvided)?; + let client: &Client = self.client.as_ref().ok_or(Error::NoClientProvided)?; if self.ssec.is_some() && !client.is_secure() { return Err(Error::SseTlsRequired(None)); @@ -199,15 +198,12 @@ impl ToS3Request for GetObject { query_params.insert(String::from("versionId"), v.to_string()); } - let req = S3Request::new( - self.client.as_ref().ok_or(Error::NoClientProvided)?, - Method::GET, - ) - .region(self.region.as_deref()) - .bucket(Some(&self.bucket)) - .object(Some(&self.object)) - .query_params(query_params) - .headers(headers); + let req = S3Request::new(client, Method::GET) + .region(self.region.as_deref()) + .bucket(Some(&self.bucket)) + .object(Some(&self.object)) + .query_params(query_params) + .headers(headers); Ok(req) } diff --git a/src/s3/builders/object_prompt.rs b/src/s3/builders/object_prompt.rs new file mode 100644 index 0000000..45d8557 --- /dev/null +++ b/src/s3/builders/object_prompt.rs @@ -0,0 +1,156 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::builders::SegmentedBytes; +use crate::s3::sse::{Sse, SseCustomerKey}; +use crate::s3::utils::{check_bucket_name, merge, Multimap}; +use crate::s3::{ + client::Client, + error::Error, + response::ObjectPromptResponse, + types::{S3Api, S3Request, ToS3Request}, +}; +use bytes::Bytes; +use http::Method; +use serde_json::json; + +#[derive(Debug, Clone, Default)] +pub struct ObjectPrompt { + client: Option, + bucket: String, + object: String, + prompt: String, + lambda_arn: Option, + + version_id: Option, + region: Option, + ssec: Option, + extra_headers: Option, + extra_query_params: Option, +} + +// builder interface +impl ObjectPrompt { + pub fn new(bucket: &str, object: &str, prompt: &str) -> Self { + ObjectPrompt { + client: None, + bucket: bucket.to_string(), + object: object.to_string(), + prompt: prompt.to_string(), + ..Default::default() + } + } + + pub fn client(mut self, client: &Client) -> Self { + self.client = Some(client.clone()); + self + } + + pub fn lambda_arn(mut self, lambda_arn: &str) -> Self { + self.lambda_arn = Some(lambda_arn.to_string()); + self + } + + pub fn extra_headers(mut self, extra_headers: Option) -> Self { + self.extra_headers = extra_headers; + self + } + + pub fn extra_query_params(mut self, extra_query_params: Option) -> Self { + self.extra_query_params = extra_query_params; + self + } + + pub fn version_id(mut self, version_id: Option) -> Self { + self.version_id = version_id; + self + } + + pub fn region(mut self, region: Option) -> Self { + self.region = region; + self + } + + pub fn ssec(mut self, ssec: Option) -> Self { + self.ssec = ssec; + self + } +} + +// internal helpers +impl ObjectPrompt { + fn get_headers(&self) -> Multimap { + let mut headers = Multimap::new(); + if let Some(v) = &self.ssec { + merge(&mut headers, &v.headers()); + } + headers + } +} + +impl ToS3Request for ObjectPrompt { + fn to_s3request(&self) -> Result { + check_bucket_name(&self.bucket, true)?; + + if self.object.is_empty() { + return Err(Error::InvalidObjectName(String::from( + "object name cannot be empty", + ))); + } + let client: &Client = self.client.as_ref().ok_or(Error::NoClientProvided)?; + + if self.ssec.is_some() && !client.is_secure() { + return Err(Error::SseTlsRequired(None)); + } + + let mut headers = Multimap::new(); + if let Some(v) = &self.extra_headers { + merge(&mut headers, v); + } + merge(&mut headers, &self.get_headers()); + + let mut query_params = Multimap::new(); + if let Some(v) = &self.extra_query_params { + merge(&mut query_params, v); + } + if let Some(v) = &self.version_id { + query_params.insert(String::from("versionId"), v.to_string()); + } + query_params.insert( + String::from("lambdaArn"), + self.lambda_arn + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + ); + + let prompt_body = json!({ "prompt": self.prompt }); + let body: SegmentedBytes = SegmentedBytes::from(Bytes::from(prompt_body.to_string())); + + let req = S3Request::new(client, Method::POST) + .region(self.region.as_deref()) + .bucket(Some(&self.bucket)) + .object(Some(&self.object)) + .query_params(query_params) + .headers(headers) + .body(Some(body)); + + Ok(req) + } +} + +impl S3Api for ObjectPrompt { + type S3Response = ObjectPromptResponse; +} diff --git a/src/s3/builders/put_object.rs b/src/s3/builders/put_object.rs index e6beecf..3719340 100644 --- a/src/s3/builders/put_object.rs +++ b/src/s3/builders/put_object.rs @@ -682,9 +682,9 @@ fn object_write_args_headers( Ok(map) } -// PutObjectContent takes a `ObjectContent` stream and uploads it to MinIO/S3. -// -// It is a higher level API and handles multipart uploads transparently. +/// PutObjectContent takes a `ObjectContent` stream and uploads it to MinIO/S3. +/// +/// It is a higher level API and handles multipart uploads transparently. pub struct PutObjectContent { client: Option, @@ -1048,8 +1048,8 @@ pub const MAX_PART_SIZE: u64 = 1024 * MIN_PART_SIZE; // 5 GiB pub const MAX_OBJECT_SIZE: u64 = 1024 * MAX_PART_SIZE; // 5 TiB pub const MAX_MULTIPART_COUNT: u16 = 10_000; -// Returns the size of each part to upload and the total number of parts. The -// number of parts is `None` when the object size is unknown. +/// Returns the size of each part to upload and the total number of parts. The +/// number of parts is `None` when the object size is unknown. fn calc_part_info(object_size: Size, part_size: Size) -> Result<(u64, Option), Error> { // Validate arguments against limits. if let Size::Known(v) = part_size { diff --git a/src/s3/builders/remove_objects.rs b/src/s3/builders/remove_objects.rs index 16943dd..bb2b540 100644 --- a/src/s3/builders/remove_objects.rs +++ b/src/s3/builders/remove_objects.rs @@ -126,10 +126,6 @@ impl RemoveObject { } } -impl S3Api for RemoveObject { - type S3Response = RemoveObjectResponse; -} - impl ToS3Request for RemoveObject { fn to_s3request(&self) -> Result { check_bucket_name(&self.bucket, true)?; @@ -160,6 +156,10 @@ impl ToS3Request for RemoveObject { } } +impl S3Api for RemoveObject { + type S3Response = RemoveObjectResponse; +} + #[derive(Debug, Clone)] pub struct RemoveObjectsApi { client: Option, @@ -302,7 +302,10 @@ impl From for DeleteObjects { } } -impl + Send + Sync + 'static> From for DeleteObjects { +impl From for DeleteObjects +where + I: Iterator + Send + Sync + 'static, +{ fn from(keys: I) -> Self { DeleteObjects::from_stream(stream_iter(keys)) } diff --git a/src/s3/client.rs b/src/s3/client.rs index 4835020..3590bc5 100644 --- a/src/s3/client.rs +++ b/src/s3/client.rs @@ -50,6 +50,7 @@ use xmltree::Element; mod get_object; mod list_objects; mod listen_bucket_notification; +mod object_prompt; mod put_object; mod remove_objects; diff --git a/src/s3/client/object_prompt.rs b/src/s3/client/object_prompt.rs new file mode 100644 index 0000000..3d62969 --- /dev/null +++ b/src/s3/client/object_prompt.rs @@ -0,0 +1,27 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! S3 APIs for downloading objects. + +use crate::s3::builders::ObjectPrompt; + +use super::Client; + +impl Client { + /// Create a ObjectPrompt request builder. Prompt an object using natural language. + pub fn object_prompt(&self, bucket: &str, object: &str, prompt: &str) -> ObjectPrompt { + ObjectPrompt::new(bucket, object, prompt).client(self) + } +} diff --git a/src/s3/response.rs b/src/s3/response.rs index c1ec984..9628218 100644 --- a/src/s3/response.rs +++ b/src/s3/response.rs @@ -35,6 +35,7 @@ mod buckets; mod get_object; pub(crate) mod list_objects; mod listen_bucket_notification; +mod object_prompt; mod put_object; mod remove_objects; @@ -42,6 +43,7 @@ pub use buckets::{GetBucketVersioningResponse, ListBucketsResponse}; pub use get_object::GetObjectResponse; pub use list_objects::ListObjectsResponse; pub use listen_bucket_notification::ListenBucketNotificationResponse; +pub use object_prompt::ObjectPromptResponse; pub use put_object::{ AbortMultipartUploadResponse2, CompleteMultipartUploadResponse2, CreateMultipartUploadResponse2, PutObjectContentResponse, PutObjectResponse, @@ -381,7 +383,7 @@ impl SelectObjectContentResponse { offset += 1; let b1 = self.data[offset] as u16; offset += 1; - length = (b0 << 8 | b1) as usize; + length = ((b0 << 8) | b1) as usize; let value = String::from_utf8(self.data[offset..offset + length].to_vec())?; offset += length; diff --git a/src/s3/response/object_prompt.rs b/src/s3/response/object_prompt.rs new file mode 100644 index 0000000..9abc1ac --- /dev/null +++ b/src/s3/response/object_prompt.rs @@ -0,0 +1,61 @@ +// MinIO Rust Library for Amazon S3 Compatible Cloud Storage +// Copyright 2024 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::s3::error::Error; +use crate::s3::types::{FromS3Response, S3Request}; +use async_trait::async_trait; + +pub struct ObjectPromptResponse { + pub headers: http::HeaderMap, + pub region: String, + pub bucket_name: String, + pub object_name: String, + pub prompt_response: String, +} + +#[async_trait] +impl FromS3Response for ObjectPromptResponse { + async fn from_s3response<'a>( + req: S3Request<'a>, + response: reqwest::Response, + ) -> Result { + let headers = response.headers().clone(); + let body = response.bytes().await?; + let prompt_response: String = String::from_utf8(body.to_vec())?; + let region: String = req.region.unwrap_or("").to_string(); // Keep this since it defaults to an empty string + + let bucket_name: String = req + .bucket + .ok_or_else(|| { + Error::InvalidBucketName(String::from("Missing bucket name in request")) + })? + .to_string(); + + let object_name: String = req + .object + .ok_or_else(|| { + Error::InvalidObjectName(String::from("Missing object name in request")) + })? + .to_string(); + + Ok(ObjectPromptResponse { + headers, + region, + bucket_name, + object_name, + prompt_response, + }) + } +} diff --git a/src/s3/response/put_object.rs b/src/s3/response/put_object.rs index e5cbae9..a5f69e3 100644 --- a/src/s3/response/put_object.rs +++ b/src/s3/response/put_object.rs @@ -74,16 +74,34 @@ impl FromS3Response for CreateMultipartUploadResponse2 { req: S3Request<'a>, response: reqwest::Response, ) -> Result { - let header_map = response.headers().clone(); + let headers = response.headers().clone(); let body = response.bytes().await?; let root = Element::parse(body.reader())?; + let region: String = req.region.unwrap_or("").to_string(); // Keep this since it defaults to an empty string + + let bucket_name: String = req + .bucket + .ok_or_else(|| { + Error::InvalidBucketName(String::from("Missing bucket name in request")) + })? + .to_string(); + + let object_name: String = req + .object + .ok_or_else(|| { + Error::InvalidObjectName(String::from("Missing object name in request")) + })? + .to_string(); + + let upload_id: String = get_text(&root, "UploadId")?; + Ok(CreateMultipartUploadResponse2 { - headers: header_map.clone(), - region: req.region.unwrap_or("").to_string(), - bucket_name: req.bucket.unwrap().to_string(), - object_name: req.object.unwrap().to_string(), - upload_id: get_text(&root, "UploadId")?, + headers, + region, + bucket_name, + object_name, + upload_id, }) } } diff --git a/src/s3/utils.rs b/src/s3/utils.rs index 9cf9716..0e8e3bf 100644 --- a/src/s3/utils.rs +++ b/src/s3/utils.rs @@ -335,7 +335,9 @@ pub fn match_region(value: &str) -> bool { /// Validates given bucket name pub fn check_bucket_name(bucket_name: &str, strict: bool) -> Result<(), Error> { - if bucket_name.trim().is_empty() { + let bucket_name: &str = bucket_name.trim(); + + if bucket_name.is_empty() { return Err(Error::InvalidBucketName(String::from( "bucket name cannot be empty", ))); @@ -368,20 +370,23 @@ pub fn check_bucket_name(bucket_name: &str, strict: bool) -> Result<(), Error> { } if bucket_name.contains("..") || bucket_name.contains(".-") || bucket_name.contains("-.") { - return Err(Error::InvalidBucketName(String::from( - "bucket name contains invalid successive characters '..', '.-' or '-.'", + return Err(Error::InvalidBucketName(format!( + "bucket name ('{}') contains invalid successive characters '..', '.-' or '-.'", + bucket_name ))); } if strict { if !VALID_BUCKET_NAME_STRICT_REGEX.is_match(bucket_name) { - return Err(Error::InvalidBucketName(String::from( - "bucket name does not follow S3 standards strictly", + return Err(Error::InvalidBucketName(format!( + "bucket name ('{}') does not follow S3 standards strictly", + bucket_name ))); } } else if !VALID_BUCKET_NAME_REGEX.is_match(bucket_name) { - return Err(Error::InvalidBucketName(String::from( - "bucket name does not follow S3 standards", + return Err(Error::InvalidBucketName(format!( + "bucket name ('{}') does not follow S3 standards", + bucket_name ))); }