feat: Make gcs and s3 credentials optional

crates/datasources/src/object_store/errors.rs

@@ -22,6 +22,9 @@ pub enum ObjectStoreSourceError {
 
     #[error("This file type is not supported: {0}")]
     NotSupportFileType(String),
+
+    #[error("{0}")]
+    Static(&'static str),
 }
 
 pub type Result<T, E = ObjectStoreSourceError> = std::result::Result<T, E>;

crates/datasources/src/object_store/gcs.rs

@@ -19,12 +19,22 @@ pub struct GcsTableAccess {
     /// GCS object store bucket name
     pub bucket_name: String,
     /// GCS object store service account key
-    pub service_acccount_key_json: String,
+    pub service_acccount_key_json: Option<String>,
     /// GCS object store table location
     pub location: String,
     pub file_type: Option<FileType>,
 }
 
+impl GcsTableAccess {
+    fn builder(&self) -> GoogleCloudStorageBuilder {
+        let builder = GoogleCloudStorageBuilder::new().with_bucket_name(&self.bucket_name);
+        match &self.service_acccount_key_json {
+            Some(key) => builder.with_service_account_key(key),
+            None => builder,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct GcsAccessor {
     /// GCS object store access info
@@ -47,12 +57,7 @@ impl TableAccessor for GcsAccessor {
 impl GcsAccessor {
     /// Setup accessor for GCS
     pub async fn new(access: GcsTableAccess) -> Result<Self> {
-        let store = Arc::new(
-            GoogleCloudStorageBuilder::new()
-                .with_service_account_key(access.service_acccount_key_json)
-                .with_bucket_name(access.bucket_name)
-                .build()?,
-        );
+        let store = Arc::new(access.builder().build()?);
 
         let location = ObjectStorePath::from(access.location);
         // Use provided file type or infer from location
@@ -69,13 +74,7 @@ impl GcsAccessor {
     }
 
     pub async fn validate_table_access(access: GcsTableAccess) -> Result<()> {
-        let store = Arc::new(
-            GoogleCloudStorageBuilder::new()
-                .with_service_account_key(access.service_acccount_key_json)
-                .with_bucket_name(access.bucket_name)
-                .build()?,
-        );
-
+        let store = Arc::new(access.builder().build()?);
         let location = ObjectStorePath::from(access.location);
         store.head(&location).await?;
         Ok(())

crates/datasources/src/object_store/s3.rs

@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use tracing::trace;
 
 use super::csv::CsvTableProvider;
-use super::errors::Result;
+use super::errors::{ObjectStoreSourceError, Result};
 use super::parquet::ParquetTableProvider;
 use super::{file_type_from_path, FileType, TableAccessor};
 
@@ -20,14 +20,31 @@ pub struct S3TableAccess {
     /// S3 object store bucket name
     pub bucket_name: String,
     /// S3 object store access key id
-    pub access_key_id: String,
+    pub access_key_id: Option<String>,
     /// S3 object store secret access key
-    pub secret_access_key: String,
+    pub secret_access_key: Option<String>,
     /// S3 object store table location
     pub location: String,
     pub file_type: Option<FileType>,
 }
 
+impl S3TableAccess {
+    fn builder(&self) -> Result<AmazonS3Builder> {
+        let builder = AmazonS3Builder::new()
+            .with_region(&self.region)
+            .with_bucket_name(&self.bucket_name);
+        match (&self.access_key_id, &self.secret_access_key) {
+            (Some(id), Some(secret)) => Ok(builder
+                .with_access_key_id(id)
+                .with_secret_access_key(secret)),
+            (None, None) => Ok(builder),
+            _ => Err(ObjectStoreSourceError::Static(
+                "Access key id and secret must both be provided",
+            )),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct S3Accessor {
     /// S3 object store access info
@@ -50,14 +67,7 @@ impl TableAccessor for S3Accessor {
 impl S3Accessor {
     /// Setup accessor for S3
     pub async fn new(access: S3TableAccess) -> Result<Self> {
-        let store = Arc::new(
-            AmazonS3Builder::new()
-                .with_region(access.region)
-                .with_bucket_name(access.bucket_name)
-                .with_access_key_id(access.access_key_id)
-                .with_secret_access_key(access.secret_access_key)
-                .build()?,
-        );
+        let store = Arc::new(access.builder()?.build()?);
 
         let location = ObjectStorePath::from(access.location);
         // Use provided file type or infer from location
@@ -74,14 +84,7 @@ impl S3Accessor {
     }
 
     pub async fn validate_table_access(access: S3TableAccess) -> Result<()> {
-        let store = Arc::new(
-            AmazonS3Builder::new()
-                .with_region(access.region)
-                .with_bucket_name(access.bucket_name)
-                .with_access_key_id(access.access_key_id)
-                .with_secret_access_key(access.secret_access_key)
-                .build()?,
-        );
+        let store = Arc::new(access.builder()?.build()?);
 
         let location = ObjectStorePath::from(access.location);
         store.head(&location).await?;

crates/metastore/proto/options.proto

@@ -117,14 +117,14 @@ message TableOptionsMysql {
 message TableOptionsLocal { string location = 1; }
 
 message TableOptionsGcs {
-  string service_account_key = 1;
+  optional string service_account_key = 1;
   string bucket = 2;
   string location = 3;
 }
 
 message TableOptionsS3 {
-  string access_key_id = 1;
-  string secret_access_key = 2;
+  optional string access_key_id = 1;
+  optional string secret_access_key = 2;
   string region = 3;
   string bucket = 4;
   string location = 5;

crates/metastore/src/types/options.rs

@@ -585,7 +585,7 @@ impl From<TableOptionsLocal> for options::TableOptionsLocal {
 
 #[derive(Debug, Clone, Arbitrary, PartialEq, Eq)]
 pub struct TableOptionsGcs {
-    pub service_account_key: String,
+    pub service_account_key: Option<String>,
     pub bucket: String,
     pub location: String,
 }
@@ -613,8 +613,8 @@ impl From<TableOptionsGcs> for options::TableOptionsGcs {
 
 #[derive(Debug, Clone, Arbitrary, PartialEq, Eq)]
 pub struct TableOptionsS3 {
-    pub access_key_id: String,
-    pub secret_access_key: String,
+    pub access_key_id: Option<String>,
+    pub secret_access_key: Option<String>,
     pub region: String,
     pub bucket: String,
     pub location: String,

crates/sqlexec/src/planner/session_planner.rs

@@ -344,7 +344,7 @@ impl<'a> SessionPlanner<'a> {
                 TableOptions::Local(TableOptionsLocal { location })
             }
             TableOptions::GCS => {
-                let service_account_key = remove_required_opt(m, "service_account_key")?;
+                let service_account_key = remove_optional_opt(m, "service_account_key")?;
                 let bucket = remove_required_opt(m, "bucket")?;
                 let location = remove_required_opt(m, "location")?;
 
@@ -368,8 +368,8 @@ impl<'a> SessionPlanner<'a> {
                 })
             }
             TableOptions::S3_STORAGE => {
-                let access_key_id = remove_required_opt(m, "access_key_id")?;
-                let secret_access_key = remove_required_opt(m, "secret_access_key")?;
+                let access_key_id = remove_optional_opt(m, "access_key_id")?;
+                let secret_access_key = remove_optional_opt(m, "secret_access_key")?;
                 let region = remove_required_opt(m, "region")?;
                 let bucket = remove_required_opt(m, "bucket")?;
                 let location = remove_required_opt(m, "location")?;