diff --git a/deploy/terraform-custom-datacommons/README.md b/deploy/terraform-custom-datacommons/README.md index fd4281e396..4a411d9369 100644 --- a/deploy/terraform-custom-datacommons/README.md +++ b/deploy/terraform-custom-datacommons/README.md @@ -167,7 +167,7 @@ Once the deployment is complete, terraform should output something like: cloud_run_service_name = "-datacommons-web-service" cloud_run_service_url = "https://-datacommons-web-service-abc123-uc.a.run.app" dc_api_key = -dc_gcs_data_bucket_path = "-datacommons-data-" +gcs_data_bucket_name = "-datacommons-data-" maps_api_key = mysql_instance_connection_name = ":us-central1:-datacommons-mysql-instance" mysql_instance_public_ip = "" @@ -178,7 +178,7 @@ redis_instance_port = 6379 ### 6. Load custom data -Upload custom sample data to the GCS bucket specified by the terraform output `dc_gcs_data_bucket_path` (`gs://-datacommons-data-`). +Upload custom sample data to the GCS bucket specified by the terraform output `gcs_data_bucket_name` (`gs://-datacommons-data-`). From the `website` repository's root directory, run: ``` diff --git a/deploy/terraform-custom-datacommons/modules/locals.tf b/deploy/terraform-custom-datacommons/modules/locals.tf index 459ea3f0d4..ea528e2a00 100644 --- a/deploy/terraform-custom-datacommons/modules/locals.tf +++ b/deploy/terraform-custom-datacommons/modules/locals.tf @@ -16,7 +16,7 @@ locals { # Data Commons Data Bucket - dc_gcs_data_bucket_path = var.dc_gcs_data_bucket_path_override != "" ? var.dc_gcs_data_bucket_path_override : "${var.namespace}-datacommons-data-${var.project_id}" + gcs_data_bucket_name = var.gcs_data_bucket_name != "" ? var.gcs_data_bucket_name : "${var.namespace}-datacommons-data-${var.project_id}" # VPC Connector CIDR block vpc_connector_cidr = cidrsubnet(var.vpc_base_cidr_block, 4, 0) # Generates the first /28 subnet from the /24 block @@ -56,7 +56,7 @@ locals { }, { name = "OUTPUT_DIR" - value = "gs://${local.dc_gcs_data_bucket_path}/output" + value = "gs://${local.gcs_data_bucket_name}/${var.gcs_data_bucket_output_folder}" }, { name = "FORCE_RESTART" diff --git a/deploy/terraform-custom-datacommons/modules/main.tf b/deploy/terraform-custom-datacommons/modules/main.tf index 61cf0f43fc..e07d2e6dc7 100644 --- a/deploy/terraform-custom-datacommons/modules/main.tf +++ b/deploy/terraform-custom-datacommons/modules/main.tf @@ -107,12 +107,26 @@ resource "google_sql_user" "mysql_user" { } # Data commons storage bucket -resource "google_storage_bucket" "dc_gcs_data_bucket" { - name = local.dc_gcs_data_bucket_path - location = var.dc_gcs_data_bucket_location +resource "google_storage_bucket" "gcs_data_bucket" { + name = local.gcs_data_bucket_name + location = var.gcs_data_bucket_location uniform_bucket_level_access = true } +# Input 'folder' for the data loading job. +resource "google_storage_bucket_object" "gcs_data_bucket_input_folder" { + name = "${var.gcs_data_bucket_input_folder}/" + content = "Input folder" + bucket = "${google_storage_bucket.gcs_data_bucket.name}" +} + +# Output 'folder' for the data loading job. +resource "google_storage_bucket_object" "gcs_data_bucket_output_folder" { + name = "${var.gcs_data_bucket_output_folder}/" + content = "Output folder" + bucket = "${google_storage_bucket.gcs_data_bucket.name}" +} + # Generate a random suffix to append to api keys. # A deleted API key fully expires 30 days after deletion, and in the 30-day # window the ID remains taken. This suffix allows terraform to give API @@ -360,7 +374,7 @@ resource "google_cloud_run_v2_job" "dc_data_job" { env { name = "INPUT_DIR" - value = "gs://${local.dc_gcs_data_bucket_path}/input" + value = "gs://${local.gcs_data_bucket_name}/${var.gcs_data_bucket_input_folder}" } } execution_environment = "EXECUTION_ENVIRONMENT_GEN2" diff --git a/deploy/terraform-custom-datacommons/modules/outputs.tf b/deploy/terraform-custom-datacommons/modules/outputs.tf index 2f1d6f5af8..ccc3ebd54d 100644 --- a/deploy/terraform-custom-datacommons/modules/outputs.tf +++ b/deploy/terraform-custom-datacommons/modules/outputs.tf @@ -44,8 +44,8 @@ output "mysql_user_password" { sensitive = true } -output "dc_gcs_data_bucket_path" { - value = local.dc_gcs_data_bucket_path +output "gcs_data_bucket_name" { + value = local.gcs_data_bucket_name } output "cloud_run_service_name" { diff --git a/deploy/terraform-custom-datacommons/modules/variables.tf b/deploy/terraform-custom-datacommons/modules/variables.tf index ee35a035b6..3d157578cf 100644 --- a/deploy/terraform-custom-datacommons/modules/variables.tf +++ b/deploy/terraform-custom-datacommons/modules/variables.tf @@ -67,13 +67,25 @@ variable "google_analytics_tag_id" { # Data Commons Cloud Storage bucket variables # If not set, the default is -datacommons-data- -variable "dc_gcs_data_bucket_path_override" { - description = "Custom GCS data bucket path." +variable "gcs_data_bucket_name" { + description = "Custom GCS data bucket name." type = string default = "" } -variable "dc_gcs_data_bucket_location" { +variable "gcs_data_bucket_input_folder" { + description = "Input data folder in the GCS data bucket" + type = string + default = "input" +} + +variable "gcs_data_bucket_output_folder" { + description = "Output data folder in the GCS data bucket" + type = string + default = "output" +} + +variable "gcs_data_bucket_location" { description = "Data Commons GCS data bucket location" type = string default = "US"