From 96609c8bbe86f3edff4e0ee7a827ac7953edd9de Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Mon, 14 Oct 2024 23:00:05 +0200 Subject: [PATCH 1/6] Use tofu binary instead of terraform one --- src/_nebari/constants.py | 5 +- src/_nebari/provider/terraform.py | 85 +++++++++++++++---------------- src/_nebari/stages/base.py | 2 +- 3 files changed, 44 insertions(+), 48 deletions(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 6e57519fee..a29a88f890 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -2,10 +2,7 @@ HELM_VERSION = "v3.15.3" KUSTOMIZE_VERSION = "5.4.3" -# NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes -# implemented in August 2023. -# https://www.hashicorp.com/license-faq -TERRAFORM_VERSION = "1.5.7" +OPENTOFU_VERSION = "1.8.3" KUBERHEALTHY_HELM_VERSION = "100" diff --git a/src/_nebari/provider/terraform.py b/src/_nebari/provider/terraform.py index 59d88e76dd..da7a5e0531 100644 --- a/src/_nebari/provider/terraform.py +++ b/src/_nebari/provider/terraform.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -class TerraformException(Exception): +class OpenTofuException(Exception): pass @@ -76,7 +76,7 @@ def deploy( return output(directory) -def download_terraform_binary(version=constants.TERRAFORM_VERSION): +def download_opentofu_binary(version=constants.OPENTOFU_VERSION): os_mapping = { "linux": "linux", "win32": "windows", @@ -94,73 +94,72 @@ def download_terraform_binary(version=constants.TERRAFORM_VERSION): "arm64": "arm64", } - download_url = f"https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{os_mapping[sys.platform]}_{architecture_mapping[platform.machine()]}.zip" - filename_directory = Path(tempfile.gettempdir()) / "terraform" / version - filename_path = filename_directory / "terraform" + download_url = f"https://github.com/opentofu/opentofu/releases/download/v{version}/tofu_{version}_{os_mapping[sys.platform]}_{architecture_mapping[platform.machine()]}.zip" + + filename_directory = Path(tempfile.gettempdir()) / "opentofu" / version + filename_path = filename_directory / "tofu" if not filename_path.is_file(): logger.info( - f"downloading and extracting terraform binary from url={download_url} to path={filename_path}" + f"downloading and extracting opentofu binary from url={download_url} to path={filename_path}" ) with urllib.request.urlopen(download_url) as f: bytes_io = io.BytesIO(f.read()) download_file = zipfile.ZipFile(bytes_io) - download_file.extract("terraform", filename_directory) + download_file.extract("tofu", filename_directory) filename_path.chmod(0o555) return filename_path -def run_terraform_subprocess(processargs, **kwargs): - terraform_path = download_terraform_binary() - logger.info(f" terraform at {terraform_path}") - exit_code, output = run_subprocess_cmd([terraform_path] + processargs, **kwargs) +def run_tofu_subprocess(processargs, **kwargs): + tofu_path = download_opentofu_binary() + logger.info(f" tofu at {tofu_path}") + exit_code, output = run_subprocess_cmd([tofu_path] + processargs, **kwargs) if exit_code != 0: - raise TerraformException("Terraform returned an error") + raise OpenTofuException("OpenTofu returned an error") return output def version(): - terraform_path = download_terraform_binary() - logger.info(f"checking terraform={terraform_path} version") + tofu_path = download_opentofu_binary() + logger.info(f"checking opentofu={tofu_path} version") - version_output = subprocess.check_output([terraform_path, "--version"]).decode( - "utf-8" - ) + version_output = subprocess.check_output([tofu_path, "--version"]).decode("utf-8") return re.search(r"(\d+)\.(\d+).(\d+)", version_output).group(0) def init(directory=None, upgrade=True): - logger.info(f"terraform init directory={directory}") - with timer(logger, "terraform init"): + logger.info(f"tofu init directory={directory}") + with timer(logger, "tofu init"): command = ["init"] if upgrade: command.append("-upgrade") - run_terraform_subprocess(command, cwd=directory, prefix="terraform") + run_tofu_subprocess(command, cwd=directory, prefix="tofu") def apply(directory=None, targets=None, var_files=None): targets = targets or [] var_files = var_files or [] - logger.info(f"terraform apply directory={directory} targets={targets}") + logger.info(f"tofu apply directory={directory} targets={targets}") command = ( ["apply", "-auto-approve"] + ["-target=" + _ for _ in targets] + ["-var-file=" + _ for _ in var_files] ) - with timer(logger, "terraform apply"): - run_terraform_subprocess(command, cwd=directory, prefix="terraform") + with timer(logger, "tofu apply"): + run_tofu_subprocess(command, cwd=directory, prefix="tofu") def output(directory=None): - terraform_path = download_terraform_binary() + tofu_path = download_opentofu_binary() - logger.info(f"terraform={terraform_path} output directory={directory}") - with timer(logger, "terraform output"): + logger.info(f"tofu={tofu_path} output directory={directory}") + with timer(logger, "tofu output"): return json.loads( subprocess.check_output( - [terraform_path, "output", "-json"], cwd=directory + [tofu_path, "output", "-json"], cwd=directory ).decode("utf8")[:-1] ) @@ -168,19 +167,19 @@ def output(directory=None): def tfimport(addr, id, directory=None, var_files=None, exist_ok=False): var_files = var_files or [] - logger.info(f"terraform import directory={directory} addr={addr} id={id}") + logger.info(f"tofu import directory={directory} addr={addr} id={id}") command = ["import"] + ["-var-file=" + _ for _ in var_files] + [addr, id] logger.error(str(command)) - with timer(logger, "terraform import"): + with timer(logger, "tofu import"): try: - run_terraform_subprocess( + run_tofu_subprocess( command, cwd=directory, - prefix="terraform", + prefix="tofu", strip_errors=True, timeout=30, ) - except TerraformException as e: + except OpenTofuException as e: if not exist_ok: raise e @@ -190,39 +189,39 @@ def show(directory=None, terraform_init: bool = True) -> dict: if terraform_init: init(directory) - logger.info(f"terraform show directory={directory}") + logger.info(f"tofu show directory={directory}") command = ["show", "-json"] - with timer(logger, "terraform show"): + with timer(logger, "tofu show"): try: output = json.loads( - run_terraform_subprocess( + run_tofu_subprocess( command, cwd=directory, - prefix="terraform", + prefix="tofu", strip_errors=True, capture_output=True, ) ) return output - except TerraformException as e: + except OpenTofuException as e: raise e def refresh(directory=None, var_files=None): var_files = var_files or [] - logger.info(f"terraform refresh directory={directory}") + logger.info(f"tofu refresh directory={directory}") command = ["refresh"] + ["-var-file=" + _ for _ in var_files] - with timer(logger, "terraform refresh"): - run_terraform_subprocess(command, cwd=directory, prefix="terraform") + with timer(logger, "tofu refresh"): + run_tofu_subprocess(command, cwd=directory, prefix="tofu") def destroy(directory=None, targets=None, var_files=None): targets = targets or [] var_files = var_files or [] - logger.info(f"terraform destroy directory={directory} targets={targets}") + logger.info(f"tofu destroy directory={directory} targets={targets}") command = ( [ "destroy", @@ -232,8 +231,8 @@ def destroy(directory=None, targets=None, var_files=None): + ["-var-file=" + _ for _ in var_files] ) - with timer(logger, "terraform destroy"): - run_terraform_subprocess(command, cwd=directory, prefix="terraform") + with timer(logger, "tofu destroy"): + run_tofu_subprocess(command, cwd=directory, prefix="tofu") def rm_local_state(directory=None): diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index cef1322e95..58b860b1e5 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -338,7 +338,7 @@ def destroy( terraform_destroy=True, ) status["stages/" + self.name] = True - except terraform.TerraformException as e: + except terraform.OpenTofuException as e: if not ignore_errors: raise e status["stages/" + self.name] = False From d1dfabe2a005f61d07322e83cac3e297cd298835 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Tue, 15 Oct 2024 11:53:49 +0200 Subject: [PATCH 2/6] Remove terraform open source license test --- tests/tests_unit/test_dependencies.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tests/tests_unit/test_dependencies.py diff --git a/tests/tests_unit/test_dependencies.py b/tests/tests_unit/test_dependencies.py deleted file mode 100644 index bcde584e08..0000000000 --- a/tests/tests_unit/test_dependencies.py +++ /dev/null @@ -1,18 +0,0 @@ -import urllib - -from _nebari.provider import terraform - - -def test_terraform_open_source_license(): - tf_version = terraform.version() - license_url = ( - f"https://raw.githubusercontent.com/hashicorp/terraform/v{tf_version}/LICENSE" - ) - - request = urllib.request.Request(license_url) - with urllib.request.urlopen(request) as response: - assert 200 == response.getcode() - - license = str(response.read()) - assert "Mozilla Public License" in license - assert "Business Source License" not in license From 80d67b9657a893a634b1a1163c48a271578a51d8 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Wed, 16 Oct 2024 23:36:01 +0200 Subject: [PATCH 3/6] Rename terraform.py file and terraform_init calls --- .../provider/{terraform.py => opentofu.py} | 34 +++++++++---------- src/_nebari/stages/base.py | 32 ++++++++--------- src/_nebari/stages/infrastructure/__init__.py | 8 ++--- .../stages/terraform_state/__init__.py | 14 ++++---- src/_nebari/stages/tf_objects.py | 2 +- 5 files changed, 43 insertions(+), 47 deletions(-) rename src/_nebari/provider/{terraform.py => opentofu.py} (91%) diff --git a/src/_nebari/provider/terraform.py b/src/_nebari/provider/opentofu.py similarity index 91% rename from src/_nebari/provider/terraform.py rename to src/_nebari/provider/opentofu.py index da7a5e0531..78936d1808 100644 --- a/src/_nebari/provider/terraform.py +++ b/src/_nebari/provider/opentofu.py @@ -24,33 +24,33 @@ class OpenTofuException(Exception): def deploy( directory, - terraform_init: bool = True, - terraform_import: bool = False, - terraform_apply: bool = True, - terraform_destroy: bool = False, + tofu_init: bool = True, + tofu_import: bool = False, + tofu_apply: bool = True, + tofu_destroy: bool = False, input_vars: Dict[str, Any] = {}, state_imports: List[Any] = [], ): - """Execute a given terraform directory. + """Execute a given directory with OpenTofu infrastructure configuration. Parameters: - directory: directory in which to run terraform operations on + directory: directory in which to run tofu operations on - terraform_init: whether to run `terraform init` default True + tofu_init: whether to run `tofu init` default True - terraform_import: whether to run `terraform import` default + tofu_import: whether to run `tofu import` default False for each `state_imports` supplied to function - terraform_apply: whether to run `terraform apply` default True + tofu_apply: whether to run `tofu apply` default True - terraform_destroy: whether to run `terraform destroy` default + tofu_destroy: whether to run `tofu destroy` default False input_vars: supply values for "variable" resources within terraform module state_imports: (addr, id) pairs for iterate through and attempt - to terraform import + to tofu import """ with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", suffix=".tfvars.json" @@ -58,19 +58,19 @@ def deploy( json.dump(input_vars, f.file) f.file.flush() - if terraform_init: + if tofu_init: init(directory) - if terraform_import: + if tofu_import: for addr, id in state_imports: tfimport( addr, id, directory=directory, var_files=[f.name], exist_ok=True ) - if terraform_apply: + if tofu_apply: apply(directory, var_files=[f.name]) - if terraform_destroy: + if tofu_destroy: destroy(directory, var_files=[f.name]) return output(directory) @@ -184,9 +184,9 @@ def tfimport(addr, id, directory=None, var_files=None, exist_ok=False): raise e -def show(directory=None, terraform_init: bool = True) -> dict: +def show(directory=None, tofu_init: bool = True) -> dict: - if terraform_init: + if tofu_init: init(directory) logger.info(f"tofu show directory={directory}") diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 58b860b1e5..42b9dda65d 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -11,7 +11,7 @@ from kubernetes import client, config from kubernetes.client.rest import ApiException -from _nebari.provider import helm, kubernetes, kustomize, terraform +from _nebari.provider import helm, kubernetes, kustomize, opentofu from _nebari.stages.tf_objects import NebariTerraformState from nebari.hookspecs import NebariStage @@ -248,7 +248,7 @@ def tf_objects(self) -> List[Dict]: def render(self) -> Dict[pathlib.Path, str]: contents = { - (self.stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects( + (self.stage_prefix / "_nebari.tf.json"): opentofu.tf_render_objects( self.tf_objects() ) } @@ -283,19 +283,19 @@ def deploy( self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False, - terraform_init: bool = True, + tofu_init: bool = True, ): deploy_config = dict( directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), - terraform_init=terraform_init, + tofu_init=tofu_init, ) state_imports = self.state_imports() if state_imports: deploy_config["terraform_import"] = True deploy_config["state_imports"] = state_imports - self.set_outputs(stage_outputs, terraform.deploy(**deploy_config)) + self.set_outputs(stage_outputs, opentofu.deploy(**deploy_config)) self.post_deploy(stage_outputs, disable_prompt) yield @@ -318,27 +318,27 @@ def destroy( ): self.set_outputs( stage_outputs, - terraform.deploy( + opentofu.deploy( directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), - terraform_init=True, - terraform_import=True, - terraform_apply=False, - terraform_destroy=False, + tofu_init=True, + tofu_import=True, + tofu_apply=False, + tofu_destroy=False, ), ) yield try: - terraform.deploy( + opentofu.deploy( directory=str(self.output_directory / self.stage_prefix), input_vars=self.input_vars(stage_outputs), - terraform_init=True, - terraform_import=True, - terraform_apply=False, - terraform_destroy=True, + tofu_init=True, + tofu_import=True, + tofu_apply=False, + tofu_destroy=True, ) status["stages/" + self.name] = True - except terraform.OpenTofuException as e: + except opentofu.OpenTofuException as e: if not ignore_errors: raise e status["stages/" + self.name] = False diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index 026f33fe82..fa47f90106 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -11,7 +11,7 @@ from pydantic import Field, field_validator, model_validator from _nebari import constants -from _nebari.provider import terraform +from _nebari.provider import opentofu from _nebari.provider.cloud import ( amazon_web_services, azure_cloud, @@ -735,7 +735,7 @@ def state_imports(self) -> List[Tuple[str, str]]: def tf_objects(self) -> List[Dict]: if self.config.provider == schema.ProviderEnum.gcp: return [ - terraform.Provider( + opentofu.Provider( "google", project=self.config.google_cloud_platform.project, region=self.config.google_cloud_platform.region, @@ -752,9 +752,7 @@ def tf_objects(self) -> List[Dict]: ] elif self.config.provider == schema.ProviderEnum.aws: return [ - terraform.Provider( - "aws", region=self.config.amazon_web_services.region - ), + opentofu.Provider("aws", region=self.config.amazon_web_services.region), NebariTerraformState(self.name, self.config), ] else: diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index 37568be130..becdbc084c 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -9,7 +9,7 @@ from pydantic import field_validator from _nebari import utils -from _nebari.provider import terraform +from _nebari.provider import opentofu from _nebari.provider.cloud import azure_cloud from _nebari.stages.base import NebariTerraformStage from _nebari.stages.tf_objects import NebariConfig @@ -175,7 +175,7 @@ def tf_objects(self) -> List[Dict]: resources = [NebariConfig(self.config)] if self.config.provider == schema.ProviderEnum.gcp: return resources + [ - terraform.Provider( + opentofu.Provider( "google", project=self.config.google_cloud_platform.project, region=self.config.google_cloud_platform.region, @@ -183,9 +183,7 @@ def tf_objects(self) -> List[Dict]: ] elif self.config.provider == schema.ProviderEnum.aws: return resources + [ - terraform.Provider( - "aws", region=self.config.amazon_web_services.region - ), + opentofu.Provider("aws", region=self.config.amazon_web_services.region), ] else: return resources @@ -236,9 +234,9 @@ def deploy( ): self.check_immutable_fields() - # No need to run terraform init here as it's being called when running the + # No need to run tofu init here as it's being called when running the # terraform show command, inside check_immutable_fields - with super().deploy(stage_outputs, disable_prompt, terraform_init=False): + with super().deploy(stage_outputs, disable_prompt, tofu_init=False): env_mapping = {} # DigitalOcean terraform remote state using Spaces Bucket # assumes aws credentials thus we set them to match spaces credentials @@ -286,7 +284,7 @@ def check_immutable_fields(self): def get_nebari_config_state(self): directory = str(self.output_directory / self.stage_prefix) - tf_state = terraform.show(directory) + tf_state = opentofu.show(directory) nebari_config_state = None # get nebari config from state diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py index 04c6d434aa..39fa136e68 100644 --- a/src/_nebari/stages/tf_objects.py +++ b/src/_nebari/stages/tf_objects.py @@ -1,4 +1,4 @@ -from _nebari.provider.terraform import Data, Provider, Resource, TerraformBackend +from _nebari.provider.opentofu import Data, Provider, Resource, TerraformBackend from _nebari.utils import ( AZURE_TF_STATE_RESOURCE_GROUP_SUFFIX, construct_azure_resource_group_name, From 8f709dacce5d22774b6895a8207e9d81b8251102 Mon Sep 17 00:00:00 2001 From: Marcelo Villa Date: Thu, 17 Oct 2024 08:03:09 +0200 Subject: [PATCH 4/6] Fix wrong function reference --- src/_nebari/stages/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py index 42b9dda65d..bcc6bb82bf 100644 --- a/src/_nebari/stages/base.py +++ b/src/_nebari/stages/base.py @@ -292,7 +292,7 @@ def deploy( ) state_imports = self.state_imports() if state_imports: - deploy_config["terraform_import"] = True + deploy_config["tofu_import"] = True deploy_config["state_imports"] = state_imports self.set_outputs(stage_outputs, opentofu.deploy(**deploy_config)) From 415b136524f185ccbe86bfc34e5fb7397a90eeda Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:44:29 +0100 Subject: [PATCH 5/6] specify terraform registry for providers not in opentofu registry --- src/_nebari/stages/infrastructure/template/local/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf index fb0d0997e1..a157dc4f3a 100644 --- a/src/_nebari/stages/infrastructure/template/local/main.tf +++ b/src/_nebari/stages/infrastructure/template/local/main.tf @@ -1,15 +1,15 @@ terraform { required_providers { kind = { - source = "tehcyx/kind" + source = "registry.terraform.io/tehcyx/kind" version = "0.4.0" } docker = { - source = "kreuzwerker/docker" + source = "registry.terraform.io/kreuzwerker/docker" version = "2.16.0" } kubectl = { - source = "gavinbunney/kubectl" + source = "registry.terraform.io/gavinbunney/kubectl" version = ">= 1.7.0" } } From e7ff0896cf7ff70b5bce6fab66e57468a9cf9c72 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Thu, 14 Nov 2024 17:56:35 +0100 Subject: [PATCH 6/6] only update kind --- src/_nebari/stages/infrastructure/template/local/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf index a157dc4f3a..77aa799cbd 100644 --- a/src/_nebari/stages/infrastructure/template/local/main.tf +++ b/src/_nebari/stages/infrastructure/template/local/main.tf @@ -5,11 +5,11 @@ terraform { version = "0.4.0" } docker = { - source = "registry.terraform.io/kreuzwerker/docker" + source = "kreuzwerker/docker" version = "2.16.0" } kubectl = { - source = "registry.terraform.io/gavinbunney/kubectl" + source = "gavinbunney/kubectl" version = ">= 1.7.0" } }