From 442df412ce2ef0734a4a1163aaee6eef44b5694d Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Tue, 18 Feb 2025 14:46:34 +0000 Subject: [PATCH 01/10] Adjust schema name generation --- .../dremio/macros/get_custom_name/get_custom_schema.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index 97159b66..50bc853d 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -17,9 +17,15 @@ limitations under the License.*/ else target.root_path -%} {%- set custom_schema_name = custom_schema_name if not is_datalake_node(node) else node.config.root_path -%} + {%- set custom_schema_name = append_schema(custom_schema_name or default_schema, node.config.schema) + if node.config.materialized == 'table' else custom_schema_name -%} {{ generate_schema_name_impl(default_schema, custom_schema_name, node) }} {%- endmacro %} +{% macro append_schema(base_path, subfolder) -%} + {{ base_path if subfolder is none else subfolder if base_path in [none, 'no_schema'] else base_path ~ '.' ~ subfolder }} +{%- endmacro %} + {% macro generate_schema_name_impl(default_schema, custom_schema_name=none, node=none) -%} {%- if custom_schema_name is none -%} From 69fa2f51262af68a0e429ad53b4e087bbfbdbbf9 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Tue, 18 Feb 2025 19:03:30 +0000 Subject: [PATCH 02/10] Add test cases --- .../get_custom_name/get_custom_schema.sql | 4 +- .../dremio_specific/test_nested_schema.py | 44 +++++++++++++++++++ tests/utils/util.py | 39 ++++++++-------- 3 files changed, 66 insertions(+), 21 deletions(-) create mode 100644 tests/functional/adapter/dremio_specific/test_nested_schema.py diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index 50bc853d..9a1e9095 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -17,8 +17,8 @@ limitations under the License.*/ else target.root_path -%} {%- set custom_schema_name = custom_schema_name if not is_datalake_node(node) else node.config.root_path -%} - {%- set custom_schema_name = append_schema(custom_schema_name or default_schema, node.config.schema) - if node.config.materialized == 'table' else custom_schema_name -%} + {%- set custom_schema_name = default_schema ~ "." ~ custom_schema_name + if not is_datalake_node(node) else append_schema(custom_schema_name or default_schema, node.config.schema) -%} {{ generate_schema_name_impl(default_schema, custom_schema_name, node) }} {%- endmacro %} diff --git a/tests/functional/adapter/dremio_specific/test_nested_schema.py b/tests/functional/adapter/dremio_specific/test_nested_schema.py new file mode 100644 index 00000000..2e366e1e --- /dev/null +++ b/tests/functional/adapter/dremio_specific/test_nested_schema.py @@ -0,0 +1,44 @@ +import pytest +from dbt.tests.util import run_dbt, get_connection +from tests.utils.util import relation_from_name +from tests.fixtures.profiles import unique_schema, dbt_profile_data + +custom_schema_table_model = """ +{{ config( + materialized='table', + schema='nested.schema' +) }} +select 1 as id +""" + +custom_schema_view_model = """ +{{ config( + materialized='view', + schema='nested.schema' +) }} +select 1 as id +""" + +class TestGetCustomSchema: + @pytest.fixture(scope="class") + def models(self): + return { + "custom_schema_table.sql": custom_schema_table_model, + "custom_schema_view.sql": custom_schema_view_model, + } + + def test_custom_schema_table(self, project): + run_dbt(["run", "--select", "custom_schema_table"]) + table_relation = relation_from_name(project.adapter, "nested.schema.custom_schema_table") + with get_connection(project.adapter): + columns = project.adapter.get_columns_in_relation(table_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_view(self, project): + run_dbt(["run", "--select", "custom_schema_view"]) + view_relation = relation_from_name(project.adapter, "nested.schema.custom_schema_view") + with get_connection(project.adapter): + columns = project.adapter.get_columns_in_relation(view_relation) + assert len(columns) == 1 + assert columns[0].name == "id" diff --git a/tests/utils/util.py b/tests/utils/util.py index 38c1ae4d..ed8cfa1a 100644 --- a/tests/utils/util.py +++ b/tests/utils/util.py @@ -17,9 +17,8 @@ from contextlib import contextmanager from dbt.tests.util import AnyInteger -from dbt.adapters.events.logging import AdapterLogger - -logger = AdapterLogger("dremio") +import logging +LOGGER = logging.getLogger(__name__) # Ensure we do not include dashes in our source # https://github.com/dremio/dbt-dremio/issues/68 @@ -42,29 +41,31 @@ def relation_from_name(adapter, name: str, materialization=""): quote_policy = cls.get_default_quote_policy().to_dict() include_policy = cls.get_default_include_policy().to_dict() + relation_parts = name.split(".") + # Make sure we have database/schema/identifier parts, even if # only identifier was supplied. - relation_parts = name.split(".") - if len(relation_parts) == 1: - if materialization == "view" or "view" in name: - relation_parts.insert(0, credentials.schema) - else: - relation_parts.insert(0, credentials.root_path) - if len(relation_parts) == 2: - # if the relation is a view then use database - if materialization == "view" or "view" in name: - relation_parts.insert(0, credentials.database) - relation_type = "view" - else: - relation_parts.insert(0, credentials.datalake) - relation_type = "table" + + # if the relation is a view then use database + LOGGER.info(f"Credentials: {credentials}") + if materialization == "view" or "view" in name: + relation_parts.insert(0, credentials.database) + relation_parts.insert(1, credentials.schema) + else: + relation_parts.insert(0, credentials.datalake) + relation_parts.insert(1, credentials.root_path) + + relation_type = "table" if materialization != "view" and "view" not in name else "view" + kwargs = { "database": relation_parts[0], - "schema": relation_parts[1], - "identifier": relation_parts[2], + "schema": ".".join(relation_parts[1:-1]), + "identifier": relation_parts[-1], "type": relation_type, } + LOGGER.info(f"Relation kwargs: {kwargs}") + relation = cls.create( include_policy=include_policy, quote_policy=quote_policy, From 3195fd1c7b58d561d5e7741ad9d5b54135eb6159 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Tue, 18 Feb 2025 19:15:35 +0000 Subject: [PATCH 03/10] Minor changes --- .../macros/get_custom_name/get_custom_schema.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index 9a1e9095..e616caea 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -17,13 +17,15 @@ limitations under the License.*/ else target.root_path -%} {%- set custom_schema_name = custom_schema_name if not is_datalake_node(node) else node.config.root_path -%} - {%- set custom_schema_name = default_schema ~ "." ~ custom_schema_name - if not is_datalake_node(node) else append_schema(custom_schema_name or default_schema, node.config.schema) -%} + {%- set custom_schema_name = append_schema(default_schema, custom_schema_name) if not is_datalake_node(node) + else append_schema(custom_schema_name or default_schema, node.config.schema) -%} {{ generate_schema_name_impl(default_schema, custom_schema_name, node) }} {%- endmacro %} -{% macro append_schema(base_path, subfolder) -%} - {{ base_path if subfolder is none else subfolder if base_path in [none, 'no_schema'] else base_path ~ '.' ~ subfolder }} +{% macro append_schema(base_path, custom_schema) -%} + {{ base_path if custom_schema is none + else custom_schema if base_path in [none, 'no_schema'] + else base_path ~ '.' ~ custom_schema}} {%- endmacro %} {% macro generate_schema_name_impl(default_schema, custom_schema_name=none, node=none) -%} From a5de7f842fe5b3e26508b73a926f32bb33e6a009 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Fri, 21 Feb 2025 11:11:04 +0000 Subject: [PATCH 04/10] Add additional test cases (currently failing) --- .../dremio_specific/test_nested_schema.py | 79 ++++++++++++++++++- tests/utils/util.py | 4 + 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/tests/functional/adapter/dremio_specific/test_nested_schema.py b/tests/functional/adapter/dremio_specific/test_nested_schema.py index 2e366e1e..c32b5edf 100644 --- a/tests/functional/adapter/dremio_specific/test_nested_schema.py +++ b/tests/functional/adapter/dremio_specific/test_nested_schema.py @@ -1,9 +1,24 @@ import pytest from dbt.tests.util import run_dbt, get_connection from tests.utils.util import relation_from_name -from tests.fixtures.profiles import unique_schema, dbt_profile_data + + +custom_schema_table_no_schema_model = """ +{{ config( + materialized='table' +) }} +select 1 as id +""" custom_schema_table_model = """ +{{ config( + materialized='table', + schema='schema' +) }} +select 1 as id +""" + +custom_schema_table_nested_model = """ {{ config( materialized='table', schema='nested.schema' @@ -11,7 +26,22 @@ select 1 as id """ +custom_schema_view_no_schema_model = """ +{{ config( + materialized='view' +) }} +select 1 as id +""" + custom_schema_view_model = """ +{{ config( + materialized='view', + schema='schema' +) }} +select 1 as id +""" + +custom_schema_view_nested_model = """ {{ config( materialized='view', schema='nested.schema' @@ -19,25 +49,68 @@ select 1 as id """ + class TestGetCustomSchema: @pytest.fixture(scope="class") def models(self): return { + "custom_schema_table_no_schema.sql": custom_schema_table_no_schema_model, "custom_schema_table.sql": custom_schema_table_model, + "custom_schema_table_nested.sql": custom_schema_table_nested_model, + "custom_schema_view_no_schema.sql": custom_schema_view_no_schema_model, "custom_schema_view.sql": custom_schema_view_model, + "custom_schema_view_nested.sql": custom_schema_view_nested_model, } + def test_custom_schema_table_no_schema(self, project): + run_dbt(["run", "--select", "custom_schema_table_no_schema"]) + table_relation = relation_from_name( + project.adapter, "custom_schema_table_no_schema") + with get_connection(project.adapter): + columns = project.adapter.get_columns_in_relation(table_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + def test_custom_schema_table(self, project): run_dbt(["run", "--select", "custom_schema_table"]) - table_relation = relation_from_name(project.adapter, "nested.schema.custom_schema_table") + table_relation = relation_from_name( + project.adapter, "schema.custom_schema_table") + with get_connection(project.adapter): + columns = project.adapter.get_columns_in_relation(table_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_table_nested(self, project): + run_dbt(["run", "--select", "custom_schema_table_nested"]) + table_relation = relation_from_name( + project.adapter, "nested.schema.custom_schema_table_nested") with get_connection(project.adapter): columns = project.adapter.get_columns_in_relation(table_relation) assert len(columns) == 1 assert columns[0].name == "id" + def test_custom_schema_view_no_schema(self, project): + run_dbt(["run", "--select", "custom_schema_view_no_schema"]) + view_relation = relation_from_name( + project.adapter, "custom_schema_view_no_schema") + with get_connection(project.adapter): + columns = project.adapter.get_columns_in_relation(view_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + def test_custom_schema_view(self, project): run_dbt(["run", "--select", "custom_schema_view"]) - view_relation = relation_from_name(project.adapter, "nested.schema.custom_schema_view") + view_relation = relation_from_name( + project.adapter, "schema.custom_schema_view") + with get_connection(project.adapter): + columns = project.adapter.get_columns_in_relation(view_relation) + assert len(columns) == 1 + assert columns[0].name == "id" + + def test_custom_schema_view_nested(self, project): + run_dbt(["run", "--select", "custom_schema_view_nested"]) + view_relation = relation_from_name( + project.adapter, "nested.schema.custom_schema_view_nested") with get_connection(project.adapter): columns = project.adapter.get_columns_in_relation(view_relation) assert len(columns) == 1 diff --git a/tests/utils/util.py b/tests/utils/util.py index ed8cfa1a..46d86b67 100644 --- a/tests/utils/util.py +++ b/tests/utils/util.py @@ -17,6 +17,10 @@ from contextlib import contextmanager from dbt.tests.util import AnyInteger +from dbt.adapters.events.logging import AdapterLogger + +logger = AdapterLogger("dremio") + import logging LOGGER = logging.getLogger(__name__) From 6d0306458fb64a95dda36991f16b44a0f139dd88 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Sat, 22 Feb 2025 15:16:47 +0000 Subject: [PATCH 05/10] Create space on folder creation --- dbt/adapters/dremio/connections.py | 3 +++ .../macros/get_custom_name/get_custom_schema.sql | 15 ++------------- tests/utils/util.py | 10 ++-------- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/dbt/adapters/dremio/connections.py b/dbt/adapters/dremio/connections.py index 5f590e71..0ff477e8 100644 --- a/dbt/adapters/dremio/connections.py +++ b/dbt/adapters/dremio/connections.py @@ -229,6 +229,9 @@ def create_catalog(self, relation): if database != credentials.datalake: logger.debug(f"Creating folder(s): {database}.{schema}") self._create_folders(database, schema, rest_client) + else: + logger.debug(f"Creating folder(s): {credentials.database}.{schema}") + self._create_folders(credentials.database, schema, rest_client) return # dbt docs integration with Dremio wikis and tags diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index e616caea..18d8624b 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -13,21 +13,10 @@ See the License for the specific language governing permissions and limitations under the License.*/ {% macro dremio__generate_schema_name(custom_schema_name, node) -%} - {%- set default_schema = target.schema if not is_datalake_node(node) - else target.root_path -%} - {%- set custom_schema_name = custom_schema_name if not is_datalake_node(node) - else node.config.root_path -%} - {%- set custom_schema_name = append_schema(default_schema, custom_schema_name) if not is_datalake_node(node) - else append_schema(custom_schema_name or default_schema, node.config.schema) -%} + {%- set default_schema = target.schema -%} {{ generate_schema_name_impl(default_schema, custom_schema_name, node) }} {%- endmacro %} -{% macro append_schema(base_path, custom_schema) -%} - {{ base_path if custom_schema is none - else custom_schema if base_path in [none, 'no_schema'] - else base_path ~ '.' ~ custom_schema}} -{%- endmacro %} - {% macro generate_schema_name_impl(default_schema, custom_schema_name=none, node=none) -%} {%- if custom_schema_name is none -%} @@ -35,7 +24,7 @@ limitations under the License.*/ {%- else -%} - {{ custom_schema_name }} + {{default_schema}}.{{ custom_schema_name }} {%- endif -%} {%- endmacro %} diff --git a/tests/utils/util.py b/tests/utils/util.py index 46d86b67..9c6a7b7a 100644 --- a/tests/utils/util.py +++ b/tests/utils/util.py @@ -21,9 +21,6 @@ logger = AdapterLogger("dremio") -import logging -LOGGER = logging.getLogger(__name__) - # Ensure we do not include dashes in our source # https://github.com/dremio/dbt-dremio/issues/68 BUCKET = "dbtdremios3" @@ -51,14 +48,13 @@ def relation_from_name(adapter, name: str, materialization=""): # only identifier was supplied. # if the relation is a view then use database - LOGGER.info(f"Credentials: {credentials}") if materialization == "view" or "view" in name: relation_parts.insert(0, credentials.database) - relation_parts.insert(1, credentials.schema) else: relation_parts.insert(0, credentials.datalake) - relation_parts.insert(1, credentials.root_path) + relation_parts.insert(1, credentials.schema) + relation_type = "table" if materialization != "view" and "view" not in name else "view" kwargs = { @@ -68,8 +64,6 @@ def relation_from_name(adapter, name: str, materialization=""): "type": relation_type, } - LOGGER.info(f"Relation kwargs: {kwargs}") - relation = cls.create( include_policy=include_policy, quote_policy=quote_policy, From 4777f1a00606a4febb451d82cec245e35ee8b2c2 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Sat, 22 Feb 2025 17:06:05 +0000 Subject: [PATCH 06/10] Start using root_path for tables --- dbt/adapters/dremio/connections.py | 4 ++-- .../dremio/macros/get_custom_name/get_custom_schema.sql | 3 ++- tests/utils/util.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/dremio/connections.py b/dbt/adapters/dremio/connections.py index 0ff477e8..db93aab5 100644 --- a/dbt/adapters/dremio/connections.py +++ b/dbt/adapters/dremio/connections.py @@ -227,10 +227,10 @@ def create_catalog(self, relation): self._create_space(database, rest_client) if database != credentials.datalake: - logger.debug(f"Creating folder(s): {database}.{schema}") + logger.info(f"Creating folder(s): {database}.{schema}") self._create_folders(database, schema, rest_client) else: - logger.debug(f"Creating folder(s): {credentials.database}.{schema}") + logger.info(f"Creating folder(s): {credentials.database}.{schema}") self._create_folders(credentials.database, schema, rest_client) return diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index 18d8624b..e1864fa8 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -13,7 +13,8 @@ See the License for the specific language governing permissions and limitations under the License.*/ {% macro dremio__generate_schema_name(custom_schema_name, node) -%} - {%- set default_schema = target.schema -%} + {%- set default_schema = target.schema if not is_datalake_node(node) + else target.root_path -%} {{ generate_schema_name_impl(default_schema, custom_schema_name, node) }} {%- endmacro %} diff --git a/tests/utils/util.py b/tests/utils/util.py index 9c6a7b7a..ce231d3b 100644 --- a/tests/utils/util.py +++ b/tests/utils/util.py @@ -50,10 +50,12 @@ def relation_from_name(adapter, name: str, materialization=""): # if the relation is a view then use database if materialization == "view" or "view" in name: relation_parts.insert(0, credentials.database) + relation_parts.insert(1, credentials.schema) else: relation_parts.insert(0, credentials.datalake) + if credentials.root_path in [None, "no_schema"]: + relation_parts.insert(1, credentials.root_path) - relation_parts.insert(1, credentials.schema) relation_type = "table" if materialization != "view" and "view" not in name else "view" From a405791232ed33a6a08466f40ab2a7e724d340b6 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Mon, 24 Feb 2025 10:13:13 +0000 Subject: [PATCH 07/10] Minor changes --- dbt/adapters/dremio/connections.py | 5 +---- .../macros/get_custom_name/get_custom_schema.sql | 4 ++++ .../dremio/macros/materializations/twin_strategy.sql | 3 --- .../adapter/dremio_specific/app.code-workspace | 11 +++++++++++ tests/utils/util.py | 2 +- 5 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 tests/functional/adapter/dremio_specific/app.code-workspace diff --git a/dbt/adapters/dremio/connections.py b/dbt/adapters/dremio/connections.py index db93aab5..5f590e71 100644 --- a/dbt/adapters/dremio/connections.py +++ b/dbt/adapters/dremio/connections.py @@ -227,11 +227,8 @@ def create_catalog(self, relation): self._create_space(database, rest_client) if database != credentials.datalake: - logger.info(f"Creating folder(s): {database}.{schema}") + logger.debug(f"Creating folder(s): {database}.{schema}") self._create_folders(database, schema, rest_client) - else: - logger.info(f"Creating folder(s): {credentials.database}.{schema}") - self._create_folders(credentials.database, schema, rest_client) return # dbt docs integration with Dremio wikis and tags diff --git a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql index e1864fa8..67d8ba94 100644 --- a/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql +++ b/dbt/include/dremio/macros/get_custom_name/get_custom_schema.sql @@ -23,6 +23,10 @@ limitations under the License.*/ {{ default_schema }} + {%- elif default_schema == 'no_schema' -%} + + {{ custom_schema_name}} + {%- else -%} {{default_schema}}.{{ custom_schema_name }} diff --git a/dbt/include/dremio/macros/materializations/twin_strategy.sql b/dbt/include/dremio/macros/materializations/twin_strategy.sql index e496b005..6eda3dcb 100644 --- a/dbt/include/dremio/macros/materializations/twin_strategy.sql +++ b/dbt/include/dremio/macros/materializations/twin_strategy.sql @@ -37,9 +37,6 @@ limitations under the License.*/ select * from {{ render_with_format_clause(target_relation) }} {%- endset -%} - {% call statement('clone_view') -%} - {{ create_view_as(view_relation, sql_view) }} - {%- endcall %} {%- endif -%} {%- endif -%} {%- endif -%} diff --git a/tests/functional/adapter/dremio_specific/app.code-workspace b/tests/functional/adapter/dremio_specific/app.code-workspace new file mode 100644 index 00000000..6f9ac413 --- /dev/null +++ b/tests/functional/adapter/dremio_specific/app.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "path": "../../../.." + }, + { + "path": "../../../../../../../home/vscode/.local/lib/python3.9/site-packages" + } + ], + "settings": {} +} \ No newline at end of file diff --git a/tests/utils/util.py b/tests/utils/util.py index ce231d3b..d4faf9c1 100644 --- a/tests/utils/util.py +++ b/tests/utils/util.py @@ -53,7 +53,7 @@ def relation_from_name(adapter, name: str, materialization=""): relation_parts.insert(1, credentials.schema) else: relation_parts.insert(0, credentials.datalake) - if credentials.root_path in [None, "no_schema"]: + if credentials.root_path not in [None, "no_schema"]: relation_parts.insert(1, credentials.root_path) From 2e8bf4acfbd333ab311029b42cb2270854eda23a Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Mon, 24 Feb 2025 10:13:54 +0000 Subject: [PATCH 08/10] Delete app.code-workspace --- .../adapter/dremio_specific/app.code-workspace | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 tests/functional/adapter/dremio_specific/app.code-workspace diff --git a/tests/functional/adapter/dremio_specific/app.code-workspace b/tests/functional/adapter/dremio_specific/app.code-workspace deleted file mode 100644 index 6f9ac413..00000000 --- a/tests/functional/adapter/dremio_specific/app.code-workspace +++ /dev/null @@ -1,11 +0,0 @@ -{ - "folders": [ - { - "path": "../../../.." - }, - { - "path": "../../../../../../../home/vscode/.local/lib/python3.9/site-packages" - } - ], - "settings": {} -} \ No newline at end of file From 367964a515015945963f55c3e4e4aa0a74f63a55 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Mon, 24 Feb 2025 14:58:55 +0000 Subject: [PATCH 09/10] Fix twin strategy --- .../macros/materializations/twin_strategy.sql | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/dbt/include/dremio/macros/materializations/twin_strategy.sql b/dbt/include/dremio/macros/materializations/twin_strategy.sql index 6eda3dcb..609acbe2 100644 --- a/dbt/include/dremio/macros/materializations/twin_strategy.sql +++ b/dbt/include/dremio/macros/materializations/twin_strategy.sql @@ -14,6 +14,7 @@ limitations under the License.*/ {%- macro apply_twin_strategy(target_relation) -%} {%- set twin_strategy = config.get('twin_strategy', validator=validation.any[basestring]) or 'clone' -%} + {{ log("TARGET: " ~ target_relation, True) }} {%- if target_relation.type == 'view' -%} {%- if twin_strategy != 'allow' -%} {%- set table_relation = api.Relation.create( @@ -24,12 +25,13 @@ limitations under the License.*/ {{ adapter.drop_relation(table_relation) }} {%- endif -%} {%- elif target_relation.type == 'table' -%} - {%- if twin_strategy in ['prevent', 'clone'] -%} - {%- set view_relation = api.Relation.create( - identifier=generate_alias_name_impl(model.name, config.get('alias', validator=validation.any[basestring]), model), - schema=generate_schema_name_impl(target.schema, config.get('schema', validator=validation.any[basestring]), model), - database=generate_database_name_impl(target.database, config.get('database', validator=validation.any[basestring]), model), - type='view') -%} + {%- set view_relation = api.Relation.create( + identifier=generate_alias_name_impl(model.name, config.get('alias', validator=validation.any[basestring]), model), + schema=generate_schema_name_impl(target.schema, config.get('schema', validator=validation.any[basestring]), model), + database=generate_database_name_impl(target.database, config.get('database', validator=validation.any[basestring]), model), + type='view') -%} + {%- set conflicting_view = adapter.get_relation(database=view_relation.database, schema=view_relation.schema, identifier=view_relation.identifier) -%} + {%- if conflicting_view is not none -%} {%- if twin_strategy == 'prevent' -%} {{ adapter.drop_relation(view_relation) }} {%- elif twin_strategy == 'clone' -%} @@ -37,6 +39,9 @@ limitations under the License.*/ select * from {{ render_with_format_clause(target_relation) }} {%- endset -%} + {% call statement('clone_view') -%} + {{ create_view_as(view_relation, sql_view) }} + {%- endcall %} {%- endif -%} {%- endif -%} {%- endif -%} From 7d391e4db30c74c86e3708fcb312f4505884bd93 Mon Sep 17 00:00:00 2001 From: Simon Pannek Date: Mon, 24 Feb 2025 15:02:14 +0000 Subject: [PATCH 10/10] Remove logging --- dbt/include/dremio/macros/materializations/twin_strategy.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/include/dremio/macros/materializations/twin_strategy.sql b/dbt/include/dremio/macros/materializations/twin_strategy.sql index 609acbe2..ee6474a1 100644 --- a/dbt/include/dremio/macros/materializations/twin_strategy.sql +++ b/dbt/include/dremio/macros/materializations/twin_strategy.sql @@ -14,7 +14,6 @@ limitations under the License.*/ {%- macro apply_twin_strategy(target_relation) -%} {%- set twin_strategy = config.get('twin_strategy', validator=validation.any[basestring]) or 'clone' -%} - {{ log("TARGET: " ~ target_relation, True) }} {%- if target_relation.type == 'view' -%} {%- if twin_strategy != 'allow' -%} {%- set table_relation = api.Relation.create(