Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(python): release GIL for common functions #2269

Closed
wants to merge 5 commits into from
Closed

feat(python): release GIL for common functions #2269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4fe0419
add more non gil blocking code
franz101 Mar 9, 2024
409d28d
formatting
franz101 Mar 9, 2024
a1b9195
fix formatting
franz101 Mar 9, 2024
70a8e9c
fix mut py obj
franz101 Mar 11, 2024
97967cb
don't modify self insinde the allow threads
franz101 Mar 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions python/deltalake/table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ def __init__(

if compression_level is not None and compression is None:
raise ValueError(
"""Providing a compression level without the compression type is not possible,
"""Providing a compression level without the compression type is not possible,
please provide the compression as well."""
)
if isinstance(compression, str):
Expand Down Expand Up @@ -1299,9 +1299,9 @@ def __init__(
self.not_matched_insert_updates: Optional[List[Dict[str, str]]] = None
self.not_matched_insert_predicate: Optional[List[Optional[str]]] = None
self.not_matched_by_source_update_updates: Optional[List[Dict[str, str]]] = None
self.not_matched_by_source_update_predicate: Optional[List[Optional[str]]] = (
None
)
self.not_matched_by_source_update_predicate: Optional[
List[Optional[str]]
] = None
self.not_matched_by_source_delete_predicate: Optional[List[str]] = None
self.not_matched_by_source_delete_all: Optional[bool] = None

Expand Down Expand Up @@ -1819,7 +1819,7 @@ def add_constraint(
"""
if len(constraints.keys()) > 1:
raise ValueError(
"""add_constraints is limited to a single constraint addition at once for now.
"""add_constraints is limited to a single constraint addition at once for now.
Please execute add_constraints multiple times with each time a different constraint."""
)

Expand Down
23 changes: 13 additions & 10 deletions python/deltalake/writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,8 @@ def write_deltalake(
large_dtypes: bool = ...,
engine: Literal["pyarrow"] = ...,
custom_metadata: Optional[Dict[str, str]] = ...,
) -> None: ...
) -> None:
...


@overload
Expand Down Expand Up @@ -137,7 +138,8 @@ def write_deltalake(
engine: Literal["rust"],
writer_properties: WriterProperties = ...,
custom_metadata: Optional[Dict[str, str]] = ...,
) -> None: ...
) -> None:
...


@overload
Expand Down Expand Up @@ -166,7 +168,8 @@ def write_deltalake(
engine: Literal["rust"],
writer_properties: WriterProperties = ...,
custom_metadata: Optional[Dict[str, str]] = ...,
) -> None: ...
) -> None:
...


def write_deltalake(
Expand Down Expand Up @@ -436,7 +439,7 @@ def visitor(written_file: Any) -> None:
raise DeltaProtocolError(
"This table's min_writer_version is "
f"{table_protocol.min_writer_version}, "
f"""but this method only supports version 2 or 7 with at max these features {SUPPORTED_WRITER_FEATURES} enabled.
f"""but this method only supports version 2 or 7 with at max these features {SUPPORTED_WRITER_FEATURES} enabled.
Try engine='rust' instead which supports more features and writer versions."""
)
if (
Expand All @@ -459,12 +462,12 @@ def check_data_is_aligned_with_partition_filtering(
) -> None:
if table is None:
return
existed_partitions: FrozenSet[FrozenSet[Tuple[str, Optional[str]]]] = (
table._table.get_active_partitions()
)
allowed_partitions: FrozenSet[FrozenSet[Tuple[str, Optional[str]]]] = (
table._table.get_active_partitions(partition_filters)
)
existed_partitions: FrozenSet[
FrozenSet[Tuple[str, Optional[str]]]
] = table._table.get_active_partitions()
allowed_partitions: FrozenSet[
FrozenSet[Tuple[str, Optional[str]]]
] = table._table.get_active_partitions(partition_filters)
partition_values = pa.RecordBatch.from_arrays(
[
batch.column(column_name)
Expand Down
6 changes: 3 additions & 3 deletions python/docs/source/_ext/edit_on_github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ def html_page_context(app, pagename, templatename, context, doctree):
context["display_github"] = True
context["github_user"] = app.config.edit_on_github_project.split("/")[0]
context["github_repo"] = app.config.edit_on_github_project.split("/")[1]
context["github_version"] = (
f"{app.config.edit_on_github_branch}/{app.config.page_source_prefix}/"
)
context[
"github_version"
] = f"{app.config.edit_on_github_branch}/{app.config.page_source_prefix}/"


def setup(app):
Expand Down
Loading
Loading