Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AMORO-3272] Complete data expiration based on partition information #3273

Merged
merged 19 commits into from
Feb 26, 2025
Merged

[AMORO-3272] Complete data expiration based on partition information #3273

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
bc12dbc
feature: data-expire by partition info
lintingbin Oct 17, 2024
f992434
Merge branch 'master' into feature/data_expire_by_partition_info
zhoujinsong Oct 21, 2024
6c8d772
Merge branch 'master' into feature/data_expire_by_partition_info
zhoujinsong Oct 30, 2024
e3b2e0f
Merge branch 'master' into feature/data_expire_by_partition_info
klion26 Dec 3, 2024
54c3700
add comments
lintingbin Dec 3, 2024
86f6b00
Merge branch 'master' into feature/data_expire_by_partition_info
klion26 Dec 4, 2024
aba77d8
Refactor the code to improve readability.
lintingbin Dec 6, 2024
49ecbdc
Merge branch 'master' into feature/data_expire_by_partition_info
XBaith Dec 6, 2024
e20e266
Merge branch 'master' into feature/data_expire_by_partition_info
baiyangtx Dec 10, 2024
c1ebdd2
Refactor the code to improve readability.
lintingbin Dec 30, 2024
6f537bd
Merge branch 'master' into feature/data_expire_by_partition_info
zhoujinsong Jan 14, 2025
5c32a8c
Merge branch 'master' into feature/data_expire_by_partition_info
zhoujinsong Feb 10, 2025
9376ad1
Merge branch 'master' into feature/data_expire_by_partition_info
XBaith Feb 19, 2025
bea5ae1
feature: add test cases
lintingbin Feb 21, 2025
d48a8d9
fix test error
lintingbin Feb 21, 2025
9cc3446
optimize code
lintingbin Feb 25, 2025
6ff7532
Merge branch 'master' into feature/data_expire_by_partition_info
XBaith Feb 25, 2025
7aa078b
update
XBaith Feb 26, 2025
896a80e
checkstyle
XBaith Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ...s/src/main/java/org/apache/amoro/server/optimizing/maintainer/IcebergTableMaintainer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -965,6 +965,9 @@ private Literal<Long> getExpireTimestampLiteral(
Object upperBound =
Conversions.fromByteBuffer(type, contentFile.upperBounds().get(field.fieldId()));
Literal<Long> literal = Literal.of(Long.MAX_VALUE);
if ("none".equals(table.properties().get("write.metadata.metrics.default"))) {
upperBound = null;
}
if (null == upperBound) {
if (canBeExpireByPartition(contentFile, field, expireValue)) {
literal = Literal.of(0L);
Expand Down
26 changes: 22 additions & 4 deletions amoro-ams/src/test/java/org/apache/amoro/server/optimizing/maintainer/TestDataExpire.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,9 @@ public static Object[] parameters() {
public static final PartitionSpec SPEC2 =
PartitionSpec.builderFor(TABLE_SCHEMA2).identity("op_time").build();

public static final String WRITE_METADATA_METRICS_DEFAULT_KEY = "write.metadata.metrics.default";
public static final String WRITE_METADATA_METRICS_NONE_VALUE = "none";

public TestDataExpire(CatalogTestHelper catalogTestHelper, TableTestHelper tableTestHelper) {
super(catalogTestHelper, tableTestHelper);
}
Expand Down Expand Up @@ -195,7 +198,7 @@ private void testUnKeyedPartitionLevel() {
List<Record> expected;
if (tableTestHelper().partitionSpec().isPartitioned()) {
// retention time is 1 day, expire partitions that order than 2022-01-02
if (expireByStringDate()) {
if (expireByStringDate() && isMetricsNotNone()) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should keep the same result even if metrics mode is none

expected =
Lists.newArrayList(
createRecord(2, "222", parseMillis("2022-01-03T12:00:00"), "2022-01-03T12:00:00"));
Expand Down Expand Up @@ -252,7 +255,7 @@ private void testKeyedPartitionLevel() {
CloseableIterable<TableFileScanHelper.FileScanResult> scanAfterExpire =
buildKeyedFileScanHelper().scan();
if (tableTestHelper().partitionSpec().isPartitioned()) {
if (expireByStringDate()) {
if (expireByStringDate() && isMetricsNotNone()) {
assertScanResult(scanAfterExpire, 1, 0);
} else {
assertScanResult(scanAfterExpire, 3, 0);
Expand All @@ -264,7 +267,7 @@ private void testKeyedPartitionLevel() {
List<Record> records = readSortedKeyedRecords(keyedTable);
List<Record> expected;
if (tableTestHelper().partitionSpec().isPartitioned()) {
if (expireByStringDate()) {
if (expireByStringDate() && isMetricsNotNone()) {
expected =
Lists.newArrayList(
createRecord(2, "222", parseMillis("2022-01-03T12:00:00"), "2022-01-03T12:00:00"));
Expand Down Expand Up @@ -465,6 +468,16 @@ public void testNormalFieldFileLevel() {
testFileLevel();
}

@Test
public void testExpireByPartition() {
getMixedTable()
.updateProperties()
.set(WRITE_METADATA_METRICS_DEFAULT_KEY, WRITE_METADATA_METRICS_NONE_VALUE)
.commit();

testPartitionLevel();
}

@Test
public void testGcDisabled() {
MixedTable testTable = getMixedTable();
Expand Down Expand Up @@ -592,7 +605,6 @@ protected static Map<String, String> getDefaultProp() {
prop.put(TableProperties.ENABLE_DATA_EXPIRATION, "true");
prop.put(TableProperties.DATA_EXPIRATION_FIELD, "op_time");
prop.put(TableProperties.DATA_EXPIRATION_RETENTION_TIME, "1d");
prop.put("write.metadata.metrics.default", "none");
return prop;
}

Expand All @@ -616,6 +628,12 @@ private boolean expireByStringDate() {
.equals(Type.TypeID.STRING);
}

private boolean isMetricsNotNone() {
return !CompatiblePropertyUtil.propertyAsString(
getMixedTable().properties(), WRITE_METADATA_METRICS_DEFAULT_KEY, "")
.equals(WRITE_METADATA_METRICS_NONE_VALUE);
}

private static DataExpirationConfig parseDataExpirationConfig(MixedTable table) {
Map<String, String> properties = table.properties();
return TableConfigurations.parseDataExpirationConfig(properties);
Expand Down
Loading