diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index c6c312d6b6eea..980c432774f6e 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -198,6 +198,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { EngineConfig.INDEX_CODEC_SETTING, EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING, EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, + EngineConfig.INDEX_USE_COMPOUND_FILE, IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS, IndexSettings.DEFAULT_PIPELINE, IndexSettings.FINAL_PIPELINE, diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 8106b65bddeec..670127b187b84 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -159,6 +159,8 @@ public Supplier retentionLeasesSupplier() { } }, Property.IndexScope, Property.NodeScope); + + /** * Index setting to change the compression level of zstd and zstd_no_dict lucene codecs. * Compression Level gives a trade-off between compression ratio and speed. The higher compression level results in higher compression ratio but slower compression and decompression speeds. @@ -236,6 +238,13 @@ private static void doValidateCodecSettings(final String codec) { Property.Dynamic ); + public static final Setting INDEX_USE_COMPOUND_FILE = Setting.boolSetting( + "index.use_compound_file", + true, + Property.IndexScope, + Property.Dynamic + ); + private final TranslogConfig translogConfig; private final TranslogFactory translogFactory; @@ -494,6 +503,10 @@ public boolean isReadOnlyReplica() { return indexSettings.isSegRepEnabledOrRemoteNode() && isReadOnlyReplica; } + public boolean isUseCompoundFile() { + return indexSettings.getValue(INDEX_USE_COMPOUND_FILE); + } + /** * Returns the underlying startedPrimarySupplier. * @return the primary mode supplier. diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 7bacec22fc850..feed307a1120b 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -2341,7 +2341,7 @@ private IndexWriterConfig getIndexWriterConfig() { iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); - iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh + iwc.setUseCompoundFile(engineConfig.isUseCompoundFile()); if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort()); } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java index 9197b959a54f1..9a88c584ef9eb 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java @@ -32,6 +32,13 @@ public void setUp() throws Exception { defaultIndexSettings = IndexSettingsModule.newIndexSettings("test", defaultIndexMetadata.getSettings()); } + public void testEngineConfig_DefaultValueFoUseCompoundFile() { + EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings) + .retentionLeasesSupplier(() -> RetentionLeases.EMPTY) + .build(); + assertTrue(config.isUseCompoundFile()); + } + public void testEngineConfig_DefaultValueForReadOnlyEngine() { EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings) .retentionLeasesSupplier(() -> RetentionLeases.EMPTY) diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 74aef54987842..6a9f4623665c0 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -342,6 +342,61 @@ public void testVerboseSegments() throws Exception { } } + public void testSegmentsWithUseCompoundFileFlag_true() throws IOException { + try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) { + ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); + engine.index(index); + engine.flush(); + final List segments = engine.segments(false); + assertThat(segments.size(), equalTo(1)); + assertTrue(segments.get(0).compound); + boolean cfeCompoundFileFound = false; + boolean cfsCompoundFileFound = false; + for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) { + if (fileName.endsWith(".cfe")) { + cfeCompoundFileFound = true; + } + if (fileName.endsWith(".cfs")) { + cfsCompoundFileFound = true; + } + } + Assert.assertTrue(cfeCompoundFileFound); + Assert.assertTrue(cfsCompoundFileFound); + } + } + + public void testSegmentsWithUseCompoundFileFlag_false() throws IOException { + final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + "test", + Settings.builder() + .put(defaultSettings.getSettings()) + .put(EngineConfig.INDEX_USE_COMPOUND_FILE.getKey(), false) + .build() + ); + try (Store store = createStore(); Engine engine = createEngine(indexSettings, store, createTempDir(), new TieredMergePolicy())) { + ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null); + Engine.Index index = indexForDoc(doc); + engine.index(index); + engine.flush(); + final List segments = engine.segments(false); + assertThat(segments.size(), equalTo(1)); + assertFalse(segments.get(0).compound); + boolean cfeCompoundFileFound = false; + boolean cfsCompoundFileFound = false; + for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) { + if (fileName.endsWith(".cfe")) { + cfeCompoundFileFound = true; + } + if (fileName.endsWith(".cfs")) { + cfsCompoundFileFound = true; + } + } + Assert.assertFalse(cfeCompoundFileFound); + Assert.assertFalse(cfsCompoundFileFound); + } + } + public void testSegmentsWithMergeFlag() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) { ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);