refactoring CodecService for 2.x

Signed-off-by: Sarthak Aggarwal <[email protected]>
opensearch-project · Jul 9, 2023 · dbbc877 · dbbc877
1 parent e217b5d
commit dbbc877
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -94,7 +94,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Refactor] OpenSearchException and ExceptionsHelper foundation to base class ([#7508](https://github.com/opensearch-project/OpenSearch/pull/7508))
 - Move ZSTD compression codecs out of the sandbox ([#7908](https://github.com/opensearch-project/OpenSearch/pull/7908))
 - Update ZSTD default compression level ([#8471](https://github.com/opensearch-project/OpenSearch/pull/8471))
-- [Search Pipelines] Pass pipeline creation context to processor factories ([#8164](https://github.com/opensearch-project/OpenSearch/pull/8164))
 - Enabling compression levels for zstd and zstd_no_dict ([#8312](https://github.com/opensearch-project/OpenSearch/pull/8312))
 - Improved performance of parsing floating point numbers ([#8467](https://github.com/opensearch-project/OpenSearch/pull/8467))
 - Move span actions to Scope ([#8411](https://github.com/opensearch-project/OpenSearch/pull/8411))

diff --git a/...main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java b/...main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecService.java
diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java
@@ -71,23 +71,54 @@ public class CodecService {
     public CodecService(@Nullable MapperService mapperService, IndexSettings indexSettings, Logger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
-        int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
-        if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene95Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+        addIndexCodecs(codecs, mapperService, logger, indexSettings);
+        addLuceneAvailableCodecs(codecs);
+        this.codecs = codecs.immutableMap();
+    }
+
+    public CodecService(@Nullable MapperService mapperService, Logger logger) {
+        final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
+        if (mapperService != null) {
+            IndexSettings indexSettings = mapperService.getIndexSettings();
+            addIndexCodecs(codecs, mapperService, logger, indexSettings);
+        } else {
+            addIndexCodecs(codecs, null, logger, null);
+        }
+        addLuceneAvailableCodecs(codecs);
+        this.codecs = codecs.immutableMap();
+    }
+
+    private void addIndexCodecs(MapBuilder<String, Codec> codecs, MapperService mapperService, Logger logger, IndexSettings indexSettings) {
+        Codec defaultCodec = mapperService == null
+            ? new Lucene95Codec()
+            : new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger);
+        Codec bestCompressionCodec = mapperService == null
+            ? new Lucene95Codec(Mode.BEST_COMPRESSION)
+            : new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger);
+
+        codecs.put(DEFAULT_CODEC, defaultCodec);
+        codecs.put(BEST_COMPRESSION_CODEC, bestCompressionCodec);
+
+        if (indexSettings != null) {
+            int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
+            if (mapperService != null) {
+                codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
+                codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            } else {
+                codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
+                codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            }
         } else {
-            codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
-            codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
-            codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            codecs.put(ZSTD_CODEC, new ZstdCodec());
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec());
         }
+    }
+
+    private void addLuceneAvailableCodecs(MapBuilder<String, Codec> codecs) {
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
         for (String codec : Codec.availableCodecs()) {
             codecs.put(codec, Codec.forName(codec));
         }
-        this.codecs = codecs.immutableMap();
     }
 
     public Codec codec(String name) {

diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java
@@ -142,6 +142,36 @@ public void testExceptionIndexSettingsNull() {
         assertThrows(AssertionError.class, () -> new CodecService(null, null, LogManager.getLogger("test")));
     }
 
+    public void testCodecServiceWithNullMapperService() {
+        CodecService codecService = new CodecService(null, LogManager.getLogger("test"));
+        assert codecService.codec("default") instanceof Lucene95Codec;
+        assert codecService.codec("best_compression") instanceof Lucene95Codec;
+        Lucene95CustomStoredFieldsFormat zstdStoredFieldsFormat = (Lucene95CustomStoredFieldsFormat) codecService.codec("zstd")
+            .storedFieldsFormat();
+        Lucene95CustomStoredFieldsFormat zstdNoDictStoredFieldsFormat = (Lucene95CustomStoredFieldsFormat) codecService.codec("zstd")
+            .storedFieldsFormat();
+        assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, zstdStoredFieldsFormat.getCompressionLevel());
+        assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, zstdNoDictStoredFieldsFormat.getCompressionLevel());
+    }
+
+    public void testCodecServiceWithOnlyMapperService() throws IOException {
+        int randomCompressionLevel = randomIntBetween(1, 6);
+        Settings nodeSettings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put("index.codec.compression_level", randomCompressionLevel)
+            .build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
+        CodecService codecService = new CodecService(getMapperService(indexSettings, nodeSettings), LogManager.getLogger("test"));
+        assert codecService.codec("default") instanceof PerFieldMappingPostingFormatCodec;
+        assert codecService.codec("best_compression") instanceof PerFieldMappingPostingFormatCodec;
+        Lucene95CustomStoredFieldsFormat zstdStoredFieldsFormat = (Lucene95CustomStoredFieldsFormat) codecService.codec("zstd")
+            .storedFieldsFormat();
+        Lucene95CustomStoredFieldsFormat zstdNoDictStoredFieldsFormat = (Lucene95CustomStoredFieldsFormat) codecService.codec("zstd")
+            .storedFieldsFormat();
+        assertEquals(randomCompressionLevel, zstdStoredFieldsFormat.getCompressionLevel());
+        assertEquals(randomCompressionLevel, zstdNoDictStoredFieldsFormat.getCompressionLevel());
+    }
+
     // write some docs with it, inspect .si to see this was the used compression
     private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception {
         SegmentReader sr = getSegmentReader(actual);
@@ -176,10 +206,15 @@ private CodecService createCodecService(int randomCompressionLevel) throws IOExc
     private CodecService buildCodecService(Settings nodeSettings) throws IOException {
 
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
+        MapperService mapperService = getMapperService(indexSettings, nodeSettings);
+        return new CodecService(mapperService, indexSettings, LogManager.getLogger("test"));
+    }
+
+    private MapperService getMapperService(IndexSettings indexSettings, Settings nodeSettings) throws IOException {
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
         IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers;
         MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER);
-        MapperService service = new MapperService(
+        return new MapperService(
             indexSettings,
             indexAnalyzers,
             xContentRegistry(),
@@ -189,7 +224,6 @@ private CodecService buildCodecService(Settings nodeSettings) throws IOException
             () -> false,
             null
         );
-        return new CodecService(service, indexSettings, LogManager.getLogger("test"));
     }
 
     private SegmentReader getSegmentReader(Codec codec) throws IOException {