Integrated logstash converter into Data Prepper core (#591)

Integrated logstash converter into Data Prepper core Signed-off-by: Asif Sohail Mohammed <[email protected]> Co-authored-by: Asif Sohail Mohammed <[email protected]>
opensearch-project · Nov 15, 2021 · f677480 · f677480
1 parent 368795f
commit f677480
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 8 deletions.
diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle
@@ -21,6 +21,7 @@ dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins')
     implementation project(':data-prepper-plugins:common')
+    implementation project(':data-prepper-logstash-configuration')
     testImplementation project(':data-prepper-plugins:common').sourceSets.test.output
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'

diff --git a/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java b/data-prepper-core/src/main/java/com/amazon/dataprepper/DataPrepperExecute.java
@@ -11,9 +11,14 @@
 
 package com.amazon.dataprepper;
 
+import org.opensearch.dataprepper.logstash.LogstashConfigConverter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
 /**
  * Execute entry into Data Prepper.
  */
@@ -30,10 +35,26 @@ public static void main(String[] args) {
         }
         final DataPrepper dataPrepper = DataPrepper.getInstance();
         if (args.length > 0) {
-            dataPrepper.execute(args[0]);
+            String configurationFileLocation = checkForLogstashConfigurationAndConvert(args[0]);
+            dataPrepper.execute(configurationFileLocation);
         } else {
             LOG.error("Configuration file is required");
             System.exit(1);
         }
     }
+
+    private static String checkForLogstashConfigurationAndConvert(String configurationFileLocation) {
+        if (configurationFileLocation.endsWith(".conf")) {
+            final LogstashConfigConverter logstashConfigConverter = new LogstashConfigConverter();
+            final Path configurationDirectory = Paths.get(configurationFileLocation).toAbsolutePath().getParent();
+
+            try {
+                configurationFileLocation = logstashConfigConverter.convertLogstashConfigurationToPipeline(
+                        configurationFileLocation, String.valueOf(configurationDirectory));
+            } catch (IOException e) {
+                LOG.error("Unable to read the Logstash configuration file", e);
+            }
+        }
+        return configurationFileLocation;
+    }
 }
diff --git a/...figuration/src/main/java/org/opensearch/dataprepper/logstash/LogstashConfigConverter.java b/...figuration/src/main/java/org/opensearch/dataprepper/logstash/LogstashConfigConverter.java
@@ -9,6 +9,7 @@
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.opensearch.dataprepper.logstash.mapping.LogstashMapper;
 import org.opensearch.dataprepper.logstash.model.LogstashConfiguration;
+import org.opensearch.dataprepper.logstash.parser.ModelConvertingLogstashVisitor;
 
 import java.io.File;
 import java.io.IOException;
@@ -33,7 +34,7 @@ public String convertLogstashConfigurationToPipeline(String logstashConfiguratio
         LogstashParser parser = new LogstashParser(tokens);
         final ParseTree tree = parser.config();
 
-        org.opensearch.dataprepper.logstash.parser.LogstashVisitor visitor = new org.opensearch.dataprepper.logstash.parser.LogstashVisitor();
+        ModelConvertingLogstashVisitor visitor = new ModelConvertingLogstashVisitor();
         LogstashConfiguration logstashConfiguration = (LogstashConfiguration) visitor.visit(tree);
 
         LogstashMapper logstashMapper = new LogstashMapper();
@@ -48,7 +49,7 @@ public String convertLogstashConfigurationToPipeline(String logstashConfiguratio
                 .build());
 
         final String confFileName = configurationFilePath.getFileName().toString();
-        final String yamlFileName = confFileName.substring(0, confFileName.lastIndexOf("."));
+        final String yamlFileName = confFileName.substring(0, confFileName.lastIndexOf(".conf"));
 
         final Path yamlFilePath = Paths.get(outputDirectory , yamlFileName + ".yaml");
 

diff --git a/...pper/logstash/parser/LogstashVisitor.java → ...arser/ModelConvertingLogstashVisitor.java b/...pper/logstash/parser/LogstashVisitor.java → ...arser/ModelConvertingLogstashVisitor.java
@@ -24,15 +24,15 @@
  * @since 1.2
  */
 @SuppressWarnings("rawtypes")
-public class LogstashVisitor extends LogstashBaseVisitor {
+public class ModelConvertingLogstashVisitor extends LogstashBaseVisitor {
 
     @Override
     public Object visitConfig(final LogstashParser.ConfigContext configContext) {
         final Map<LogstashPluginType, List<LogstashPlugin>> pluginSections = new LinkedHashMap<>();
 
         configContext.plugin_section().forEach(pluginSection -> {
             final String pluginType = pluginSection.plugin_type().getText();
-            if (!Arrays.asList(new String[]{"input", "filter", "output"}).contains(pluginType))
+            if (!Arrays.asList("input", "filter", "output").contains(pluginType))
                 throw new LogstashParsingException("only input, filter and output plugin sections are supported.");
             final LogstashPluginType logstashPluginType = LogstashPluginType.getByValue(pluginType);
             final List<LogstashPlugin> logstashPluginList = (List<LogstashPlugin>) visitPlugin_section(pluginSection);

diff --git a/.../logstash/parser/LogstashVisitorTest.java → ...r/ModelConvertingLogstashVisitorTest.java b/.../logstash/parser/LogstashVisitorTest.java → ...r/ModelConvertingLogstashVisitorTest.java
@@ -27,13 +27,13 @@
 import java.util.List;
 import java.util.LinkedList;
 
-class LogstashVisitorTest {
+class ModelConvertingLogstashVisitorTest {
 
-    private static LogstashVisitor logstashVisitor;
+    private static ModelConvertingLogstashVisitor logstashVisitor;
 
     @BeforeEach
     void createObjectUnderTest() {
-        logstashVisitor = spy(new LogstashVisitor());
+        logstashVisitor = spy(new ModelConvertingLogstashVisitor());
     }
 
     @Mock