Feature: parse tree evaulator and listener (#1169)

* ADD: ParseTreeCoercionService and tests

Signed-off-by: Chen <[email protected]>

* ENH: add rule index in operators

Signed-off-by: Chen <[email protected]>

* ADD: OperatorProvider to encapsulate operator retrieval

Signed-off-by: Chen <[email protected]>

* MNT: reuse nodeStringValue

Signed-off-by: Chen <[email protected]>

* MAINT: address all PR comments

Signed-off-by: Chen <[email protected]>

* MAINT: address PR comments

Signed-off-by: Chen <[email protected]>

* MAINT: test case name

Signed-off-by: Chen <[email protected]>

* MAINT: use more generic interface

Signed-off-by: Chen <[email protected]>

* TST: parameterized tests on escape json pointer

Signed-off-by: Chen <[email protected]>

* MAINT: fix test cases

Signed-off-by: Chen <[email protected]>

* MNT: separate variable

Signed-off-by: Chen <[email protected]>

* ADD: ParseTreeEvaluator and ParseTreeEvaluatorListener

Signed-off-by: Chen <[email protected]>

* MAINT: javadoc

Signed-off-by: Chen <[email protected]>

* TST: visit error node

Signed-off-by: Chen <[email protected]>

* FIX: more conflicts

Signed-off-by: Chen <[email protected]>

* STY: unused imports

Signed-off-by: Chen <[email protected]>

* ENH: expression coercion exception runtime

Signed-off-by: Chen <[email protected]>

* MAINT: address PR comments

Signed-off-by: Chen <[email protected]>

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/AndOperator.java

@@ -18,6 +18,11 @@ class AndOperator implements Operator<Boolean> {
     private static final String DISPLAY_NAME = DataPrepperExpressionParser.VOCABULARY
             .getDisplayName(DataPrepperExpressionParser.AND);
 
+    @Override
+    public int getNumberOfOperands() {
+        return 2;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_conditionalExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ExpressionCoercionException.java

@@ -9,7 +9,7 @@
  * @since 1.3
  * Exception thrown by {@link ParseTreeCoercionService} methods to indicate type coercion failure.
  */
-public class ExpressionCoercionException extends Exception {
+public class ExpressionCoercionException extends RuntimeException {
     public ExpressionCoercionException(final String message) {
         super(message);
     }

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenericEqualOperator.java

@@ -23,6 +23,11 @@ public GenericEqualOperator(final int symbol, BiPredicate<Object, Object> operat
         this.operation = operation;
     }
 
+    @Override
+    public int getNumberOfOperands() {
+        return 2;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_equalityOperatorExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenericInSetOperator.java

@@ -24,6 +24,11 @@ public GenericInSetOperator(final int symbol, BiPredicate<Object, Object> operat
         this.operation = operation;
     }
 
+    @Override
+    public int getNumberOfOperands() {
+        return 2;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_setOperatorExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenericRegexMatchOperator.java

@@ -24,6 +24,11 @@ public GenericRegexMatchOperator(final int symbol, BiPredicate<Object, Object> o
         this.operation = operation;
     }
 
+    @Override
+    public int getNumberOfOperands() {
+        return 2;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_regexOperatorExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/NotOperator.java

@@ -18,6 +18,11 @@ class NotOperator implements Operator<Boolean> {
     private static final String DISPLAY_NAME = DataPrepperExpressionParser.VOCABULARY
             .getDisplayName(DataPrepperExpressionParser.NOT);
 
+    @Override
+    public int getNumberOfOperands() {
+        return 1;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_unaryOperatorExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/NumericCompareOperator.java

@@ -26,6 +26,11 @@ public NumericCompareOperator(final int symbol,
     }
 
 
+    @Override
+    public int getNumberOfOperands() {
+        return 2;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_relationalOperatorExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/Operator.java

@@ -8,6 +8,8 @@
 import org.antlr.v4.runtime.RuleContext;
 
 interface Operator<T> {
+    int getNumberOfOperands();
+
     boolean shouldEvaluate(final RuleContext ctx);
 
     int getSymbol();

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/OperatorProvider.java

@@ -35,4 +35,8 @@ public Operator<?> getOperator(final int symbol) {
         }
         return operator;
     }
+
+    public boolean containsOperator(final int symbol) {
+        return symbolToOperators.containsKey(symbol);
+    }
 }

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/OrOperator.java

@@ -18,6 +18,11 @@ class OrOperator implements Operator<Boolean> {
     private static final String DISPLAY_NAME = DataPrepperExpressionParser.VOCABULARY
             .getDisplayName(DataPrepperExpressionParser.OR);
 
+    @Override
+    public int getNumberOfOperands() {
+        return 2;
+    }
+
     @Override
     public boolean shouldEvaluate(final RuleContext ctx) {
         return ctx.getRuleIndex() == DataPrepperExpressionParser.RULE_conditionalExpression;

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java

@@ -13,7 +13,7 @@
 
 @Named
 class ParseTreeCoercionService {
-    public Object coercePrimaryTerminalNode(final TerminalNode node, final Event event) throws ExpressionCoercionException {
+    public Object coercePrimaryTerminalNode(final TerminalNode node, final Event event) {
         final int nodeType = node.getSymbol().getType();
         final String nodeStringValue = node.getText();
         switch (nodeType) {

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeEvaluator.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.expression;
+
+import com.amazon.dataprepper.model.event.Event;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+@Named
+class ParseTreeEvaluator implements Evaluator<ParseTree, Event> {
+    private static final Logger LOG = LoggerFactory.getLogger(ParseTreeEvaluator.class);
+
+    private final OperatorProvider operatorProvider;
+    private final ParseTreeWalker walker;
+    private final ParseTreeCoercionService coercionService;
+
+    @Inject
+    public ParseTreeEvaluator(final OperatorProvider operatorProvider, final ParseTreeWalker walker,
+                              final ParseTreeCoercionService coercionService) {
+        this.operatorProvider = operatorProvider;
+        this.walker = walker;
+        this.coercionService = coercionService;
+    }
+
+    @Override
+    public Boolean evaluate(ParseTree parseTree, Event event) {
+        try {
+            final ParseTreeEvaluatorListener listener = new ParseTreeEvaluatorListener(operatorProvider, coercionService, event);
+            walker.walk(listener, parseTree);
+            return coercionService.coerce(listener.getResult(), Boolean.class);
+        } catch (final Exception e) {
+            LOG.error("Unable to evaluate event", e);
+            throw new ExpressionEvaluationException(e.getMessage(), e);
+        }
+    }
+}

data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeEvaluatorListener.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.expression;
+
+import com.amazon.dataprepper.model.event.Event;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.tree.ErrorNode;
+import org.antlr.v4.runtime.tree.TerminalNode;
+import org.opensearch.dataprepper.expression.antlr.DataPrepperExpressionBaseListener;
+import org.opensearch.dataprepper.expression.antlr.DataPrepperExpressionListener;
+import org.opensearch.dataprepper.expression.antlr.DataPrepperExpressionParser;
+
+import java.util.Stack;
+
+/**
+ * @since 1.3
+ * This listener implements {@link DataPrepperExpressionListener} to provide callbacks to handle evaluation of
+ * {@link org.antlr.v4.runtime.tree.ParseTree} representation of an expression while {@link org.antlr.v4.runtime.tree.ParseTreeWalker}
+ * traverses through the {@link org.antlr.v4.runtime.tree.ParseTree}.
+ *
+ * Use case:
+ * ParseTreeWalker walker = new ParseTreeWalker();
+ * ParseTreeEvaluatorListener listener = new ParseTreeEvaluatorListener(...);
+ * walker.walk(listener, ...);
+ * final Object result = listener.getResult();
+ */
+class ParseTreeEvaluatorListener extends DataPrepperExpressionBaseListener {
+
+    private final OperatorProvider operatorProvider;
+    private final ParseTreeCoercionService coercionService;
+    private final Stack<Integer> operatorSymbolStack;
+    private final Stack<Object> operandStack;
+    private final Event event;
+
+    public ParseTreeEvaluatorListener(final OperatorProvider operatorProvider,
+                                      final ParseTreeCoercionService coercionService,
+                                      final Event event) {
+        this.coercionService = coercionService;
+        this.operatorProvider = operatorProvider;
+        this.event = event;
+        operatorSymbolStack = new Stack<>();
+        operandStack = new Stack<>();
+    }
+
+    public Object getResult() {
+        if (operandStack.size() != 1) {
+            throw new IllegalStateException("The ParseTreeEvaluatorListener has not been walked through exactly once by " +
+                    "a ParseTreeWalker.");
+        }
+        return operandStack.peek();
+    }
+
+    @Override
+    public void visitTerminal(TerminalNode node) {
+        final int nodeType = node.getSymbol().getType();
+        if (nodeType == DataPrepperExpressionParser.EOF) {
+            return;
+        }
+        if (operatorProvider.containsOperator(nodeType) || nodeType == DataPrepperExpressionParser.LPAREN) {
+            operatorSymbolStack.push(nodeType);
+        } else if (nodeType == DataPrepperExpressionParser.RPAREN) {
+            // pop LPAREN at operatorSymbolStack top
+            operatorSymbolStack.pop();
+        } else {
+            final Object arg = coercionService.coercePrimaryTerminalNode(node, event);
+            operandStack.push(arg);
+        }
+    }
+
+    @Override
+    public void visitErrorNode(ErrorNode node) {
+        throw new RuntimeException("Hit error node in the parse tree: " + node.getText());
+    }
+
+    @Override
+    public void exitEveryRule(ParserRuleContext ctx) {
+        if (!operatorSymbolStack.isEmpty()) {
+            final int operatorSymbol = operatorSymbolStack.peek();
+            if (operatorSymbol != DataPrepperExpressionParser.LPAREN) {
+                final Operator<?> op = operatorProvider.getOperator(operatorSymbol);
+                if (op.shouldEvaluate(ctx)) {
+                    operatorSymbolStack.pop();
+                    try {
+                        performSingleOperation(op);
+                    } catch (final Exception e) {
+                        throw new ExpressionEvaluationException("Unable to evaluate the part of input statement: "
+                                + getPartialStatementFromContext(ctx), e);
+                    }
+                }
+            }
+        }
+    }
+
+    private void performSingleOperation(final Operator<?> operator) {
+        final int numOfArgs = operator.getNumberOfOperands();
+        final Object[] args = new Object[numOfArgs];
+        for (int i = numOfArgs - 1; i >= 0; i--) {
+            args[i] = operandStack.pop();
+        }
+        final Object result = operator.evaluate(args);
+        operandStack.push(result);
+    }
+
+    private String getPartialStatementFromContext(final ParserRuleContext ctx) {
+        final Token startToken = ctx.getStart();
+        final Token stopToken = ctx.getStop();
+        final String fullStatement = startToken.getInputStream().toString();
+        return fullStatement.substring(startToken.getStartIndex(), stopToken.getStopIndex() + 1);
+    }
+}

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/AndOperatorTest.java

@@ -24,6 +24,11 @@ class AndOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_conditionalExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/EqualOperatorTest.java

@@ -25,6 +25,11 @@ class EqualOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_equalityOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GreaterThanOperatorTest.java

@@ -24,6 +24,11 @@ class GreaterThanOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_relationalOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GreaterThanOrEqualOperatorTest.java

@@ -24,6 +24,11 @@ class GreaterThanOrEqualOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_relationalOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/InSetOperatorTest.java

@@ -27,6 +27,11 @@ class InSetOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_setOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/LessThanOperatorTest.java

@@ -24,6 +24,11 @@ class LessThanOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_relationalOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/LessThanOrEqualOperatorTest.java

@@ -24,6 +24,11 @@ class LessThanOrEqualOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_relationalOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/NotEqualOperatorTest.java

@@ -25,6 +25,11 @@ class NotEqualOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_equalityOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/NotInSetOperatorTest.java

@@ -27,6 +27,11 @@ class NotInSetOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(2));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_setOperatorExpression);

data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/NotOperatorTest.java

@@ -24,6 +24,11 @@ class NotOperatorTest {
     @Mock
     private ParserRuleContext ctx;
 
+    @Test
+    void testGetNumberOfOperands() {
+        assertThat(objectUnderTest.getNumberOfOperands(), is(1));
+    }
+
     @Test
     void testShouldEvaluate() {
         when(ctx.getRuleIndex()).thenReturn(DataPrepperExpressionParser.RULE_unaryOperatorExpression);