Draft of the extended vectorscan API

This commit extends the vectorscan API with access to low level algorithms.
This would enable a develloper to bypass most of the overhead of the regular
vectorscan scan when the pattern to check is simple. Currently it only
targets pure literal patterns.

Signed-off-by: Yoan Picchi <[email protected]>

CMakeLists.txt

@@ -291,6 +291,7 @@ set (hs_exec_SRCS
     src/crc32.h
     src/report.h
     src/runtime.c
+    src/hs_extended_api.cpp
     src/stream_compress.c
     src/stream_compress.h
     src/stream_compress_impl.h

benchmarks/CMakeLists.txt

@@ -6,4 +6,9 @@ if (NOT FAT_RUNTIME AND (BUILD_SHARED_LIBS OR BUILD_STATIC_LIBS))
   set_source_files_properties(benchmarks.cpp PROPERTIES COMPILE_FLAGS
       "-Wall -Wno-unused-variable")
   target_link_libraries(benchmarks hs)
+
+  add_executable(test_api test_extended_api.cpp)
+  set_source_files_properties(test_extended_api.cpp PROPERTIES COMPILE_FLAGS
+      "-Wall -Wno-unused-variable")
+  target_link_libraries(test_api hs)
 endif()

benchmarks/test_extended_api.cpp

@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2024-2025, Arm ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *  * Neither the name of Intel Corporation nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+
+#include "hs_compile.h"
+#include "hs_runtime.h"
+
+#include "hwlm/hwlm_literal.h"
+#include "hwlm/noodle_build.h"
+#include "hwlm/noodle_engine.h"
+#include "hwlm/noodle_internal.h"
+
+const char *buf1 = "azertyuioperty";
+int buf1_len = 14;
+const char *buf2 = "AZERTYUIOPERTY";
+int buf2_len = 14;
+
+typedef struct context {
+    /* array of indices in the string where we expect match to be reported */
+    size_t *expected_array;
+    size_t array_size;
+    /* counter of matches hapenning at a position in expected_array */
+    size_t number_matched;
+    /* counter of matches hapenning at a position NOT in expected_array */
+    size_t number_wrong;
+} context_t;
+
+int callback(unsigned int id, unsigned long long start,
+             unsigned long long end_offset, unsigned int flags,
+             void *raw_context) {
+    (void)id;
+    (void)start;
+    (void)flags;
+    context_t *context = (context_t *)raw_context;
+    bool matched = false;
+    // Check if the match is expected
+    for (size_t i = 0; i < context->array_size; i++) {
+        if (end_offset == context->expected_array[i]) {
+            matched = true;
+        }
+    }
+    // Tally the right counter wether the match was expected or not
+    if (matched) {
+        context->number_matched += 1;
+    } else {
+        context->number_wrong += 1;
+        printf("unplanned match at index %llu\n", end_offset);
+    }
+
+    return CB_CONTINUE_MATCHING;
+}
+
+int test_noodle() {
+    const char *pattern = "ert";
+    hs_short_literal_compiled_pattern_t noodle_database;
+
+    hs_error_t ret =
+        hs_compile_short_literal_search(pattern, 3, &noodle_database);
+    if (ret != HS_SUCCESS) {
+        printf("Fail to build the pattern\n");
+        return 1;
+    }
+
+    size_t expected_array[2] = {4, 12};
+    context_t context = {&(expected_array[0]), 2, 0, 0};
+    ret = hs_short_literal_search(&noodle_database, buf1, buf1_len, nullptr,
+                                  callback, &context);
+    if (ret != HS_SUCCESS) {
+        printf("Fail to run noodle\n");
+        return 1;
+    }
+    if (context.number_matched != context.array_size) {
+        printf("1- missed some matches. Expected: %lu, got %lu\n",
+               context.array_size, context.number_matched);
+    }
+
+    expected_array[0] = 8;
+    context = {&(expected_array[0]), 1, 0, 0};
+    ret = hs_short_literal_search(&noodle_database, buf1 + 4, buf1_len - 4,
+                                  nullptr, callback, &context);
+    if (ret != HS_SUCCESS) {
+        printf("Fail to run noodle\n");
+        return 1;
+    }
+    if (context.number_matched != context.array_size) {
+        printf("2- missed some matches. Expected: %lu, got %lu\n",
+               context.array_size, context.number_matched);
+    }
+
+    pattern = "ERT";
+    ret = hs_compile_short_literal_search(pattern, 3, &noodle_database);
+    if (ret != HS_SUCCESS) {
+        printf("Fail to build the pattern\n");
+        return 1;
+    }
+
+    expected_array[0] = 4;
+    context = {&(expected_array[0]), 2, 0, 0};
+    ret = hs_short_literal_search(&noodle_database, buf2, buf2_len, nullptr,
+                                  callback, &context);
+    if (ret != HS_SUCCESS) {
+        printf("Fail to run noodle\n");
+        return 1;
+    }
+    if (context.number_matched != context.array_size) {
+        printf("3- missed some matches. Expected: %lu, got %lu\n",
+               context.array_size, context.number_matched);
+    }
+
+    expected_array[0] = 8;
+    context = {&(expected_array[0]), 1, 0, 0};
+    ret = hs_short_literal_search(&noodle_database, buf2 + 4, buf2_len - 4,
+                                  nullptr, callback, &context);
+    if (ret != HS_SUCCESS) {
+        printf("Fail to run noodle\n");
+        return 1;
+    }
+    if (context.number_matched != context.array_size) {
+        printf("4- missed some matches. Expected: %lu, got %lu\n",
+               context.array_size, context.number_matched);
+    }
+
+    return 0;
+}
+
+int main() {
+    // test_plain_noodle();
+    if (!test_noodle()) {
+        printf("all test passed\n");
+    }
+
+    return 0;
+}

src/hs_common.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2015-2019, Intel Corporation
+ * Copyright (c) 2024-2025, Arm ltd
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -583,6 +584,155 @@ hs_error_t HS_CDECL hs_valid_platform(void);
  */
 #define HS_UNKNOWN_ERROR   (-13)
 
+/**
+ * The following functions and types are part of the extended API and are not
+ * cross compatible with hyperscan. This extension intends on providing the
+ * developer with minimal overhead search functions.
+ */
+
+/**
+ * The size threshold after which a pattern is considered long and must be fed
+ * to @ref hs_compile_long_literal_search(). Patterns up to this length may be
+ * fed to hs_compile_short_literal_search() instead.
+ */
+#define HS_SHORT_PATTERN_THRESHOLD 16
+
+/**
+ * The compiled pattern type for searches short literals
+ * Generated by @ref hs_compile_short_literal_search()
+ */
+typedef struct {
+    unsigned char data[32];
+} hs_short_literal_compiled_pattern_t;
+
+/**
+ * The compiled pattern type for searching long literals
+ * Generated by @ref hs_compile_long_literal_search()
+ * Note that given the unbounded nature of the pattern size, it is impossible to
+ * give a constant size to the structure. You'll need to use @ref
+ * hs_get_long_literal_search_database_size() and allocate enough memory to
+ * store the compiled pattern.
+ */
+typedef struct hs_long_literal_compiled_pattern hs_long_literal_compiled_pattern_t;
+
+/**
+ * RFC: hs_compile_long_literal_search() and a few other don't have any obvious
+ * bounds on the size of the pattern. To make it generic we need to get the time
+ * at runtime. The issue is that it is slow. I'd like to also provide an
+ * optional static size in some way when the backing algorithm allows it. It
+ * might be in the form of a polynomial function in a #define that takes in the
+ * expected max size of the pattern and the number of patterns. This would allow
+ * the user to use a static size in their own algo if they know in advance what
+ * kind of pattern they'll use. The problem with this though is that I'm not
+ * sure how often such feature would be used and I don't want to clutter the
+ * API. Any feedback?
+ */
+
+/**
+ * This function calculate the size needed to store the compiled version of the
+ * given @p expression .
+ *
+ * @param expression
+ *      The expression to parse. Note that this string must represent ONLY the
+ * pattern to be matched, with no delimiters. Null characters are accepted as
+ * part of the expression.
+ *
+ * @param expression_length
+ *      The length of the expression in byte.
+ *
+ * @return
+ *      On success, the size in byte that needs to be allocated to store the
+ * given pattern. Otherwise returns 0
+ */
+size_t HS_CDECL hs_get_long_literal_search_database_size(const char *expression,
+                                                         size_t pattern_len);
+
+/**
+ * The compiled pattern type for searching several long literal
+ * Generated by @ref hs_compile_multi_literal_search()
+ * Note that given the unbounded nature of the pattern size, it is impossible to
+ * give a constant size to the structure. You'll need to use @ref
+ * hs_get_multi_literal_search_database_size() and allocate enough memory to
+ * store the compiled pattern.
+ */
+typedef struct hs_multi_literal_compiled_pattern hs_multi_literal_compiled_pattern_t;
+
+/**
+ * This function calculate the size needed to store the compiled version of the
+ * given @p expression .
+ *
+ * @param expression
+ *      The array of expressions to parse. Note that the strings must represent
+ * ONLY the patterns to be matched, with no delimiters. Null characters are
+ * accepted as part of the expression.
+ *
+ * @param pattern_count
+ *      The number of expressions in the @p expression array.
+ *
+ * @param expression_length
+ *      The array of length of each expression in the @p expression array.
+ * Expressed in byte.
+ *
+ * @return
+ *      On success, the size in byte that needs to be allocated to store the
+ * given pattern. Otherwise returns 0
+ */
+size_t HS_CDECL hs_get_multi_literal_search_database_size(
+    const char **expression, size_t pattern_count, size_t *pattern_len);
+
+/**
+ * The compiled pattern type for searching a single character
+ * Generated by @ref hs_compile_single_char_search()
+ */
+typedef struct {
+    unsigned char data[1];
+} hs_single_char_compiled_pattern_t;
+
+/**
+ * The compiled pattern type for searching a character set
+ * Generated by @ref hs_compile_multi_char_search()
+ */
+typedef struct {
+    unsigned char data[32];
+} hs_multi_char_compiled_pattern_t;
+
+/**
+ * The compiled pattern type for searching a character pair
+ * Generated by @ref hs_compile_char_pair_search()
+ */
+typedef struct {
+    unsigned char data[32];
+} hs_single_char_pair_compiled_pattern_t;
+
+/**
+ * The compiled pattern type for searching a set of character pairs
+ * Generated by @ref hs_compile_multi_char_pair_search()
+ * Note that given the unbounded maximum number of pair, it is impossible to
+ * give a constant size to the structure. You'll need to use @ref
+ * hs_get_multi_char_pair_search_database_size() and allocate enough memory to
+ * store the compiled pattern.
+ */
+typedef struct hs_multi_char_pair_compiled_pattern hs_multi_char_pair_compiled_pattern_t;
+
+/**
+ * This function calculate the size needed to store the compiled version of the
+ * given @p expression .
+ *
+ * @param expression
+ *      The concatenation of all pairs to be parsed. If one want to search for
+ * "ab" or "Cd", then @p expression would be ['a','b','C','d']. Null terminator
+ * is optional.
+ *
+ * @param pair_count
+ *      The number of characters pair in @p expression
+ *
+ * @return
+ *      On success, the size in byte that needs to be allocated to store the
+ * given pattern. Otherwise returns 0
+ */
+size_t HS_CDECL hs_get_multi_char_pair_search_database_size(
+    const char *expression, size_t pair_count);
+
 /** @} */
 
 #ifdef __cplusplus