Merge remote-tracking branch 'origin/main' into tests/rescore-knn-vec…

…tor-query-test-coverage
carlosdelest · Feb 17, 2025 · be6010c · be6010c
2 parents 3513d20 + 4908c92
commit be6010c
Show file tree

Hide file tree

Showing 77 changed files with 2,058 additions and 620 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -76,7 +76,7 @@ steps:
             ES_VERSION:
               - "9.0.0"
             ES_COMMIT:
-              - "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
+              - "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump / head of combat-lucene-10-0-0
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -93,7 +93,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-security-with-mustache");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:xpack-prefix-rest-compat");
         map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-geoip:qa:file-based-update");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2:qa:amazon-ec2");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce:qa:gce");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license");

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -17,7 +17,7 @@ jna               = 5.12.1
 netty             = 4.1.115.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv1sdk            = 1.12.270
+awsv1sdk            = 1.12.746
 awsv2sdk            = 2.28.13
 reactive_streams    = 1.0.4
 

diff --git a/docs/changelog/121784.yaml b/docs/changelog/121784.yaml
@@ -0,0 +1,5 @@
+pr: 121784
+summary: Optionally allow text similarity reranking to fail
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/121971.yaml b/docs/changelog/121971.yaml
@@ -0,0 +1,5 @@
+pr: 121971
+summary: Do not fetch reserved roles from native store when Get Role API is called
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/122247.yaml b/docs/changelog/122247.yaml
@@ -0,0 +1,5 @@
+pr: 122247
+summary: Improve jwt logging on failed auth
+area: Authentication
+type: bug
+issues: []
diff --git a/docs/changelog/122431.yaml b/docs/changelog/122431.yaml
@@ -0,0 +1,5 @@
+pr: 122431
+summary: Upgrade AWS SDK to v1.12.746
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/122640.yaml b/docs/changelog/122640.yaml
@@ -0,0 +1,5 @@
+pr: 122640
+summary: Fix redact processor arraycopy bug
+area: Ingest Node
+type: bug
+issues: []
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -89,29 +89,29 @@
             <sha256 value="ccc7efe5cd3ce22d6046cafd4d2f8bff5adcb43e0d27da482178fac5daadef81" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.amazonaws" name="aws-java-sdk-core" version="1.12.270">
-         <artifact name="aws-java-sdk-core-1.12.270.jar">
-            <sha256 value="4e41d9f54606151674fc550e5e6291b0ddf917d55a2a3465a45a4e6ac98c9f8f" origin="Generated by Gradle"/>
+      <component group="com.amazonaws" name="aws-java-sdk-core" version="1.12.746">
+         <artifact name="aws-java-sdk-core-1.12.746.jar">
+            <sha256 value="798fd30dafcf6816e760ad8aef8b3f09c43351ed2e166993bddc4527dbafb0be" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.amazonaws" name="aws-java-sdk-ec2" version="1.12.270">
-         <artifact name="aws-java-sdk-ec2-1.12.270.jar">
-            <sha256 value="faadf443751822e205338e80d2cea5eabd6373c1c3cef6348c24809ca82a9dd0" origin="Generated by Gradle"/>
+      <component group="com.amazonaws" name="aws-java-sdk-ec2" version="1.12.746">
+         <artifact name="aws-java-sdk-ec2-1.12.746.jar">
+            <sha256 value="cec22d57e05ed75417b1342e9dd468c6fe7f2fab97c626c065d6495e44d732ad" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.amazonaws" name="aws-java-sdk-s3" version="1.12.270">
-         <artifact name="aws-java-sdk-s3-1.12.270.jar">
-            <sha256 value="41bbea44bac7cfce3898e2e598a17526984337e265f6b16814839c17168a570e" origin="Generated by Gradle"/>
+      <component group="com.amazonaws" name="aws-java-sdk-s3" version="1.12.746">
+         <artifact name="aws-java-sdk-s3-1.12.746.jar">
+            <sha256 value="dcd839802c71ffc4d3e6bebc8769a2149bc423baf95f3e6c8214f9c91536bc38" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.amazonaws" name="aws-java-sdk-sts" version="1.12.270">
-         <artifact name="aws-java-sdk-sts-1.12.270.jar">
-            <sha256 value="8cf2d3705381b81808c2e75a5e25a7097385b121ef15c001b18fde3d79657571" origin="Generated by Gradle"/>
+      <component group="com.amazonaws" name="aws-java-sdk-sts" version="1.12.746">
+         <artifact name="aws-java-sdk-sts-1.12.746.jar">
+            <sha256 value="2916c28f9a6b6ade40c7e2ffdea3788b198a98b2b16830e02a24ec49fc0fb06f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.amazonaws" name="jmespath-java" version="1.12.270">
-         <artifact name="jmespath-java-1.12.270.jar">
-            <sha256 value="515d1afb0cd0176630c0707acabd4a3e48424ea938b89359774f61a24b6450f1" origin="Generated by Gradle"/>
+      <component group="com.amazonaws" name="jmespath-java" version="1.12.746">
+         <artifact name="jmespath-java-1.12.746.jar">
+            <sha256 value="d4239a7a1bfacbb9cd1f0e48a46ac95960ab7942c6fbb41ea825161efea72351" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="com.avast.gradle" name="gradle-docker-compose-plugin" version="0.17.5">

diff --git a/...tlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/...tlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java
@@ -10,6 +10,8 @@
 package org.elasticsearch.entitlement.bridge;
 
 import java.io.File;
+import java.io.FileFilter;
+import java.io.FilenameFilter;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.io.PrintWriter;
@@ -58,6 +60,8 @@
 import java.nio.file.LinkOption;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchService;
 import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.UserPrincipal;
 import java.nio.file.spi.FileSystemProvider;
@@ -510,6 +514,12 @@ public interface EntitlementChecker {
     //
 
     // old io (ie File)
+    void check$java_io_File$canExecute(Class<?> callerClass, File file);
+
+    void check$java_io_File$canRead(Class<?> callerClass, File file);
+
+    void check$java_io_File$canWrite(Class<?> callerClass, File file);
+
     void check$java_io_File$createNewFile(Class<?> callerClass, File file);
 
     void check$java_io_File$$createTempFile(Class<?> callerClass, String prefix, String suffix, File directory);
@@ -518,6 +528,28 @@ public interface EntitlementChecker {
 
     void check$java_io_File$deleteOnExit(Class<?> callerClass, File file);
 
+    void check$java_io_File$exists(Class<?> callerClass, File file);
+
+    void check$java_io_File$isDirectory(Class<?> callerClass, File file);
+
+    void check$java_io_File$isFile(Class<?> callerClass, File file);
+
+    void check$java_io_File$isHidden(Class<?> callerClass, File file);
+
+    void check$java_io_File$lastModified(Class<?> callerClass, File file);
+
+    void check$java_io_File$length(Class<?> callerClass, File file);
+
+    void check$java_io_File$list(Class<?> callerClass, File file);
+
+    void check$java_io_File$list(Class<?> callerClass, File file, FilenameFilter filter);
+
+    void check$java_io_File$listFiles(Class<?> callerClass, File file);
+
+    void check$java_io_File$listFiles(Class<?> callerClass, File file, FileFilter filter);
+
+    void check$java_io_File$listFiles(Class<?> callerClass, File file, FilenameFilter filter);
+
     void check$java_io_File$mkdir(Class<?> callerClass, File file);
 
     void check$java_io_File$mkdirs(Class<?> callerClass, File file);
@@ -654,6 +686,19 @@ void checkNewByteChannel(
 
     void checkType(Class<?> callerClass, FileStore that);
 
+    // path
+    void checkPathToRealPath(Class<?> callerClass, Path that, LinkOption... options);
+
+    void checkPathRegister(Class<?> callerClass, Path that, WatchService watcher, WatchEvent.Kind<?>... events);
+
+    void checkPathRegister(
+        Class<?> callerClass,
+        Path that,
+        WatchService watcher,
+        WatchEvent.Kind<?>[] events,
+        WatchEvent.Modifier... modifiers
+    );
+
     ////////////////////
     //
     // Thread management
@@ -674,5 +719,4 @@ void checkNewByteChannel(
     void check$java_lang_Thread$setUncaughtExceptionHandler(Class<?> callerClass, Thread thread, Thread.UncaughtExceptionHandler ueh);
 
     void check$java_lang_ThreadGroup$setMaxPriority(Class<?> callerClass, ThreadGroup threadGroup, int pri);
-
 }
diff --git a/...ent-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java b/...ent-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java
@@ -46,6 +46,21 @@ static Path readWriteFile() {
         return testRootDir.resolve("read_write_file");
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileCanExecute() throws IOException {
+        readFile().toFile().canExecute();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileCanRead() throws IOException {
+        readFile().toFile().canRead();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileCanWrite() throws IOException {
+        readFile().toFile().canWrite();
+    }
+
     @EntitlementTest(expectedAccess = PLUGINS)
     static void fileCreateNewFile() throws IOException {
         readWriteDir().resolve("new_file").toFile().createNewFile();
@@ -70,6 +85,61 @@ static void fileDeleteOnExit() throws IOException {
         toDelete.toFile().deleteOnExit();
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileExists() throws IOException {
+        readFile().toFile().exists();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileIsDirectory() throws IOException {
+        readFile().toFile().isDirectory();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileIsFile() throws IOException {
+        readFile().toFile().isFile();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileIsHidden() throws IOException {
+        readFile().toFile().isHidden();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileLastModified() throws IOException {
+        readFile().toFile().lastModified();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileLength() throws IOException {
+        readFile().toFile().length();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileList() throws IOException {
+        readDir().toFile().list();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileListWithFilter() throws IOException {
+        readDir().toFile().list((dir, name) -> true);
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileListFiles() throws IOException {
+        readDir().toFile().listFiles();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileListFilesWithFileFilter() throws IOException {
+        readDir().toFile().listFiles(pathname -> true);
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void fileListFilesWithFilenameFilter() throws IOException {
+        readDir().toFile().listFiles((dir, name) -> true);
+    }
+
     @EntitlementTest(expectedAccess = PLUGINS)
     static void fileMkdir() throws IOException {
         Path mkdir = readWriteDir().resolve("mkdir");

diff --git a/...itlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/PathActions.java b/...itlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/PathActions.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.test;
+
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.LinkOption;
+import java.nio.file.WatchEvent;
+
+import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.PLUGINS;
+
+class PathActions {
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void checkToRealPath() throws IOException {
+        FileCheckActions.readFile().toRealPath();
+    }
+
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void checkToRealPathNoFollow() throws IOException {
+        FileCheckActions.readFile().toRealPath(LinkOption.NOFOLLOW_LINKS);
+    }
+
+    @SuppressWarnings("rawtypes")
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void checkRegister() throws IOException {
+        try (var watchService = FileSystems.getDefault().newWatchService()) {
+            FileCheckActions.readFile().register(watchService, new WatchEvent.Kind[0]);
+        } catch (IllegalArgumentException e) {
+            // intentionally no events registered
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void checkRegisterWithModifiers() throws IOException {
+        try (var watchService = FileSystems.getDefault().newWatchService()) {
+            FileCheckActions.readFile().register(watchService, new WatchEvent.Kind[0], new WatchEvent.Modifier[0]);
+        } catch (IllegalArgumentException e) {
+            // intentionally no events registered
+        }
+    }
+}
diff --git a/...ugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java b/...ugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java
@@ -190,6 +190,7 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
         getTestEntries(ManageThreadsActions.class),
         getTestEntries(NativeActions.class),
         getTestEntries(NioFileSystemActions.class),
+        getTestEntries(PathActions.class),
         getTestEntries(SpiActions.class),
         getTestEntries(SystemActions.class)
     )