Added option to trim trailing newline in migration scripts for backward compatibility / checksum stability

README.md

@@ -193,6 +193,8 @@ Elasticsearch-Evolution can be configured to your needs:
 -   **baselineVersion** (default=1.0): Version to use as a baseline. versions lower than it will not be applied.
 -   **lineSeparator** (default=\n): Line separator, used only temporary between reading raw migration file line-by-line and parsing it later. Only needed for backward compatibility / checksum stability! Should be one of `\n`, `\r` or `\r\n`
 -   **outOfOrder** (default=false): Allows migrations to be run "out of order". If you already have versions 1.0 and 3.0 applied, and now a version 2.0 is found, it will be applied too instead of being rejected.
+- **trimTrailingNewlineInMigrations** (default=false): Whether to remove a trailing newline in migration scripts. Only
+  needed for backward compatibility / checksum stability!
 
 ### 5.1 Spring Boot
 
@@ -281,6 +283,7 @@ ElasticsearchEvolution.configure()
 
 ### v0.6.0-SNAPSHOT
 
+- Added option to trim a trailing newline in migration scripts (fixes [#298](https://github.com/senacor/elasticsearch-evolution/issues/298)). NOTE: This option is only needed for backward compatibility / checksum stability!
 - The minimum supported Java version is now 17.
 - Drop spring boot 2 compatibility. Further versions may run on spring boot 2, but it is not tested anymore.
 

elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/ElasticsearchEvolution.java

@@ -151,7 +151,8 @@ protected MigrationScriptReader createMigrationScriptReader() {
                 getConfig().getEncoding(),
                 getConfig().getEsMigrationPrefix(),
                 getConfig().getEsMigrationSuffixes(),
-                getConfig().getLineSeparator());
+                getConfig().getLineSeparator(),
+                getConfig().isTrimTrailingNewlineInMigrations());
     }
 
     protected HistoryRepository createHistoryRepository() {

elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/api/config/ElasticsearchEvolutionConfig.java

@@ -41,7 +41,8 @@ public class ElasticsearchEvolutionConfig {
 
     /**
      * Line separator, used only temporary between reading raw migration file line-by-line and parsing it later.
-     * Only needed for backward compatibility / checksum stability!
+     * <p>
+     * NOTE: Only needed for backward compatibility / checksum stability!
      * <p>
      * Should be one of
      * - '\n' (LF - Linux/Unix/OS X)
@@ -103,6 +104,13 @@ public class ElasticsearchEvolutionConfig {
      */
     private boolean validateOnMigrate = true;
 
+    /**
+     * Whether to remove a trailing newline in migration scripts.
+     * <p>
+     * NOTE: This is only needed for backward compatibility / checksum stability!
+     */
+    private boolean trimTrailingNewlineInMigrations = false;
+
     /**
      * version to use as a baseline.
      * The baseline version will be the first one applied, the versions below will be ignored.
@@ -295,6 +303,15 @@ public ElasticsearchEvolutionConfig setValidateOnMigrate(boolean validateOnMigra
         return this;
     }
 
+    public boolean isTrimTrailingNewlineInMigrations() {
+        return trimTrailingNewlineInMigrations;
+    }
+
+    public ElasticsearchEvolutionConfig setTrimTrailingNewlineInMigrations(boolean trimTrailingNewlineInMigrations) {
+        this.trimTrailingNewlineInMigrations = trimTrailingNewlineInMigrations;
+        return this;
+    }
+
     public String getBaselineVersion() {
         return baselineVersion;
     }
@@ -330,6 +347,7 @@ public String toString() {
                 ", historyIndex='" + historyIndex + '\'' +
                 ", historyMaxQuerySize=" + historyMaxQuerySize +
                 ", validateOnMigrate=" + validateOnMigrate +
+                ", trimTrailingNewlineInMigrations=" + trimTrailingNewlineInMigrations +
                 ", baselineVersion='" + baselineVersion + '\'' +
                 ", outOfOrder='" + outOfOrder + '\'' +
                 '}';

elasticsearch-evolution-core/src/main/java/com/senacor/elasticsearch/evolution/core/internal/migration/input/MigrationScriptReaderImpl.java

@@ -38,25 +38,28 @@ public class MigrationScriptReaderImpl implements MigrationScriptReader {
     private final String esMigrationPrefix;
     private final List<String> esMigrationSuffixes;
     private final String lineSeparator;
+    private final boolean trimTrailingNewlineInMigrations;
 
     /**
-     * @param locations               Locations of migrations scripts, e.g classpath:es/migration or file:/home/migration
-     * @param encoding                migrations scripts encoding
-     * @param esMigrationFilePrefix   File name prefix for ES migrations.
-     * @param esMigrationFileSuffixes File name suffix for ES migrations.
-     * @param lineSeparator           Line separator. should be '\n' per default and only something else for backward compatibility / hash stability
+     * @param locations                       Locations of migrations scripts, e.g classpath:es/migration or file:/home/migration
+     * @param encoding                        migrations scripts encoding
+     * @param esMigrationFilePrefix           File name prefix for ES migrations.
+     * @param esMigrationFileSuffixes         File name suffix for ES migrations.
+     * @param lineSeparator                   Line separator. should be '\n' per default and only something else for backward compatibility / checksum stability
+     * @param trimTrailingNewlineInMigrations Whether to remove a trailing newline in migration scripts.
      */
-
     public MigrationScriptReaderImpl(List<String> locations,
                                      Charset encoding,
                                      String esMigrationFilePrefix,
                                      List<String> esMigrationFileSuffixes,
-                                     String lineSeparator) {
+                                     String lineSeparator,
+                                     boolean trimTrailingNewlineInMigrations) {
         this.locations = locations;
         this.encoding = encoding;
         this.esMigrationPrefix = esMigrationFilePrefix;
         this.esMigrationSuffixes = esMigrationFileSuffixes;
         this.lineSeparator = lineSeparator;
+        this.trimTrailingNewlineInMigrations = trimTrailingNewlineInMigrations;
     }
 
     /**
@@ -164,6 +167,11 @@ Stream<RawMigrationScript> read(BufferedReader reader, String filename) throws I
         if (content.isEmpty()) {
             return Stream.empty();
         }
+
+        if (trimTrailingNewlineInMigrations && content.endsWith(lineSeparator)) {
+            content = content.substring(0, content.length() - lineSeparator.length());
+        }
+
         return Stream.of(new RawMigrationScript().setFileName(filename).setContent(content));
     }
 

elasticsearch-evolution-core/src/test/java/com/senacor/elasticsearch/evolution/core/internal/migration/input/MigrationScriptReaderImplTest.java

@@ -36,7 +36,7 @@ void nonJarFile() {
                         singletonList("classpath:scriptreader"),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
                 List<RawMigrationScript> actual = reader.read();
                 assertThat(actual)
                         .containsExactlyInAnyOrder(
@@ -50,7 +50,7 @@ void inJarFile() {
                         singletonList("classpath:META-INF"),
                         StandardCharsets.UTF_8,
                         "MANIFEST",
-                        singletonList(".MF"), "\n");
+                        singletonList(".MF"), "\n", false);
 
                 List<RawMigrationScript> res = reader.read();
 
@@ -67,7 +67,7 @@ void invalidClasspath() {
                         singletonList(classpath),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n") {
+                        singletonList(".http"), "\n", false) {
                     @Override
                     protected Stream<RawMigrationScript> readFromLocation(String location) throws URISyntaxException, IOException {
                         throw new URISyntaxException("input", "reason");
@@ -85,7 +85,7 @@ void multipleSuffixes() {
                         singletonList("classpath:scriptreader"),
                         StandardCharsets.UTF_8,
                         "c",
-                        Arrays.asList(".http", ".other"), "\n");
+                        Arrays.asList(".http", ".other"), "\n", false);
                 List<RawMigrationScript> actual = reader.read();
                 assertThat(actual)
                         .containsExactlyInAnyOrder(
@@ -100,7 +100,7 @@ void handlingDuplicates() {
                         Arrays.asList("classpath:scriptreader", "classpath:scriptreader"),
                         StandardCharsets.UTF_8,
                         "c",
-                        Arrays.asList(".http", ".other"), "\n");
+                        Arrays.asList(".http", ".other"), "\n", false);
                 List<RawMigrationScript> actual = reader.read();
                 assertThat(actual)
                         .containsExactlyInAnyOrder(
@@ -115,7 +115,7 @@ void exclude_locations_with_suffix() {
                         singletonList("classpath:scriptreader/issue36/location"),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
 
                 List<RawMigrationScript> actual = reader.read();
 
@@ -132,7 +132,7 @@ void handle_locations_with_suffix() {
                                 "classpath:scriptreader/issue36/location_with_suffix"),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
 
                 List<RawMigrationScript> actual = reader.read();
 
@@ -148,7 +148,7 @@ void include_trailing_newlines() {
                         Arrays.asList("classpath:scriptreader/issue293_trailing_newlines"),
                         StandardCharsets.UTF_8,
                         "w",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
 
                 List<RawMigrationScript> actual = reader.read();
 
@@ -163,7 +163,7 @@ void withWrongProtocol() {
                         Arrays.asList("classpath:scriptreader", "http:scriptreader"),
                         StandardCharsets.UTF_8,
                         "c",
-                        Arrays.asList(".http", ".other"), "\n");
+                        Arrays.asList(".http", ".other"), "\n", false);
                 assertThatThrownBy(reader::read)
                         .isInstanceOf(MigrationException.class)
                         .hasMessage("""
@@ -183,7 +183,7 @@ void normalPath() throws URISyntaxException {
                         singletonList("file:" + absolutePathToScriptreader),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
                 List<RawMigrationScript> actual = reader.read();
                 assertThat(actual)
                         .containsExactlyInAnyOrder(
@@ -199,7 +199,7 @@ void exclude_locations_with_suffix() throws URISyntaxException {
                         singletonList("file:"+absolutePathToScriptreader+"/issue36/location"),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
 
                 List<RawMigrationScript> actual = reader.read();
 
@@ -218,7 +218,7 @@ void handle_locations_with_suffix() throws URISyntaxException {
                                 "file:"+absolutePathToScriptreader+"/issue36/location_with_suffix"),
                         StandardCharsets.UTF_8,
                         "c",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
 
                 List<RawMigrationScript> actual = reader.read();
 
@@ -230,12 +230,12 @@ void handle_locations_with_suffix() throws URISyntaxException {
 
             @Test
             void invalidPath() {
-                assertThatThrownBy(() ->
-                        new MigrationScriptReaderImpl(
-                                singletonList("file:X:/snc/scripts"),
-                                StandardCharsets.UTF_8,
-                                "c",
-                                singletonList(".http"), "\n").read())
+                final MigrationScriptReaderImpl underTest = new MigrationScriptReaderImpl(
+                        singletonList("file:X:/snc/scripts"),
+                        StandardCharsets.UTF_8,
+                        "c",
+                        singletonList(".http"), "\n", false);
+                assertThatThrownBy(() -> underTest.read())
                         .isInstanceOf(MigrationException.class)
                         .hasMessage("couldn't read scripts from file:X:/snc/scripts");
             }
@@ -245,12 +245,12 @@ void invalidPath() {
             void validAndInvalidPath() throws URISyntaxException {
                 URL resourceDirectory = resolveURL("scriptreader");
                 String absolutePathToScriptreader = Paths.get(resourceDirectory.toURI()).toFile().getAbsolutePath();
-                assertThatThrownBy(() ->
-                        new MigrationScriptReaderImpl(
-                                Arrays.asList("file:X:/snc/scripts", "file:" + absolutePathToScriptreader),
-                                StandardCharsets.UTF_8,
-                                "c",
-                                singletonList(".http"), "\n").read())
+                final MigrationScriptReaderImpl underTest = new MigrationScriptReaderImpl(
+                        Arrays.asList("file:X:/snc/scripts", "file:" + absolutePathToScriptreader),
+                        StandardCharsets.UTF_8,
+                        "c",
+                        singletonList(".http"), "\n", false);
+                assertThatThrownBy(() -> underTest.read())
                         .isInstanceOf(MigrationException.class)
                         .hasMessage("couldn't read scripts from file:X:/snc/scripts");
             }
@@ -263,7 +263,7 @@ void validPathButNoFiles() throws URISyntaxException {
                         singletonList("file:" + absolutePathToScriptreader),
                         StandardCharsets.UTF_8,
                         "d",
-                        singletonList(".http"), "\n");
+                        singletonList(".http"), "\n", false);
                 List<RawMigrationScript> actual = reader.read();
                 assertThat(actual).isEmpty();
             }
@@ -272,25 +272,52 @@ void validPathButNoFiles() throws URISyntaxException {
 
     @ParameterizedTest
     @ValueSource(strings = {
-            "foo\nbar",
-            "foo\r\nbar",
-            "foo\rbar"
+            "foo\nbar\n",
+            "foo\r\nbar\r\n",
+            "foo\rbar\r"
     })
     void read_should_normalize_new_lines_to_defined_line_separator(String input) throws IOException {
         final String lineSeparator = "<my-line-separator>";
         MigrationScriptReaderImpl reader = new MigrationScriptReaderImpl(
                 singletonList("ignore"),
                 StandardCharsets.UTF_8,
                 "ignore",
-                singletonList(".ignore"), lineSeparator);
+                singletonList(".ignore"),
+                lineSeparator,
+                false);
+
+        final Stream<RawMigrationScript> res;
+        try (BufferedReader bufferedReader = new BufferedReader(new StringReader(input))) {
+            res = reader.read(bufferedReader, "filename");
+        }
+
+        assertThat(res)
+                .containsExactlyInAnyOrder(new RawMigrationScript().setFileName("filename").setContent("foo" + lineSeparator + "bar" + lineSeparator));
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "foo\nbar\n",
+            "foo\r\nbar\r\n",
+            "foo\rbar\r"
+    })
+    void read_should_trim_trailing_newlines_if_config_is_set(String input) throws IOException {
+        final String lineSeparator = "<my-line-separator>";
+        MigrationScriptReaderImpl reader = new MigrationScriptReaderImpl(
+                singletonList("ignore"),
+                StandardCharsets.UTF_8,
+                "ignore",
+                singletonList(".ignore"),
+                lineSeparator,
+                true);
 
         final Stream<RawMigrationScript> res;
         try (BufferedReader bufferedReader = new BufferedReader(new StringReader(input))) {
             res = reader.read(bufferedReader, "filename");
         }
 
         assertThat(res)
-                .containsExactlyInAnyOrder(new RawMigrationScript().setFileName("filename").setContent("foo"+lineSeparator+"bar"));
+                .containsExactlyInAnyOrder(new RawMigrationScript().setFileName("filename").setContent("foo" + lineSeparator + "bar"));
     }
 
     private URL resolveURL(String path) {