init elasticsearch migration WIP

backend/src/main/kotlin/ch/boxi/boxit/infrastructur/logging/LoggerHelper.kt

@@ -75,8 +75,7 @@ object LoggerHelper {
     }
 
     fun setNewRequestId(prefix: String): String {
-        var requestId = generateRequestId(prefix)
-        requestId = requestId
+        requestId = generateRequestId(prefix)
         return requestId
     }
 

backend/src/main/kotlin/ch/boxi/boxit/infrastructur/util/Streams.kt

@@ -1,5 +1,6 @@
 package ch.boxi.boxit.infrastructur.util
 
+import java.util.SortedSet
 import java.util.stream.Collectors
 import java.util.stream.DoubleStream
 import java.util.stream.IntStream
@@ -7,9 +8,13 @@ import java.util.stream.LongStream
 import java.util.stream.Stream
 
 fun <T> Stream<T>.toSet(): Set<T> = collect(Collectors.toSet<T>())
+fun <T> Stream<T>.toSortedSet(): SortedSet<T> = collect(ch.boxi.boxit.infrastructur.util.Collectors.toSortedSet<T>())
 
 fun IntStream.toSet(): Set<Int> = toArray().toSet()
+fun IntStream.toSortedSet(): SortedSet<Int> = toArray().toSortedSet()
 
 fun LongStream.toSet(): Set<Long> = toArray().toSet()
+fun LongStream.toSortedSet(): SortedSet<Long> = toArray().toSortedSet()
 
 fun DoubleStream.toSet(): Set<Double> = toArray().toSet()
+fun DoubleStream.toSortedSet(): SortedSet<Double> = toArray().toSortedSet()

backend/src/main/kotlin/ch/boxi/boxit/searchindex/ElasticSearchConfiguration.kt

@@ -1,32 +0,0 @@
-package ch.boxi.boxit.searchindex
-
-import co.elastic.clients.elasticsearch.ElasticsearchClient
-import co.elastic.clients.json.jackson.JacksonJsonpMapper
-import co.elastic.clients.transport.ElasticsearchTransport
-import co.elastic.clients.transport.rest_client.RestClientTransport
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.datatype.jsr310.JSR310Module
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
-import org.apache.http.HttpHost
-import org.elasticsearch.client.RestClient
-import org.springframework.beans.factory.annotation.Value
-import org.springframework.context.annotation.Bean
-import org.springframework.context.annotation.Configuration
-
-
-//@Configuration
-class ElasticSearchConfiguration(@Value("\${boxit.elasticsearch.host}") private val host: String,
-                                 @Value("\${boxit.elasticsearch.port}") private val port: Int) {
-//    @Bean
-    fun elasticsearchClient(): ElasticsearchClient {
-        val restClient: RestClient = RestClient.builder(HttpHost(host, port)).build()
-        val transport: ElasticsearchTransport = RestClientTransport(restClient, createJsonMapper())
-        return ElasticsearchClient(transport)
-    }
-
-    private fun createJsonMapper(): JacksonJsonpMapper {
-        val objectMapper = ObjectMapper().registerModule(JavaTimeModule())
-        val mapper = JacksonJsonpMapper(objectMapper)
-        return mapper
-    }
-}

backend/src/main/kotlin/ch/boxi/boxit/searchindex/NoteIndex.kt

@@ -3,6 +3,7 @@ package ch.boxi.boxit.searchindex
 import ch.boxi.boxit.domain.location.Coordinates
 import ch.boxi.boxit.domain.note.Note
 import ch.boxi.boxit.infrastructur.logging.Logger
+import ch.boxi.boxit.searchindex.client.EsClientProvider
 import co.elastic.clients.elasticsearch.core.IndexRequest.Builder
 import co.elastic.clients.elasticsearch.core.IndexResponse
 import org.springframework.stereotype.Component

backend/src/main/kotlin/ch/boxi/boxit/searchindex/client/EsClientProvider.kt

@@ -1,10 +1,9 @@
-package ch.boxi.boxit.searchindex
+package ch.boxi.boxit.searchindex.client
 
 import co.elastic.clients.elasticsearch.ElasticsearchClient
 import co.elastic.clients.json.jackson.JacksonJsonpMapper
 import co.elastic.clients.transport.rest_client.RestClientTransport
 import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.databind.SerializationFeature.CLOSE_CLOSEABLE
 import com.fasterxml.jackson.databind.SerializationFeature.WRITE_DATES_AS_TIMESTAMPS
 import com.fasterxml.jackson.databind.util.StdDateFormat
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
@@ -26,20 +25,28 @@ class EsClientProvider(@Value("\${boxit.elasticsearch.host}") private val host:
     private val jsonMapper = createJsonMapper()
 
     fun createClient(): ElasticsearchClient {
-        val credentialsProvider = BasicCredentialsProvider()
-        credentialsProvider.setCredentials(
-            AuthScope.ANY,
-            UsernamePasswordCredentials(user, password)
-        )
+        val restClient = createRestClient()
+        val transport = RestClientTransport(restClient, jsonMapper)
+        return ElasticsearchClient(transport)
+    }
 
+    fun createRestClient(): RestClient {
+        val credentialsProvider = createCredentialsProvider()
         val restClient = RestClient.builder(HttpHost(host, port, "https"))
             .setHttpClientConfigCallback { httpClientBuilder ->
                 httpClientBuilder.disableAuthCaching()
                 httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
             }.build()
+        return restClient
+    }
 
-        val transport = RestClientTransport(restClient, jsonMapper)
-        return ElasticsearchClient(transport)
+    private fun createCredentialsProvider(): BasicCredentialsProvider {
+        val credentialsProvider = BasicCredentialsProvider()
+        credentialsProvider.setCredentials(
+            AuthScope.ANY,
+            UsernamePasswordCredentials(user, password)
+        )
+        return credentialsProvider
     }
 
     fun createJsonMapper(): JacksonJsonpMapper {

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/ElasticSearchMigrationService.kt

@@ -0,0 +1,104 @@
+package ch.boxi.boxit.searchindex.migration
+
+import ch.boxi.boxit.infrastructur.logging.Logger
+import ch.boxi.boxit.infrastructur.logging.LoggerHelper
+import ch.boxi.boxit.searchindex.client.EsClientProvider
+import ch.boxi.boxit.searchindex.migration.data.MigrationEntity
+import ch.boxi.boxit.searchindex.migration.data.MigrationScript
+import ch.boxi.boxit.security.SSLClientConfig
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery
+import co.elastic.clients.elasticsearch._types.query_dsl.Query
+import co.elastic.clients.elasticsearch.core.SearchRequest
+import co.elastic.clients.elasticsearch.core.search.Hit
+import org.elasticsearch.client.Request
+import org.springframework.http.HttpMethod
+import org.springframework.stereotype.Service
+import java.util.SortedSet
+import javax.annotation.PostConstruct
+
+@Service
+class ElasticSearchMigrationService(private val clientProvider: EsClientProvider,
+                                    private val sslConfig: SSLClientConfig) {
+
+    companion object : Logger {
+        private val MIGRATION_INDEX_NAME = "migration_history"
+    }
+
+
+    @PostConstruct
+    fun migrate() {
+        LoggerHelper.setNewValues("ELASTICSEARCH_MIGRATION", "ESMIG")
+
+        getOrCreateMigrationIndex()
+        loadMigrations()
+        LoggerHelper.cleanUp()
+    }
+
+    private fun loadMigrations(): SortedSet<MigrationScript> {
+        return MigrationLoader().loadMigrations()
+    }
+
+    private fun getOrCreateMigrationIndex(): List<MigrationEntity> {
+        if (!migrationIndexExists()) {
+            createMigrationHistoryIndex()
+        }
+        LOG.debug("load migrations")
+        val response = clientProvider.createClient().search(
+            { s: SearchRequest.Builder ->
+                s.index(MIGRATION_INDEX_NAME)
+                    .query { q: Query.Builder ->
+                        q.matchAll() { a: MatchAllQuery.Builder -> a }
+                    }
+            },
+            MigrationEntity::class.java
+        )
+        LOG.info("found ${response.hits().total()?.value()} executed migrations:")
+        val migrations: List<Hit<MigrationEntity>> = response.hits().hits()
+        for (migration in migrations) {
+            LOG.info(migration.source()?.toString())
+        }
+        return migrations.stream()
+            .map{ m -> m.source()!! }
+            .toList()
+    }
+
+    private fun migrationIndexExists(): Boolean {
+        LOG.debug("search for existing \"$MIGRATION_INDEX_NAME\" index")
+        val indices = clientProvider.createClient().cat().indices()
+        indices.valueBody().stream().anyMatch { record -> record.index() == MIGRATION_INDEX_NAME }
+        val found = indices.valueBody().stream()
+            .anyMatch { record -> record.index() == MIGRATION_INDEX_NAME }
+        if (found) {
+            LOG.debug("found existing \"$MIGRATION_INDEX_NAME\" index")
+        }
+        return found
+    }
+
+    private fun createMigrationHistoryIndex() {
+        LOG.info("create \"$MIGRATION_INDEX_NAME\" index")
+        val createIndexRequest = Request(HttpMethod.PUT.name, "/$MIGRATION_INDEX_NAME")
+        createIndexRequest.setJsonEntity("""
+            {
+                "mappings": {
+                    "dynamic": "strict",
+                    "properties": {
+                        "installationRank": { "type": "integer" },
+                        "version": { "type": "keyword" },
+                        "description": { "type": "keyword" },
+                        "type": { "type": "keyword" },
+                        "script": { "type": "keyword" },
+                        "checksum": { "type": "integer" },
+                        "installedOn": { "type": "date" },
+                        "executionTime": { "type": "keyword" },
+                        "requestId": { "type": "keyword" },
+                        "correlationId": { "type": "keyword" },
+                        "success": { "type": "boolean" },
+                        "errorMsq": { "type": "text", "analyzer": "standard" }
+                    }
+                }
+            }
+        """.trimIndent())
+        clientProvider.createRestClient().performRequest(createIndexRequest)
+        LOG.debug("create index successfully done")
+    }
+}

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/MigrationLoader.kt

@@ -0,0 +1,63 @@
+package ch.boxi.boxit.searchindex.migration
+
+import ch.boxi.boxit.infrastructur.logging.Logger
+import ch.boxi.boxit.infrastructur.util.toSortedSet
+import ch.boxi.boxit.searchindex.migration.data.MigrationOperation
+import ch.boxi.boxit.searchindex.migration.data.MigrationScript
+import ch.boxi.boxit.searchindex.migration.data.Version
+import com.google.common.io.PatternFilenameFilter
+import org.springframework.core.io.ClassPathResource
+import org.springframework.http.HttpMethod
+import java.io.BufferedReader
+import java.io.File
+import java.io.FileReader
+import java.nio.file.Path
+import java.util.Arrays
+import java.util.SortedSet
+import java.util.regex.Pattern
+
+class MigrationLoader {
+
+    companion object : Logger {
+        private val FILENAME_PATTERN = Pattern.compile("(V\\d+.\\d+)__(.+)\\.edql")
+    }
+
+    fun loadMigrations(): SortedSet<MigrationScript> {
+        val filenameFilter = PatternFilenameFilter(FILENAME_PATTERN)
+        val folderResource = ClassPathResource("elasticsearch")
+        return Arrays.stream(folderResource.file.listFiles(filenameFilter))
+            .map(File::toPath)
+            .map(this::loadMigration)
+            .toSortedSet()
+    }
+
+    private fun loadMigration(path: Path): MigrationScript {
+        LOG.debug("found migration file $path")
+        val matcher = FILENAME_PATTERN.matcher(path.fileName.toString())
+
+        val script = matcher.group(0)
+        val version = Version.parse(matcher.group(1))
+        val description = matcher.group(2)
+
+        val operations = readOperations(path)
+
+        return MigrationScript(version, description, script, operations)
+    }
+
+    private fun readOperations(path: Path): List<MigrationOperation> {
+        BufferedReader(FileReader(path.toFile())).use {
+            it.lines().forEach {
+                val subLine = it.trim()
+                val isStart : Boolean = Arrays.stream(HttpMethod.values())
+                    .map { HttpMethod::name }
+                    .anyMatch { method -> subLine.startsWith(it) }
+                if (subLine.startsWith("HOST")
+                    || subLine.startsWith("Timeout")
+                    || subLine.startsWith("Authorization")) {
+
+                }
+            }
+        }
+        TODO("Not yet implemented")
+    }
+}

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/data/MigrationEntity.kt

@@ -0,0 +1,19 @@
+package ch.boxi.boxit.searchindex.migration.data
+
+import java.time.Duration
+import java.time.Instant
+
+data class MigrationEntity(
+    val installationRank: Int,
+    val version: String,
+    val description: String,
+    val type: MigrationType,
+    val script: String,
+    val checksum: Int,
+    val installedOn: Instant,
+    val executionTime: Duration,
+    val requestId: String,
+    val correlationId: String,
+    val success: Boolean,
+    val errorMsq: String
+)

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/data/MigrationOperation.kt

@@ -0,0 +1,9 @@
+package ch.boxi.boxit.searchindex.migration.data
+
+import org.springframework.http.HttpMethod
+
+data class MigrationOperation(
+    val httpMethod: HttpMethod,
+    val path: String,
+    val body: String?
+)

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/data/MigrationScript.kt

@@ -0,0 +1,8 @@
+package ch.boxi.boxit.searchindex.migration.data
+
+data class MigrationScript (
+    val version: Version,
+    val description: String,
+    val script: String,
+    val operations: List<MigrationOperation>
+)

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/data/MigrationType.kt

@@ -0,0 +1,5 @@
+package ch.boxi.boxit.searchindex.migration.data
+
+enum class MigrationType {
+    EDQL, CLASS
+}

backend/src/main/kotlin/ch/boxi/boxit/searchindex/migration/data/Version.kt

@@ -0,0 +1,36 @@
+package ch.boxi.boxit.searchindex.migration.data
+
+import ch.boxi.boxit.infrastructur.exception.BoxitException
+import java.util.regex.Pattern
+
+data class Version(val major: Int,
+                   val minor: Int): Comparable<Version> {
+    companion object {
+        fun parse(value: String): Version {
+            val pattern = Pattern.compile("V(\\d+)\\.(\\d+)")
+            val matcher = pattern.matcher(value)
+            if (!matcher.matches()) {
+                throw VersionParseException(value)
+            }
+            val major = matcher.group(1).toInt()
+            val minor = matcher.group(2).toInt()
+            return Version(major, minor)
+        }
+    }
+
+    override fun compareTo(other: Version): Int {
+        return if (this.major != other.major) {
+            this.major.compareTo(other.major)
+        } else {
+            this.minor.compareTo(other.minor)
+        }
+    }
+
+    override fun toString(): String {
+        return "V$major.$minor"
+    }
+
+
+}
+
+class VersionParseException(value: String) : BoxitException("versionStringNotValid", "illageal VersionString must be in Format \"V<major>.<minor>\" but was \"$value\"")

backend/src/main/resources/elasticsearch/V1.0__init.edql

@@ -92,7 +92,7 @@ PUT _ingest/pipeline/translatedNote
 DELETE /note
 DELETE /task
 
-POST _analyze
+POST /_analyze
 {
   "analyzer": "default",
   "text":     "Ich will das wollen und wollte richtig analysiert wird."

backend/src/test/kotlin/ch/boxi/boxit/searchindex/migration/data/VersionTest.kt

@@ -0,0 +1,51 @@
+package ch.boxi.boxit.searchindex.migration.data
+
+import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.equalTo
+import org.hamcrest.Matchers.greaterThan
+import org.hamcrest.Matchers.`is`
+import org.hamcrest.Matchers.lessThan
+import org.junit.jupiter.api.Assertions.assertThrows
+import org.junit.jupiter.api.Test
+
+
+internal class VersionTest {
+    @Test
+    fun parse() {
+        assertThat(Version.parse("V1.2"), `is`(Version(1, 2)))
+        assertThat(Version.parse("V13.2"), `is`(Version(13, 2)))
+    }
+
+    @Test
+    fun parse_illegal_format_throws_exception() {
+        assertThrows(VersionParseException::class.java) { Version.parse("1.2") }
+        assertThrows(VersionParseException::class.java) { Version.parse("V1") }
+        assertThrows(VersionParseException::class.java) { Version.parse("V1.") }
+        assertThrows(VersionParseException::class.java) { Version.parse("V1.2.3") }
+        assertThrows(VersionParseException::class.java) { Version.parse("V1.2rc1") }
+    }
+
+    @Test
+    fun toStringTest() {
+        assertThat(Version(1, 2).toString(), `is`("V1.2"))
+    }
+
+    @Test
+    fun compareTo() {
+        val versionSmall = Version(1, 2)
+        val versionMiddle = Version( 1, 3)
+        val versionMiddle2 = Version( 1, 3)
+        val versionBig = Version(2, 1)
+
+        assertThat(versionSmall, lessThan(versionMiddle))
+        assertThat(versionSmall, lessThan(versionBig))
+
+        assertThat(versionMiddle, greaterThan(versionSmall))
+        assertThat(versionMiddle, lessThan(versionBig))
+
+        assertThat(versionBig, greaterThan(versionMiddle))
+        assertThat(versionBig, greaterThan(versionSmall))
+
+        assertThat(versionMiddle, equalTo(versionMiddle2))
+    }
+}

backend/src/test/resources/application.properties

@@ -7,8 +7,8 @@ spring.datasource.password=${DB_PASSWD:testpw}
 
 spring.jackson.date-format="yyyy.MM.dd HH:mm:ss"
 
-boxit.server.ssl.trust-store=truststore
-#boxit.server.ssl.trust-store=/Users/boxi/IdeaProjects/boxit/backend/src/main/resources/truststore.jks.jks.jks
+#boxit.server.ssl.trust-store=truststore
+boxit.server.ssl.trust-store=/Users/boxi/IdeaProjects/boxit/backend/src/main/resources/truststore.jks
 boxit.server.ssl.trust-store-password=Tester11
 
 management.endpoint.metrics.enabled=true