From e5d80f89b031c69695c9c79deb3fe6b5bee1d719 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 2 Jan 2024 06:48:11 +0100 Subject: feat #15: add connection checks to bwHC backend --- .../dnpm/etl/processor/config/AppConfiguration.kt | 4 +- .../etl/processor/config/AppKafkaConfiguration.kt | 9 ++- .../etl/processor/config/AppRestConfiguration.kt | 12 ++- .../processor/monitoring/ConnectionCheckService.kt | 85 ++++++++++++++++++++++ .../dev/dnpm/etl/processor/web/ConfigController.kt | 51 +++++++++++++ .../etl/processor/web/TransformationController.kt | 41 ----------- src/main/resources/templates/configs.html | 76 +++++++++++++++++++ src/main/resources/templates/fragments.html | 2 +- src/main/resources/templates/transformations.html | 47 ------------ 9 files changed, 235 insertions(+), 92 deletions(-) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt delete mode 100644 src/main/kotlin/dev/dnpm/etl/processor/web/TransformationController.kt create mode 100644 src/main/resources/templates/configs.html delete mode 100644 src/main/resources/templates/transformations.html diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index c8e86fb..b6bedf5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -1,7 +1,7 @@ /* * This file is part of ETL-Processor * - * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published @@ -32,6 +32,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.scheduling.annotation.EnableScheduling import reactor.core.publisher.Sinks @Configuration @@ -42,6 +43,7 @@ import reactor.core.publisher.Sinks GPasConfigProperties::class ] ) +@EnableScheduling class AppConfiguration { private val logger = LoggerFactory.getLogger(AppConfiguration::class.java) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt index 309ff2d..c8fbdf5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppKafkaConfiguration.kt @@ -1,7 +1,7 @@ /* * This file is part of ETL-Processor * - * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published @@ -20,6 +20,8 @@ package dev.dnpm.etl.processor.config import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.monitoring.ConnectionCheckService +import dev.dnpm.etl.processor.monitoring.KafkaConnectionCheckService import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor @@ -76,4 +78,9 @@ class AppKafkaConfiguration { return KafkaResponseProcessor(applicationEventPublisher, objectMapper) } + @Bean + fun connectionCheckService(consumerFactory: ConsumerFactory): ConnectionCheckService { + return KafkaConnectionCheckService(consumerFactory.createConsumer()) + } + } \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt index a830597..2596e1c 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt @@ -1,7 +1,7 @@ /* * This file is part of ETL-Processor * - * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published @@ -19,6 +19,8 @@ package dev.dnpm.etl.processor.config +import dev.dnpm.etl.processor.monitoring.ConnectionCheckService +import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.output.RestMtbFileSender import org.slf4j.LoggerFactory @@ -54,5 +56,13 @@ class AppRestConfiguration { return RestMtbFileSender(restTemplate, restTargetProperties) } + @Bean + fun connectionCheckService( + restTemplate: RestTemplate, + restTargetProperties: RestTargetProperties + ): ConnectionCheckService { + return RestConnectionCheckService(restTemplate, restTargetProperties) + } + } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt new file mode 100644 index 0000000..d109326 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt @@ -0,0 +1,85 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +package dev.dnpm.etl.processor.monitoring + +import dev.dnpm.etl.processor.config.RestTargetProperties +import jakarta.annotation.PostConstruct +import org.apache.kafka.clients.consumer.Consumer +import org.apache.kafka.common.errors.TimeoutException +import org.springframework.http.HttpStatus +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.web.client.RestTemplate +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toJavaDuration + +interface ConnectionCheckService { + + fun connectionAvailable(): Boolean + +} + +class KafkaConnectionCheckService( + private val consumer: Consumer +) : ConnectionCheckService { + + private var connectionAvailable: Boolean = false + + + @PostConstruct + @Scheduled(cron = "0 * * * * *") + fun check() { + connectionAvailable = try { + null != consumer.listTopics(5.seconds.toJavaDuration()) + } catch (e: TimeoutException) { + false + } + } + + override fun connectionAvailable(): Boolean { + return this.connectionAvailable + } + +} + +class RestConnectionCheckService( + private val restTemplate: RestTemplate, + private val restTargetProperties: RestTargetProperties +) : ConnectionCheckService { + + private var connectionAvailable: Boolean = false + + @PostConstruct + @Scheduled(cron = "0 * * * * *") + fun check() { + connectionAvailable = try { + restTemplate.getForEntity( + restTargetProperties.uri?.replace("/etl/api", "").toString(), + String::class.java + ).statusCode == HttpStatus.OK + } catch (e: Exception) { + false + } + } + + override fun connectionAvailable(): Boolean { + return this.connectionAvailable + } +} \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt new file mode 100644 index 0000000..7bdbb82 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt @@ -0,0 +1,51 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package dev.dnpm.etl.processor.web + +import dev.dnpm.etl.processor.monitoring.ConnectionCheckService +import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.pseudonym.Generator +import dev.dnpm.etl.processor.services.TransformationService +import org.springframework.stereotype.Controller +import org.springframework.ui.Model +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping + +@Controller +@RequestMapping(path = ["configs"]) +class ConfigController( + private val transformationService: TransformationService, + private val pseudonymGenerator: Generator, + private val mtbFileSender: MtbFileSender, + private val connectionCheckService: ConnectionCheckService + +) { + + @GetMapping + fun index(model: Model): String { + model.addAttribute("pseudonymGenerator", pseudonymGenerator.javaClass.simpleName) + model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName) + model.addAttribute("connectionAvailable", connectionCheckService.connectionAvailable()) + model.addAttribute("transformations", transformationService.getTransformations()) + + return "configs" + } + +} \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/TransformationController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/TransformationController.kt deleted file mode 100644 index f811e9e..0000000 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/TransformationController.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * This file is part of ETL-Processor - * - * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package dev.dnpm.etl.processor.web - -import dev.dnpm.etl.processor.services.TransformationService -import org.springframework.stereotype.Controller -import org.springframework.ui.Model -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.RequestMapping - -@Controller -@RequestMapping(path = ["transformations"]) -class TransformationController( - private val transformationService: TransformationService -) { - - @GetMapping - fun index(model: Model): String { - model.addAttribute("transformations", transformationService.getTransformations()) - - return "transformations" - } - -} \ No newline at end of file diff --git a/src/main/resources/templates/configs.html b/src/main/resources/templates/configs.html new file mode 100644 index 0000000..1d76063 --- /dev/null +++ b/src/main/resources/templates/configs.html @@ -0,0 +1,76 @@ + + + + + ETL-Prozessor + + + +
+
+

Konfiguration

+ +

Allgemeine Konfiguration

+ + + + + + + + + + + + + + + + + +
NameWert
Pseudonym erzeugt über[[ ${pseudonymGenerator} ]]
MTBFile-Sender[[ ${mtbFileSender} ]]
+ +

✅⚡ Verbindung zum bwHC-Backend

+

+ Verbindung über [[ ${mtbFileSender} ]]. Die Verbindung ist aktuell + verfügbar. + nicht verfügbar! +

+ +

Transformationen

+ +

Syntax

+ Hier einige Beispiele zum Syntax des JSON-Path +
    +
  • diagnoses[*].icdO3T.version: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung
    • +
  • patient.gender: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format
    • +
+ +

Konfigurierte Transformationen

+

+ Hier sehen Sie eine Übersicht der konfigurierten Transformationen. +

+ + + + + + + + + + + + + + +
JSON-PathTransformation von ⇒ nach
+ [[ ${transformation.path} ]] + + [[ ${transformation.existingValue} ]] + ⇒ + [[ ${transformation.newValue} ]] +
+
+ + \ No newline at end of file diff --git a/src/main/resources/templates/fragments.html b/src/main/resources/templates/fragments.html index ea0fe1d..fbbe47d 100644 --- a/src/main/resources/templates/fragments.html +++ b/src/main/resources/templates/fragments.html @@ -10,7 +10,7 @@ diff --git a/src/main/resources/templates/transformations.html b/src/main/resources/templates/transformations.html deleted file mode 100644 index b8e2f37..0000000 --- a/src/main/resources/templates/transformations.html +++ /dev/null @@ -1,47 +0,0 @@ - - - - - ETL-Prozessor - - - -
-
-

Transformationen

- -

Syntax

- Hier einige Beispiele zum Syntax des JSON-Path -
    -
  • diagnoses[*].icdO3T.version: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung
    • -
  • patient.gender: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format
    • -
- -

Konfigurierte Transformationen

-

- Hier sehen Sie eine Übersicht der konfigurierten Transformationen. -

- - - - - - - - - - - - - - -
JSON-PathTransformation von ⇒ nach
- [[ ${transformation.path} ]] - - [[ ${transformation.existingValue} ]] - ⇒ - [[ ${transformation.newValue} ]] -
-
- - \ No newline at end of file -- cgit v1.2.3