diff options
25 files changed, 949 insertions, 101 deletions
@@ -18,6 +18,14 @@ Zudem ist eine minimalistische Weboberfläche integriert, die einen Einblick in  +### 🔥 Wichtige Änderungen in Version 0.18 + +Ab Version 0.18 kann diese Anwendung zur Datenausleitung über die nNGM-REST-API verwendet werden. +Dazu ist eine Konfiguration des in dieser Version eingeführten [Routing-Mechanismus](#routed-rest) erforderlich + +Hier wird jeder `PatientRecord`, dessen (letzte) Diagnose einen ICD-10-Code beginnend mit `C34` und beliebiger Endung hat, +nicht über den Standard-Weg versendet, sondern an den für nNGM konfigurierten API-Endpunkt. + ### 🔥 Wichtige Änderungen in Version 0.17 Ab Version 0.17 verwendet diese Anwendung [pcvolkmer/mv64e-mtb-model](https://github.com/pcvolkmer/mv64e-mtb-model) und ersetzt @@ -402,8 +410,27 @@ Werden sowohl REST als auch Kafka-Endpunkt konfiguriert, wird nur der REST-Endpu Folgende Umgebungsvariablen müssen gesetzt sein, damit ein MTB-File an DNPM:DIP gesendet wird: * `APP_REST_URI`: URI der zu benutzenden API der Backend-Instanz. Zum Beispiel `http://localhost:9000/api` +* `APP_REST_TYPE`: Mögliche Werte `dip` (Standard) für DNPM:DIP und `nngm` für die nNGM-API. * `APP_REST_USERNAME`: Basic-Auth-Benutzername für den REST-Endpunkt * `APP_REST_PASSWORD`: Basic-Auth-Passwort für den REST-Endpunkt +* `APP_REST_API_KEY`: Angabe eines API-Keys zur Verwendung mit `APP_REST_TYPE=nngm` für die nNGM-API + +#### Routed REST + +Seit Version 0.18 ist eine Konfiguration des neu eingeführten **Routing**-Mechanismus möglich. +Damit kann für bestimmte Entitäten und Diagnosen ein abweichender REST-Endpunkt definiert werden. + +Beispiel: + +* `APP_ROUTING_NNGM_URI`: URI der nNGM-REST-API. +* `APP_ROUTING_NNGM_API_KEY`: API-Key zur Nutzung der nNGM-REST-API. +* `APP_ROUTING_NNGM_ICD10[0]`: Erster ICD-Code - weitere möglich. + +Mit dieser Konfiguration wird jeder `PatientRecord`, dessen (letzte) Diagnose einen ICD-10-Code hat, dessen Anfang +konfiguriert ist, nicht über den Standard-Weg versendet, sondern an den für nNGM konfigurierten API-Endpunkt. + +Wird zum Beispiel `APP_ROUTING_NNGM_ICD10[0] = C34` konfiguriert, werden Diagnosen wie "C34.0", "C34.1", etc. an das nNGM +übermittelt. #### Kafka-Topics diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt index 9f1e183..d54c0bf 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/EtlProcessorApplicationTests.kt @@ -24,6 +24,8 @@ import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.output.DnpmV2MtbFileRequest import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.output.RestDipMtbFileSender +import dev.dnpm.etl.processor.output.RestMtbFileSender import dev.pcvolkmer.mv64e.model.Coding import dev.pcvolkmer.mv64e.model.ConsentProvisionType import dev.pcvolkmer.mv64e.model.ModelProjectConsentPurpose @@ -57,7 +59,7 @@ import tools.jackson.databind.json.JsonMapper @Testcontainers @ExtendWith(SpringExtension::class) @SpringBootTest -@MockitoBean(types = [MtbFileSender::class]) +@MockitoBean(types = [RestMtbFileSender::class]) @TestPropertySource( properties = [ @@ -91,7 +93,7 @@ class EtlProcessorApplicationTests : AbstractTestcontainerTest() { ) inner class TransformationTest { - @MockitoBean private lateinit var mtbFileSender: MtbFileSender + @MockitoBean private lateinit var mtbFileSender: RestMtbFileSender private lateinit var mockMvc: MockMvc private lateinit var jsonMapper: JsonMapper diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt index 8052ed2..f2bfd90 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/config/AppConfigurationTest.kt @@ -29,6 +29,8 @@ import dev.dnpm.etl.processor.input.KafkaInputListener import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.RestDipMtbFileSender +import dev.dnpm.etl.processor.output.RestNngmMtbFileSender +import dev.dnpm.etl.processor.output.RoutedRestNngmMtbFileSender import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.GpasSoapPseudonymGenerator @@ -61,6 +63,7 @@ import tools.jackson.databind.json.JsonMapper KafkaAutoConfiguration::class, AppKafkaConfiguration::class, AppRestConfiguration::class, + AppRoutedRestConfiguration::class, ConsentEvaluator::class, ], ) @@ -83,6 +86,24 @@ class AppConfigurationTest { assertThrows<NoSuchBeanDefinitionException> { context.getBean<KafkaMtbFileSender>() } + assertThrows<NoSuchBeanDefinitionException> { + context.getBean<RestNngmMtbFileSender>() + } + } + } + + @Nested + @TestPropertySource(properties = ["app.rest.uri=http://localhost:9000", "app.routing.nngm.uri=http://localhost:9999"]) + inner class AppConfigurationSwitchedRestTest( + private val context: ApplicationContext, + ) { + @Test + fun shouldUseRestMtbFileSenderAndSwitchedNngmFileSender() { + assertThat(context.getBean<RestDipMtbFileSender>()).isNotNull + assertThat(context.getBean<RoutedRestNngmMtbFileSender>()).isNotNull + assertThrows<NoSuchBeanDefinitionException> { + context.getBean<KafkaMtbFileSender>() + } } } diff --git a/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt index bbade1c..b599134 100644 --- a/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt +++ b/src/integrationTest/kotlin/dev/dnpm/etl/processor/services/RequestServiceIntegrationTest.kt @@ -21,13 +21,9 @@ package dev.dnpm.etl.processor.services import dev.dnpm.etl.processor.* -import dev.dnpm.etl.processor.monitoring.Request -import dev.dnpm.etl.processor.monitoring.RequestRepository -import dev.dnpm.etl.processor.monitoring.RequestStatus -import dev.dnpm.etl.processor.monitoring.RequestType -import dev.dnpm.etl.processor.monitoring.SubmissionType -import dev.dnpm.etl.processor.output.MtbFileSender -import java.time.Instant +import dev.dnpm.etl.processor.monitoring.* +import dev.dnpm.etl.processor.output.RestDipMtbFileSender +import dev.dnpm.etl.processor.output.RestMtbFileSender import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -39,14 +35,15 @@ import org.springframework.test.context.bean.override.mockito.MockitoBean import org.springframework.test.context.junit.jupiter.SpringExtension import org.springframework.transaction.annotation.Transactional import org.testcontainers.junit.jupiter.Testcontainers +import java.time.Instant @Testcontainers @ExtendWith(SpringExtension::class) @SpringBootTest @Transactional -@MockitoBean(types = [MtbFileSender::class]) +@MockitoBean(types = [RestMtbFileSender::class]) @TestPropertySource( - properties = ["app.pseudonymize.generator=buildin", "app.rest.uri=http://example.com"] + properties = ["app.pseudonymize.generator=buildin", "app.rest.uri=http://example.com", "app.rest.type=dip"] ) class RequestServiceIntegrationTest : AbstractTestcontainerTest() { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 01bab32..fb93618 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -111,12 +111,36 @@ data class RestTargetProperties( val uri: String?, val username: String?, val password: String?, + val apiKey: String? = null, + val type: RestTargetType = RestTargetType.DIP, ) { companion object { const val NAME = "app.rest" } } +enum class RestTargetType { + DIP, + NNGM, +} + +@ConfigurationProperties(RoutingProperties.NAME) +data class RoutingProperties( + val nngm: SwitchTargetProperties? = null, +) { + companion object { + const val NAME = "app.routing" + } +} + +data class SwitchTargetProperties( + val uri: String, + val username: String?, + val password: String?, + val apiKey: String?, + val icd10: List<String> = listOf(), +) + @ConfigurationProperties(KafkaProperties.NAME) data class KafkaProperties( val inputTopic: String?, diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt index d10a32f..70cfca9 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt @@ -24,10 +24,11 @@ import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult import dev.dnpm.etl.processor.monitoring.ConnectionCheckService import dev.dnpm.etl.processor.monitoring.ReportService import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService -import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.output.RestDipMtbFileSender +import dev.dnpm.etl.processor.output.RestMtbFileSender +import dev.dnpm.etl.processor.output.RestNngmMtbFileSender import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean @@ -40,7 +41,6 @@ import reactor.core.publisher.Sinks @Configuration @EnableConfigurationProperties(value = [RestTargetProperties::class]) @ConditionalOnProperty(value = ["app.rest.uri"]) -@ConditionalOnMissingBean(MtbFileSender::class) @Order(-10) class AppRestConfiguration { private val logger = LoggerFactory.getLogger(AppRestConfiguration::class.java) @@ -51,12 +51,20 @@ class AppRestConfiguration { restTargetProperties: RestTargetProperties, retryTemplate: RetryTemplate, reportService: ReportService, - ): MtbFileSender { - logger.info("Selected 'RestDipMtbFileSender'") - return RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService) - } + ): RestMtbFileSender = + when (restTargetProperties.type) { + RestTargetType.DIP -> { + logger.info("Selected 'RestDipMtbFileSender'") + RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService) + } + RestTargetType.NNGM -> { + logger.info("Selected 'RestNngmMtbFileSender'") + RestNngmMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService) + } + } @Bean + @ConditionalOnBean(RestDipMtbFileSender::class) fun restConnectionCheckService( restTemplate: RestTemplate, restTargetProperties: RestTargetProperties, diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRoutedRestConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRoutedRestConfiguration.kt new file mode 100644 index 0000000..f962a6c --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRoutedRestConfiguration.kt @@ -0,0 +1,52 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken + * Copyright (c) 2025-2026 Paul-Christian Volkmer, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.config + +import dev.dnpm.etl.processor.monitoring.ReportService +import dev.dnpm.etl.processor.output.RoutedMtbFileSender +import dev.dnpm.etl.processor.output.RoutedRestNngmMtbFileSender +import org.slf4j.LoggerFactory +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.annotation.Order +import org.springframework.retry.support.RetryTemplate +import org.springframework.web.client.RestTemplate + +@Configuration +@EnableConfigurationProperties(value = [RoutingProperties::class]) +@Order(-10) +class AppRoutedRestConfiguration { + private val logger = LoggerFactory.getLogger(AppRoutedRestConfiguration::class.java) + + @Bean + @ConditionalOnProperty(prefix = "app.routing", name = ["nngm.uri"]) + fun switchRestMtbFileSender( + restTemplate: RestTemplate, + routingProperties: RoutingProperties, + retryTemplate: RetryTemplate, + reportService: ReportService, + ): RoutedMtbFileSender { + logger.info("Added switched 'RestNngmMtbFileSender' ... ") + return RoutedRestNngmMtbFileSender(restTemplate, routingProperties, retryTemplate, reportService) + } +} diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Destination.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Destination.kt new file mode 100644 index 0000000..e3d1915 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Destination.kt @@ -0,0 +1,30 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken + * Copyright (c) 2026 Paul-Christian Volkmer, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.monitoring + +enum class Destination( + val value: String, +) { + UNKNOWN("unknown"), + KAFKA("kafka"), + DNPM_DIP("dnpm_dip"), + NNGM_API("nngm_api"), +} diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index b9359fc..96cdb5c 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -35,19 +35,23 @@ class ReportService(private val jsonMapper: JsonMapper) { return listOf() } return try { - jsonMapper.readValue(dataQualityReport, DataQualityReport::class.java).issues.sortedBy { + jsonMapper.readValue(dataQualityReport, DipDataQualityReport::class.java).issues.sortedBy { it.severity } } catch (_: JacksonException) { - val otherIssue = - Issue(Severity.ERROR, Optional.of("Not parsable data quality report '$dataQualityReport'")) - return listOf(otherIssue) + return try { + jsonMapper.readValue(dataQualityReport, NngmDataQualityReport::class.java).getIssues() + } catch (_: JacksonException) { + val otherIssue = + Issue(Severity.ERROR, Optional.of("Not parsable data quality report '$dataQualityReport'")) + return listOf(otherIssue) + } } catch (e: Exception) { throw e } } - private data class DataQualityReport( + private data class DipDataQualityReport( val issues: List<Issue> ) @@ -57,7 +61,7 @@ class ReportService(private val jsonMapper: JsonMapper) { val details: Optional<String> = Optional.empty(), val path: Optional<String> = Optional.empty(), ) { - fun getMessage() = message.orElse(details.orElse("No details available")) + fun getMessageText() = message.orElse(details.orElse("No details available")) } @EnumNaming(EnumNamingStrategies.LowerCaseStrategy::class) @@ -67,6 +71,22 @@ class ReportService(private val jsonMapper: JsonMapper) { WARNING("warning"), INFO("info"), } + + private data class NngmDataQualityReport( + val qualityChecksReport: QualityChecksReport, + ) { + fun getIssues(): List<Issue> { + val severity = if (this.qualityChecksReport.importAccepted) Severity.INFO else Severity.ERROR + return this.qualityChecksReport.qualityChecks + .split('|') + .map { Issue(severity, Optional.of(it)) } + } + } + + private data class QualityChecksReport( + val importAccepted: Boolean, + val qualityChecks: String + ) } fun List<Issue>.asRequestStatus(): RequestStatus { @@ -74,6 +94,7 @@ fun List<Issue>.asRequestStatus(): RequestStatus { return when (severity) { Severity.FATAL, Severity.ERROR -> RequestStatus.ERROR + Severity.WARNING -> RequestStatus.WARNING else -> RequestStatus.SUCCESS } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt index d62a503..290df76 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt @@ -55,6 +55,7 @@ data class Request( @Column("submission_accepted") var submissionAccepted: Boolean = false, @Column("followup_count") var followupCount: Int = 0, @Column("expected_followup_count") var expectedFollowupCount: Int = 0, + @Column("destination") var destination: Destination = Destination.UNKNOWN, ) { constructor( uuid: RequestId, diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt index 04e673a..1bcca5e 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt @@ -39,14 +39,12 @@ import org.springframework.web.client.exchange import org.springframework.web.util.UriComponentsBuilder class RestDipMtbFileSender( - private val restTemplate: RestTemplate, + restTemplate: RestTemplate, private val restTargetProperties: RestTargetProperties, - private val retryTemplate: RetryTemplate, - private val reportService: ReportService, -) : MtbFileSender { - private val logger = LoggerFactory.getLogger(RestDipMtbFileSender::class.java) - - fun sendUrl(): String = + retryTemplate: RetryTemplate, + reportService: ReportService, +) : RestMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService) { + override fun sendUrl(): String = UriComponentsBuilder .fromUriString(restTargetProperties.uri.toString()) .pathSegment("mtb") @@ -54,7 +52,7 @@ class RestDipMtbFileSender( .pathSegment("patient-record") .toUriString() - fun deleteUrl(patientId: PatientPseudonym): String = + override fun deleteUrl(patientId: PatientPseudonym): String = UriComponentsBuilder .fromUriString(restTargetProperties.uri.toString()) .pathSegment("mtb") @@ -63,60 +61,9 @@ class RestDipMtbFileSender( .pathSegment(patientId.value) .toUriString() - override fun <T> send(request: MtbFileRequest<T>): MtbFileSender.Response { - try { - return retryTemplate.execute<MtbFileSender.Response, Exception> { - val headers = getHttpHeaders(request) - val entityReq = HttpEntity(request.content, headers) - val response = - restTemplate.exchange<String>(sendUrl(), HttpMethod.POST, entityReq) - if (!response.statusCode.is2xxSuccessful) { - logger.warn("Error sending to remote system: {}", response.body) - return@execute MtbFileSender.Response( - reportService.deserialize(response.body).asRequestStatus(), - "Status-Code: ${response.statusCode.value()}", - ) - } - logger.debug("Sent file via RestMtbFileSender") - return@execute MtbFileSender.Response( - reportService.deserialize(response.body).asRequestStatus(), - response.body.orEmpty(), - ) - } - } catch (e: IllegalArgumentException) { - logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) - } catch (e: RestClientResponseException) { - logger.info(restTargetProperties.uri!!.toString()) - logger.error("Request data not accepted by remote system", e) - return MtbFileSender.Response( - reportService.deserialize(e.responseBodyAsString).asRequestStatus(), - e.responseBodyAsString, - ) - } - return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") - } - - override fun send(request: DeleteRequest): MtbFileSender.Response { - try { - return retryTemplate.execute<MtbFileSender.Response, Exception> { - val headers = getHttpHeaders(request) - val entityReq = HttpEntity(null, headers) - restTemplate.delete(deleteUrl(request.patientId), entityReq, String::class.java) - logger.debug("Sent file via RestMtbFileSender") - return@execute MtbFileSender.Response(RequestStatus.SUCCESS) - } - } catch (e: IllegalArgumentException) { - logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) - } catch (e: RestClientException) { - logger.info(restTargetProperties.uri!!.toString()) - logger.error("Cannot send data to remote system", e) - } - return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") - } - override fun endpoint(): String = this.restTargetProperties.uri.orEmpty() - private fun getHttpHeaders(request: MtbRequest): HttpHeaders { + override fun getHttpHeaders(request: MtbRequest): HttpHeaders { val username = restTargetProperties.username val password = restTargetProperties.password val headers = HttpHeaders() diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt new file mode 100644 index 0000000..c88f606 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt @@ -0,0 +1,107 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken + * Copyright (c) 2025-2026 Paul-Christian Volkmer, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.output + +import dev.dnpm.etl.processor.CustomMediaType +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.config.RestTargetProperties +import dev.dnpm.etl.processor.monitoring.ReportService +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.asRequestStatus +import org.slf4j.LoggerFactory +import org.springframework.http.HttpEntity +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.retry.support.RetryTemplate +import org.springframework.web.client.RestClientException +import org.springframework.web.client.RestClientResponseException +import org.springframework.web.client.RestTemplate +import org.springframework.web.client.exchange +import org.springframework.web.util.UriComponentsBuilder + +abstract class RestMtbFileSender( + private val restTemplate: RestTemplate, + private val restTargetProperties: RestTargetProperties, + private val retryTemplate: RetryTemplate, + private val reportService: ReportService, +) : MtbFileSender { + private val logger = LoggerFactory.getLogger(RestMtbFileSender::class.java) + + abstract fun sendUrl(): String + + abstract fun deleteUrl(patientId: PatientPseudonym): String + + override fun <T> send(request: MtbFileRequest<T>): MtbFileSender.Response { + try { + return retryTemplate.execute<MtbFileSender.Response, Exception> { + val headers = getHttpHeaders(request) + val entityReq = HttpEntity(request.content, headers) + val response = + restTemplate.exchange<String>(sendUrl(), HttpMethod.POST, entityReq) + if (!response.statusCode.is2xxSuccessful) { + logger.warn("Error sending to remote system: {}", response.body) + return@execute MtbFileSender.Response( + reportService.deserialize(response.body).asRequestStatus(), + "Status-Code: ${response.statusCode.value()}", + ) + } + logger.debug("Sent file via ${this::class.java.simpleName}") + return@execute MtbFileSender.Response( + reportService.deserialize(response.body).asRequestStatus(), + response.body.orEmpty(), + ) + } + } catch (_: IllegalArgumentException) { + logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) + } catch (e: RestClientResponseException) { + logger.info(restTargetProperties.uri!!.toString()) + logger.error("Request data not accepted by remote system", e) + return MtbFileSender.Response( + RequestStatus.ERROR, + e.responseBodyAsString, + ) + } + return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") + } + + override fun send(request: DeleteRequest): MtbFileSender.Response { + try { + return retryTemplate.execute<MtbFileSender.Response, Exception> { + val headers = getHttpHeaders(request) + val entityReq = HttpEntity(null, headers) + restTemplate.delete(deleteUrl(request.patientId), entityReq, String::class.java) + logger.debug("Sent file via RestDipMtbFileSender") + return@execute MtbFileSender.Response(RequestStatus.SUCCESS) + } + } catch (_: IllegalArgumentException) { + logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) + } catch (e: RestClientException) { + logger.info(restTargetProperties.uri!!.toString()) + logger.error("Cannot send data to remote system", e) + } + return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") + } + + override fun endpoint(): String = this.restTargetProperties.uri.orEmpty() + + protected abstract fun getHttpHeaders(request: MtbRequest): HttpHeaders +} diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestNngmMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestNngmMtbFileSender.kt new file mode 100644 index 0000000..13722d2 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestNngmMtbFileSender.kt @@ -0,0 +1,59 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken + * Copyright (c) 2025-2026 Paul-Christian Volkmer, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.output + +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.config.RestTargetProperties +import dev.dnpm.etl.processor.monitoring.ReportService +import org.springframework.http.HttpHeaders +import org.springframework.http.MediaType +import org.springframework.retry.support.RetryTemplate +import org.springframework.web.client.RestTemplate +import org.springframework.web.util.UriComponentsBuilder + +class RestNngmMtbFileSender( + restTemplate: RestTemplate, + private val restTargetProperties: RestTargetProperties, + retryTemplate: RetryTemplate, + reportService: ReportService, +) : RestMtbFileSender(restTemplate, restTargetProperties, retryTemplate, reportService) { + override fun sendUrl(): String = + UriComponentsBuilder + .fromUriString(restTargetProperties.uri.toString()) + .toUriString() + + override fun deleteUrl(patientId: PatientPseudonym): String = throw UnsupportedOperationException("Not implemented") + + override fun endpoint(): String = this.restTargetProperties.uri.orEmpty() + + override fun getHttpHeaders(request: MtbRequest): HttpHeaders { + val apiKey = restTargetProperties.apiKey + val headers = HttpHeaders() + headers.contentType = MediaType.APPLICATION_JSON + + if (apiKey.isNullOrBlank()) { + return headers + } + + headers.set(HttpHeaders.AUTHORIZATION, "X-API-KEY $apiKey") + return headers + } +} diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RoutedMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RoutedMtbFileSender.kt new file mode 100644 index 0000000..1eab675 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RoutedMtbFileSender.kt @@ -0,0 +1,31 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken + * Copyright (c) 2023-2026 Paul-Christian Volkmer, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.output + +import dev.dnpm.etl.processor.output.MtbFileSender.Response +import dev.pcvolkmer.mv64e.model.MtbDiagnosis +import dev.pcvolkmer.mv64e.model.PatientRecord + +interface RoutedMtbFileSender { + fun supportsDiagnosis(diagnosis: MtbDiagnosis): Boolean + + fun send(request: MtbFileRequest<PatientRecord>): Response +} diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RoutedRestNngmMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RoutedRestNngmMtbFileSender.kt new file mode 100644 index 0000000..957645e --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RoutedRestNngmMtbFileSender.kt @@ -0,0 +1,117 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken + * Copyright (c) 2025-2026 Paul-Christian Volkmer, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.output + +import dev.dnpm.etl.processor.config.RoutingProperties +import dev.dnpm.etl.processor.monitoring.ReportService +import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.monitoring.asRequestStatus +import dev.pcvolkmer.mv64e.model.MtbDiagnosis +import dev.pcvolkmer.mv64e.model.PatientRecord +import org.slf4j.LoggerFactory +import org.springframework.http.HttpEntity +import org.springframework.http.HttpHeaders +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.retry.support.RetryTemplate +import org.springframework.web.client.RestClientResponseException +import org.springframework.web.client.RestTemplate +import org.springframework.web.client.exchange +import org.springframework.web.util.UriComponentsBuilder + +class RoutedRestNngmMtbFileSender( + private val restTemplate: RestTemplate, + private val routingProperties: RoutingProperties, + private val retryTemplate: RetryTemplate, + private val reportService: ReportService, +) : RoutedMtbFileSender { + private val logger = LoggerFactory.getLogger(RoutedRestNngmMtbFileSender::class.java) + + fun sendUrl(): String = + UriComponentsBuilder + .fromUriString(routingProperties.nngm?.uri.toString()) + .toUriString() + + override fun supportsDiagnosis(diagnosis: MtbDiagnosis): Boolean { + val code = diagnosis.code?.code ?: return false + + if (code.isBlank()) { + return false + } + + this.routingProperties.nngm?.icd10?.forEach { + if (code.startsWith(it)) { + return true + } + } + return false + } + + override fun send(request: MtbFileRequest<PatientRecord>): MtbFileSender.Response { + try { + return retryTemplate.execute<MtbFileSender.Response, Exception> { + val entityReq = HttpEntity(request.content, getHttpHeaders()) + val response = + restTemplate.exchange<String>(sendUrl(), HttpMethod.POST, entityReq) + if (!response.statusCode.is2xxSuccessful) { + logger.warn("Error sending to remote system: {}", response.body) + return@execute MtbFileSender.Response( + reportService.deserialize(response.body).asRequestStatus(), + "Status-Code: ${response.statusCode.value()}", + ) + } + logger.debug("Sent file via RoutedRestNngmMtbFileSender") + return@execute MtbFileSender.Response( + reportService.deserialize(response.body).asRequestStatus(), + response.body.orEmpty(), + ) + } + } catch (_: IllegalArgumentException) { + logger.error("Not a valid URI to export to: '{}'", routingProperties.nngm?.uri!!) + } catch (e: RestClientResponseException) { + logger.info(routingProperties.nngm?.uri!!) + logger.error("Request data not accepted by remote system", e) + return MtbFileSender.Response( + reportService.deserialize(e.responseBodyAsString).asRequestStatus(), + e.responseBodyAsString, + ) + } + return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") + } + + fun endpoint(): String = + this.routingProperties.nngm + ?.uri + .orEmpty() + + private fun getHttpHeaders(): HttpHeaders { + val apiKey = routingProperties.nngm?.apiKey + val headers = HttpHeaders() + headers.contentType = MediaType.APPLICATION_JSON + + if (apiKey.isNullOrBlank()) { + return headers + } + + headers.set(HttpHeaders.AUTHORIZATION, "X-API-KEY $apiKey") + return headers + } +} diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 320f828..6ce9d14 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -24,10 +24,7 @@ import dev.dnpm.etl.processor.* import dev.dnpm.etl.processor.config.AppConfigProperties import dev.dnpm.etl.processor.consent.TtpConsentStatus import dev.dnpm.etl.processor.monitoring.* -import dev.dnpm.etl.processor.output.DeleteRequest -import dev.dnpm.etl.processor.output.DnpmV2MtbFileRequest -import dev.dnpm.etl.processor.output.MtbFileRequest -import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.output.* import dev.dnpm.etl.processor.pseudonym.PseudonymizeService import dev.dnpm.etl.processor.pseudonym.addGenomDeTan import dev.dnpm.etl.processor.pseudonym.anonymizeContentWith @@ -49,6 +46,7 @@ class RequestProcessor( private val pseudonymizeService: PseudonymizeService, private val transformationService: TransformationService, private val sender: MtbFileSender, + private val switchedSenders: List<RoutedMtbFileSender>, private val requestService: RequestService, private val jsonMapper: JsonMapper, private val applicationEventPublisher: ApplicationEventPublisher, @@ -111,6 +109,18 @@ class RequestProcessor( } } + var destination = when (sender) { + is RestDipMtbFileSender -> Destination.DNPM_DIP + is RestNngmMtbFileSender -> Destination.NNGM_API + is KafkaMtbFileSender -> Destination.KAFKA + else -> Destination.UNKNOWN + } + + val routedSender = selectRoutedSender(request) + if (routedSender.isPresent && routedSender.get() is RoutedRestNngmMtbFileSender) { + destination = Destination.NNGM_API + } + val maxFollowUpCount = this.requestService.allRequestsByPatientPseudonym(request.patientPseudonym()) .maxByOrNull { it.followupCount } ?.followupCount ?: -1 @@ -132,6 +142,7 @@ class RequestProcessor( tan = Tan(request.content.metadata?.transferTAN.orEmpty()), followupCount = maxFollowUpCount, expectedFollowupCount = request.content.followUps?.size ?: 0, + destination = destination, ) ) // Exit - no further processing @@ -172,6 +183,7 @@ class RequestProcessor( tan = Tan(request.content.metadata?.transferTAN.orEmpty()), followupCount = maxFollowUpCount, expectedFollowupCount = request.content.followUps?.size ?: 0, + destination = destination, ) ) @@ -182,7 +194,11 @@ class RequestProcessor( return } - val responseStatus = sender.send(request) + val responseStatus = if (!routedSender.isEmpty) { + routedSender.get().send(request) + } else { + sender.send(request) + } applicationEventPublisher.publishEvent( ResponseEvent( @@ -199,6 +215,28 @@ class RequestProcessor( ) } + /* + * Selects MtbFileSender based on latest diagnosis ICD10 code + */ + private fun selectRoutedSender(request: MtbFileRequest<PatientRecord>): Optional<RoutedMtbFileSender> { + + when (request) { + is DnpmV2MtbFileRequest -> { + val latestDiagnosis = request.content.diagnoses.maxBy { it.recordedOn } ?: return Optional.empty() + + this.switchedSenders.forEach { sender -> + if (sender.supportsDiagnosis(latestDiagnosis)) { + return Optional.of(sender) + } + } + + } + } + + // Default + return Optional.empty() + } + private fun hasFollowUpAfterLastSuccessfulSubmission(request: DnpmV2MtbFileRequest): Boolean { val lastSuccessfulSubmission = this.requestService.allRequestsByPatientPseudonym(request.patientPseudonym()) .sortedBy { it.processedAt } diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 6c1e51f..77bf157 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -119,6 +119,12 @@ app: # input-topic: test_input # output-topic: test_output # output-response-topic: test_response + switch: + nngm: + uri: https://mvh-test.medicalsyn.com/api/v1.0/mvh-import/import-dnpm + api-key: a12b34decacdefg1213 + icd10: + - C34 security: admin-user: admin admin-password: "{noop}very-secret" @@ -126,4 +132,4 @@ app: server: port: 8000 - shutdown: immediate
\ No newline at end of file + shutdown: immediate diff --git a/src/main/resources/db/migration/mariadb/V0_18_0_1__Destination.sql b/src/main/resources/db/migration/mariadb/V0_18_0_1__Destination.sql new file mode 100644 index 0000000..a65c016 --- /dev/null +++ b/src/main/resources/db/migration/mariadb/V0_18_0_1__Destination.sql @@ -0,0 +1,2 @@ +ALTER TABLE request ADD COLUMN destination varchar(16) DEFAULT 'UNKNOWN'; +UPDATE request SET destination = 'UNKNOWN'; diff --git a/src/main/resources/db/migration/postgresql/V0_18_0_1__Destination.sql b/src/main/resources/db/migration/postgresql/V0_18_0_1__Destination.sql new file mode 100644 index 0000000..a65c016 --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V0_18_0_1__Destination.sql @@ -0,0 +1,2 @@ +ALTER TABLE request ADD COLUMN destination varchar(16) DEFAULT 'UNKNOWN'; +UPDATE request SET destination = 'UNKNOWN'; diff --git a/src/main/resources/templates/fragments.html b/src/main/resources/templates/fragments.html index ea83ed2..1a7a257 100644 --- a/src/main/resources/templates/fragments.html +++ b/src/main/resources/templates/fragments.html @@ -54,7 +54,10 @@ <div th:if="${request.type.value == 'delete'}"> <span>🗑 Löschanfrage vom </span><time th:datetime="${request.processedAt}">[[ ${request.processedAt} ]]</time> </div> - <div sec:authorize="hasRole('USER') or hasRole('ADMIN')" th:if="${postInitialSubmissionBlock}"> + <div class="destination" th:if="${request.destination.value == 'kafka'}">Kafka Broker</div> + <div class="destination" th:if="${request.destination.value == 'dnpm_dip'}">DNPM:DIP</div> + <div class="destination" th:if="${request.destination.value == 'nngm_api'}">nNGM-API</div> + <div class="accepted" sec:authorize="hasRole('USER') or hasRole('ADMIN')" th:if="${postInitialSubmissionBlock}"> <button class="btn btn-green" hx-swap="outerHTML" th:hx-target="${'#request_' + request.uuid}" th:hx-delete="@{/submission/{requestId}/accepted(requestId=${request.uuid})}" th:if="${ request.submissionType.value == 'initial' and (request.status.value == 'success' or request.status.value == 'warning') and request.submissionAccepted == true}" title="Doch keine Meldebestätigung - jetzt blockieren"> 🔓 Bestätigte Initialmeldung </button> diff --git a/src/main/resources/templates/report.html b/src/main/resources/templates/report.html index bdc7457..b739bbf 100644 --- a/src/main/resources/templates/report.html +++ b/src/main/resources/templates/report.html @@ -34,7 +34,7 @@ <td th:if="${issue.severity.value == 'error'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td> <td th:if="${issue.severity.value == 'fatal'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td> <td> - <div class="issue-message">[[ ${issue.getMessage()} ]]</div> + <div class="issue-message">[[ ${issue.getMessageText()} ]]</div> <div class="issue-path" th:if="${issue.path.isPresent()}">[[ ${issue.path.get()} ]]</div> <div class="issue-path" th:if="${issue.path.isEmpty()}"><i>Keine Angabe</i></div> </td> diff --git a/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt index 62ece6c..c5cc839 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt @@ -56,6 +56,7 @@ class ConnectionCheckServiceTest { "http://localhost/api", "user", "password", + null, ) this.sink = Sinks.many().multicast().onBackpressureBuffer() this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt index e9c3107..c45dd95 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/ReportServiceTest.kt @@ -60,13 +60,13 @@ class ReportServiceTest { assertThat(actual).hasSize(4) assertThat(actual[0].severity).isEqualTo(ReportService.Severity.FATAL) - assertThat(actual[0].getMessage()).isEqualTo("Fatal Message") + assertThat(actual[0].getMessageText()).isEqualTo("Fatal Message") assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR) - assertThat(actual[1].getMessage()).isEqualTo("Error Message") + assertThat(actual[1].getMessageText()).isEqualTo("Error Message") assertThat(actual[2].severity).isEqualTo(ReportService.Severity.WARNING) - assertThat(actual[2].getMessage()).isEqualTo("Warning Message") + assertThat(actual[2].getMessageText()).isEqualTo("Warning Message") assertThat(actual[3].severity).isEqualTo(ReportService.Severity.INFO) - assertThat(actual[3].getMessage()).isEqualTo("Info Message") + assertThat(actual[3].getMessageText()).isEqualTo("Info Message") assertThat(actual.asRequestStatus()).isEqualTo(RequestStatus.ERROR) } @@ -87,7 +87,7 @@ class ReportServiceTest { assertThat(actual).hasSize(1) assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR) - assertThat(actual[0].getMessage()).isEqualTo("/specimens(0)/type/code: error.path.missing") + assertThat(actual[0].getMessageText()).isEqualTo("/specimens(0)/type/code: error.path.missing") assertThat(actual.asRequestStatus()).isEqualTo(RequestStatus.ERROR) } @@ -110,7 +110,7 @@ class ReportServiceTest { assertThat(actual).hasSize(1) assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR) - assertThat(actual[0].getMessage()).isEqualTo("Not parsable data quality report '$invalidResponse'") + assertThat(actual[0].getMessageText()).isEqualTo("Not parsable data quality report '$invalidResponse'") } companion object { diff --git a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt index 3edb480..8433e93 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/services/RequestProcessorTest.kt @@ -29,11 +29,15 @@ import dev.dnpm.etl.processor.monitoring.RequestType import dev.dnpm.etl.processor.monitoring.SubmissionType import dev.dnpm.etl.processor.output.DeleteRequest import dev.dnpm.etl.processor.output.DnpmV2MtbFileRequest +import dev.dnpm.etl.processor.output.MtbFileRequest import dev.dnpm.etl.processor.output.MtbFileSender +import dev.dnpm.etl.processor.output.RoutedMtbFileSender import dev.dnpm.etl.processor.pseudonym.PseudonymizeService +import dev.pcvolkmer.mv64e.model.Coding import dev.pcvolkmer.mv64e.model.ConsentProvisionType import dev.pcvolkmer.mv64e.model.FollowUp import dev.pcvolkmer.mv64e.model.ModelProjectConsentPurpose +import dev.pcvolkmer.mv64e.model.MtbDiagnosis import dev.pcvolkmer.mv64e.model.MtbEpisodeOfCare import dev.pcvolkmer.mv64e.model.MvhMetadata import dev.pcvolkmer.mv64e.model.MvhMetadataModelProjectConsent @@ -66,6 +70,7 @@ class RequestProcessorTest { private lateinit var pseudonymizeService: PseudonymizeService private lateinit var transformationService: TransformationService private lateinit var sender: MtbFileSender + private lateinit var routedMtbFileSender: RoutedMtbFileSender private lateinit var requestService: RequestService private lateinit var applicationEventPublisher: ApplicationEventPublisher private lateinit var appConfigProperties: AppConfigProperties @@ -78,6 +83,7 @@ class RequestProcessorTest { @Mock pseudonymizeService: PseudonymizeService, @Mock transformationService: TransformationService, @Mock sender: MtbFileSender, + @Mock routedMtbFileSender: RoutedMtbFileSender, @Mock requestService: RequestService, @Mock applicationEventPublisher: ApplicationEventPublisher, @Mock consentProcessor: ConsentProcessor, @@ -85,6 +91,7 @@ class RequestProcessorTest { this.pseudonymizeService = pseudonymizeService this.transformationService = transformationService this.sender = sender + this.routedMtbFileSender = routedMtbFileSender this.requestService = requestService this.applicationEventPublisher = applicationEventPublisher this.appConfigProperties = AppConfigProperties() @@ -96,6 +103,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -112,7 +120,7 @@ class RequestProcessorTest { randomRequestId(), PatientPseudonym("TEST_12345678901"), PatientId("P1"), - Fingerprint("syehahte4oyqd5m2rbd5imnth4rx6md32g2msb7sztnayxoc4kaq"), + Fingerprint("tyfimuuc7okhnoholo6jx5o5os3n7haelezphr4477uxbzlotvra"), RequestType.MTB_FILE, SubmissionType.TEST, RequestStatus.SUCCESS, @@ -138,6 +146,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .episodesOfCare( listOf( MtbEpisodeOfCare.builder() @@ -169,7 +183,7 @@ class RequestProcessorTest { randomRequestId(), PatientPseudonym("TEST_12345678901"), PatientId("P1"), - Fingerprint("me6ockoru4boi4ypghfia5myfqtuffwlbszwhtop2rtltb3ycjva"), + Fingerprint("q3qfwecsw3tcmdsg4kg5ki4ob3qw3warnwi4qc54nptjgzft53nq"), RequestType.MTB_FILE, SubmissionType.TEST, RequestStatus.SUCCESS, @@ -195,6 +209,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .episodesOfCare( listOf( MtbEpisodeOfCare.builder() @@ -256,6 +276,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .episodesOfCare( listOf( MtbEpisodeOfCare.builder() @@ -321,6 +347,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata( MvhMetadata.builder() .modelProjectConsent( @@ -423,6 +455,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -433,6 +466,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata()) .episodesOfCare( listOf( @@ -516,6 +555,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata()) .episodesOfCare( listOf( @@ -596,6 +641,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .episodesOfCare( listOf( MtbEpisodeOfCare.builder() @@ -647,6 +698,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -657,6 +709,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2023-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata()) .episodesOfCare( listOf( @@ -717,6 +775,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -789,6 +848,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -799,6 +859,7 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses(listOf(MtbDiagnosis().code(Coding().code("C80.9")).recordedOn(Date()))) .episodesOfCare( listOf( MtbEpisodeOfCare.builder() @@ -894,6 +955,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -904,6 +966,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1006,6 +1074,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1016,6 +1085,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1105,6 +1180,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1115,6 +1191,7 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses(listOf(MtbDiagnosis().code(Coding().code("C80.9")))) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1187,6 +1264,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1262,6 +1340,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1273,6 +1352,12 @@ class RequestProcessorTest { PatientRecord.builder() .patient(Patient.builder().id("123").build()) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .episodesOfCare( listOf( MtbEpisodeOfCare.builder() @@ -1384,6 +1469,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1394,6 +1480,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1492,6 +1584,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1502,6 +1595,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1616,6 +1715,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1626,6 +1726,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1754,6 +1860,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1764,6 +1871,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1883,6 +1996,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1893,6 +2007,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -1979,6 +2099,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -1989,6 +2110,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -2089,6 +2216,7 @@ class RequestProcessorTest { pseudonymizeService, transformationService, sender, + listOf(routedMtbFileSender), requestService, jsonMapper, applicationEventPublisher, @@ -2099,6 +2227,12 @@ class RequestProcessorTest { val mtbFile = PatientRecord.builder() .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) .metadata(MvhMetadata.builder().type(MvhSubmissionType.INITIAL).build()) .episodesOfCare( listOf( @@ -2135,6 +2269,206 @@ class RequestProcessorTest { } } + @Nested + inner class SwitchedSenders { + + private lateinit var pseudonymizeService: PseudonymizeService + private lateinit var transformationService: TransformationService + private lateinit var sender: MtbFileSender + private lateinit var routedMtbFileSender: RoutedMtbFileSender + private lateinit var requestService: RequestService + private lateinit var applicationEventPublisher: ApplicationEventPublisher + private lateinit var appConfigProperties: AppConfigProperties + private lateinit var consentProcessor: ConsentProcessor + private lateinit var requestProcessor: RequestProcessor + private lateinit var jsonMapper: JsonMapper + + @BeforeEach + fun setup( + @Mock pseudonymizeService: PseudonymizeService, + @Mock transformationService: TransformationService, + @Mock sender: MtbFileSender, + @Mock routedMtbFileSender: RoutedMtbFileSender, + @Mock requestService: RequestService, + @Mock applicationEventPublisher: ApplicationEventPublisher, + @Mock consentProcessor: ConsentProcessor, + ) { + this.pseudonymizeService = pseudonymizeService + this.transformationService = transformationService + this.sender = sender + this.routedMtbFileSender = routedMtbFileSender + this.requestService = requestService + this.applicationEventPublisher = applicationEventPublisher + this.appConfigProperties = AppConfigProperties() + this.consentProcessor = consentProcessor + this.jsonMapper = JsonMapper() + + requestProcessor = + RequestProcessor( + pseudonymizeService, + transformationService, + sender, + listOf(routedMtbFileSender), + requestService, + jsonMapper, + applicationEventPublisher, + appConfigProperties, + consentProcessor, + ) + + doAnswer { + val diagnosis = it.arguments[0] as MtbDiagnosis + diagnosis.code.code.equals("C34.9") + }.whenever(routedMtbFileSender).supportsDiagnosis(any<MtbDiagnosis>()) + } + + @Test + fun testShouldSendMtbFileUsingSwitchedSenderAndSendSuccessEvent() { + doAnswer { + Request( + 1L, + randomRequestId(), + PatientPseudonym("TEST_12345678901"), + PatientId("P1"), + Fingerprint("different"), + RequestType.MTB_FILE, + SubmissionType.TEST, + RequestStatus.SUCCESS, + Tan.empty(), + Instant.parse("2023-08-08T02:00:00Z"), + ) + } + .whenever(requestService) + .lastMtbFileRequestForPatientPseudonym(anyValueClass()) + + doAnswer { false } + .whenever(requestService) + .isLastRequestWithKnownStatusDeletion(anyValueClass()) + + doAnswer { MtbFileSender.Response(status = RequestStatus.SUCCESS) } + .whenever(routedMtbFileSender) + .send(any<DnpmV2MtbFileRequest>()) + + doAnswer { it.arguments[0] as String } + .whenever(pseudonymizeService) + .patientPseudonym(anyValueClass()) + + doAnswer { it.arguments[0] }.whenever(transformationService).transform(any<PatientRecord>()) + + whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true) + + val mtbFile = + PatientRecord.builder() + .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C34.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) + .episodesOfCare( + listOf( + MtbEpisodeOfCare.builder() + .id("1") + .patient(Reference.builder().id("123").build()) + .period( + PeriodDate.builder() + .start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))) + .build() + ) + .build() + ) + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + // Do not use DNPM:DIP sender + verify(sender, never()).send(any<MtbFileRequest<PatientRecord>>()) + // Use switched sender instead + verify(routedMtbFileSender, times(1)).send(any<MtbFileRequest<PatientRecord>>()) + + val eventCaptor = argumentCaptor<ResponseEvent>() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + } + + @Test + fun testShouldNotSendMtbFileUsingSwitchedSenderAndSendSuccessEvent() { + doAnswer { + Request( + 1L, + randomRequestId(), + PatientPseudonym("TEST_12345678901"), + PatientId("P1"), + Fingerprint("different"), + RequestType.MTB_FILE, + SubmissionType.TEST, + RequestStatus.SUCCESS, + Tan.empty(), + Instant.parse("2023-08-08T02:00:00Z"), + ) + } + .whenever(requestService) + .lastMtbFileRequestForPatientPseudonym(anyValueClass()) + + doAnswer { false } + .whenever(requestService) + .isLastRequestWithKnownStatusDeletion(anyValueClass()) + + doAnswer { MtbFileSender.Response(status = RequestStatus.SUCCESS) } + .whenever(sender) + .send(any<DnpmV2MtbFileRequest>()) + + doAnswer { it.arguments[0] as String } + .whenever(pseudonymizeService) + .patientPseudonym(anyValueClass()) + + doAnswer { it.arguments[0] }.whenever(transformationService).transform(any<PatientRecord>()) + + whenever(consentProcessor.consentGatedCheckAndTryEmbedding(any())).thenReturn(true) + + val mtbFile = + PatientRecord.builder() + .patient(Patient.builder().id("123").build()) + .diagnoses( + listOf( + MtbDiagnosis().code(Coding().code("C80.9")) + .recordedOn(Date.from(Instant.parse("2020-01-01T02:00:00.00Z"))) + ) + ) + .episodesOfCare( + listOf( + MtbEpisodeOfCare.builder() + .id("1") + .patient(Reference.builder().id("123").build()) + .period( + PeriodDate.builder() + .start(Date.from(Instant.parse("2021-01-01T00:00:00.00Z"))) + .build() + ) + .build() + ) + ) + .build() + + this.requestProcessor.processMtbFile(mtbFile) + + // Use DNPM:DIP sender + verify(sender, times(1)).send(any<MtbFileRequest<PatientRecord>>()) + // Not switched sender + verify(routedMtbFileSender, never()).send(any<MtbFileRequest<PatientRecord>>()) + + val eventCaptor = argumentCaptor<ResponseEvent>() + verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture()) + assertThat(eventCaptor.firstValue).isNotNull + assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS) + } + + } + + @Test fun shouldCatchExceptionsWhenProcessingMtbFileAndSaveError() { val invalidMtbFile = PatientRecord.builder().build() diff --git a/src/web/style.css b/src/web/style.css index 6ae77b0..5c88bd6 100644 --- a/src/web/style.css +++ b/src/web/style.css @@ -521,7 +521,25 @@ table.config-table td > button:last-of-type { background: #ffdddd; } -.card > .card-sub-header > *:nth-child(2) { +.card > .card-sub-header > .destination { + background: #ccc; + border: 1px solid #333; + border-radius: 3px; + padding-right: 4px; + font-size: small; + margin-left: 1rem; +} + +.card > .card-sub-header > .destination:before { + content: "Ziel"; + border-right: 1px solid #333; + padding: 0 4px; + margin-right: 4px; + color: #ccc; + background: #333; +} + +.card > .card-sub-header > .accepted { margin: 0 0 0 auto; } |