From 0defbb05bac3f585e75d72c02c9168d1c2de1834 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Wed, 26 Jul 2023 12:28:03 +0200 Subject: fix: add message key value while producing messages to kafka topic --- .../kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/main/kotlin/dev') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index 374c0af..9520caa 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -25,15 +25,16 @@ import org.slf4j.LoggerFactory import org.springframework.kafka.core.KafkaTemplate class KafkaMtbFileSender( - private val kafkaTemplate: KafkaTemplate, - private val objectMapper: ObjectMapper + private val kafkaTemplate: KafkaTemplate, + private val objectMapper: ObjectMapper ) : MtbFileSender { private val logger = LoggerFactory.getLogger(KafkaMtbFileSender::class.java) override fun send(mtbFile: MtbFile): MtbFileSender.Response { return try { - kafkaTemplate.sendDefault(objectMapper.writeValueAsString(mtbFile)) + kafkaTemplate.sendDefault(String.format("{\"pid\": %s, \"eid\": %s}", mtbFile.patient.id, + mtbFile.episode.id), objectMapper.writeValueAsString(mtbFile)) logger.debug("Sent file via KafkaMtbFileSender") MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } catch (e: Exception) { -- cgit v1.2.3 From 4c0a444725f1107e44db41e509587b969a33ea4b Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Thu, 27 Jul 2023 11:23:47 +0200 Subject: fix: fix kafka key values + add missing component attribute --- src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/main/kotlin/dev') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index 9520caa..f83a2ab 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -23,7 +23,9 @@ import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile import org.slf4j.LoggerFactory import org.springframework.kafka.core.KafkaTemplate +import org.springframework.stereotype.Component +@Component class KafkaMtbFileSender( private val kafkaTemplate: KafkaTemplate, private val objectMapper: ObjectMapper @@ -33,7 +35,7 @@ class KafkaMtbFileSender( override fun send(mtbFile: MtbFile): MtbFileSender.Response { return try { - kafkaTemplate.sendDefault(String.format("{\"pid\": %s, \"eid\": %s}", mtbFile.patient.id, + kafkaTemplate.sendDefault(String.format("{\"pid\": \"%s\", \"eid\": \"%s\"}", mtbFile.patient.id, mtbFile.episode.id), objectMapper.writeValueAsString(mtbFile)) logger.debug("Sent file via KafkaMtbFileSender") MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) -- cgit v1.2.3 From cf8c5a86928da3a109e700a5221b5aa26cfe4aa7 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Thu, 27 Jul 2023 11:49:31 +0200 Subject: fix: wait for kafka to accept message and return success than --- .../dnpm/etl/processor/output/KafkaMtbFileSender.kt | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'src/main/kotlin/dev') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index f83a2ab..18faaf9 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -27,18 +27,27 @@ import org.springframework.stereotype.Component @Component class KafkaMtbFileSender( - private val kafkaTemplate: KafkaTemplate, - private val objectMapper: ObjectMapper + private val kafkaTemplate: KafkaTemplate, + private val objectMapper: ObjectMapper ) : MtbFileSender { private val logger = LoggerFactory.getLogger(KafkaMtbFileSender::class.java) override fun send(mtbFile: MtbFile): MtbFileSender.Response { return try { - kafkaTemplate.sendDefault(String.format("{\"pid\": \"%s\", \"eid\": \"%s\"}", mtbFile.patient.id, - mtbFile.episode.id), objectMapper.writeValueAsString(mtbFile)) - logger.debug("Sent file via KafkaMtbFileSender") - MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) + var result = kafkaTemplate.sendDefault( + String.format( + "{\"pid\": \"%s\", \"eid\": \"%s\"}", mtbFile.patient.id, + mtbFile.episode.id + ), objectMapper.writeValueAsString(mtbFile) + ) + if (result.get() != null) { + logger.debug("Sent file via KafkaMtbFileSender"); + MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS); + } else { + MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) + } + } catch (e: Exception) { logger.error("An error occured sending to kafka", e) MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) -- cgit v1.2.3 From 79d83ef04a31aa227a203aaa9345b6e12a4a8620 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Thu, 27 Jul 2023 12:18:17 +0200 Subject: fix: if no prefix is set, `_` is also not added as prefix to patient pseudonym. --- .../kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/main/kotlin/dev') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt index 364e296..21e0f9a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt @@ -19,6 +19,7 @@ package dev.dnpm.etl.processor.pseudonym +import ca.uhn.fhir.util.StringUtil import de.ukw.ccc.bwhc.dto.MtbFile import dev.dnpm.etl.processor.config.PseudonymizeConfigProperties @@ -28,7 +29,9 @@ class PseudonymizeService( ) { fun pseudonymize(mtbFile: MtbFile): MtbFile { - val patientPseudonym = "${configProperties.prefix}_${generator.generate(mtbFile.patient.id)}" + val patientPseudonym: String = if(configProperties.prefix.isNotEmpty()) + "${configProperties.prefix}_${generator.generate(mtbFile.patient.id)}" + else generator.generate(mtbFile.patient.id) mtbFile.episode.patient = patientPseudonym mtbFile.carePlans.forEach { it.patient = patientPseudonym } -- cgit v1.2.3 From 90c5b81c2b6412a7bf01fa872ca84b5bb13ccb6a Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Thu, 27 Jul 2023 13:01:58 +0200 Subject: feat: patient pid may be replaced with gPAS pseudonym, now. --- .../dev/dnpm/etl/processor/config/AppConfigProperties.kt | 6 +++++- .../kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt | 10 ++-------- .../kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt | 8 ++++---- 3 files changed, 11 insertions(+), 13 deletions(-) (limited to 'src/main/kotlin/dev') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 1575c39..0f257e8 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -45,7 +45,11 @@ data class PseudonymizeConfigProperties( @ConfigurationProperties(GPasConfigProperties.NAME) data class GPasConfigProperties( val uri: String?, - val target: String = "etl-processor" + val target: String = "etl-processor", + val username: String?, + val password: String?, + val sslCaLocation: String?, + ) { companion object { const val NAME = "app.pseudonymize.gpas" diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 5c3add2..50d848a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -24,20 +24,14 @@ import dev.dnpm.etl.processor.monitoring.ReportService import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.output.RestMtbFileSender -import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator -import dev.dnpm.etl.processor.pseudonym.Generator -import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator -import dev.dnpm.etl.processor.pseudonym.PseudonymizeService -import org.reactivestreams.Publisher +import dev.dnpm.etl.processor.pseudonym.* import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.kafka.core.KafkaTemplate -import reactor.core.publisher.Flux import reactor.core.publisher.Sinks import java.net.URI -import java.time.Duration @Configuration @EnableConfigurationProperties( @@ -54,7 +48,7 @@ class AppConfiguration { @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS") @Bean fun gpasPseudonymGenerator(configProperties: GPasConfigProperties): Generator { - return GpasPseudonymGenerator(URI.create(configProperties.uri!!), configProperties.target) + return GpasPseudonymGenerator(configProperties) } @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "BUILDIN", matchIfMissing = true) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index 18faaf9..4bb0206 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -35,21 +35,21 @@ class KafkaMtbFileSender( override fun send(mtbFile: MtbFile): MtbFileSender.Response { return try { - var result = kafkaTemplate.sendDefault( + val result = kafkaTemplate.sendDefault( String.format( "{\"pid\": \"%s\", \"eid\": \"%s\"}", mtbFile.patient.id, mtbFile.episode.id ), objectMapper.writeValueAsString(mtbFile) ) if (result.get() != null) { - logger.debug("Sent file via KafkaMtbFileSender"); - MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS); + logger.debug("Sent file via KafkaMtbFileSender") + MtbFileSender.Response(MtbFileSender.ResponseStatus.SUCCESS) } else { MtbFileSender.Response(MtbFileSender.ResponseStatus.ERROR) } } catch (e: Exception) { - logger.error("An error occured sending to kafka", e) + logger.error("An error occurred sending to kafka", e) MtbFileSender.Response(MtbFileSender.ResponseStatus.UNKNOWN) } } -- cgit v1.2.3 From aa5b35970af967e972edd6ed435d75184b36d4d5 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 27 Jul 2023 23:16:19 +0200 Subject: Code polish for PR * Do not use component creation using annotation on classes since components will be created within AppConfiguration config class * Patient PseudonymizeService intentionally uses prefix. If no prefix is configured, the default value "UNKNOWN" will be used to prevent conflicts with other organizations and to show the source organization. --- src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt | 6 ++++-- src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt | 5 ++--- .../kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt | 5 +---- 3 files changed, 7 insertions(+), 9 deletions(-) (limited to 'src/main/kotlin/dev') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 50d848a..c677f2b 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -24,14 +24,16 @@ import dev.dnpm.etl.processor.monitoring.ReportService import dev.dnpm.etl.processor.output.KafkaMtbFileSender import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.output.RestMtbFileSender -import dev.dnpm.etl.processor.pseudonym.* +import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator +import dev.dnpm.etl.processor.pseudonym.Generator +import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator +import dev.dnpm.etl.processor.pseudonym.PseudonymizeService import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.kafka.core.KafkaTemplate import reactor.core.publisher.Sinks -import java.net.URI @Configuration @EnableConfigurationProperties( diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index 4bb0206..9867deb 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -23,9 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile import org.slf4j.LoggerFactory import org.springframework.kafka.core.KafkaTemplate -import org.springframework.stereotype.Component -@Component class KafkaMtbFileSender( private val kafkaTemplate: KafkaTemplate, private val objectMapper: ObjectMapper @@ -37,7 +35,8 @@ class KafkaMtbFileSender( return try { val result = kafkaTemplate.sendDefault( String.format( - "{\"pid\": \"%s\", \"eid\": \"%s\"}", mtbFile.patient.id, + "{\"pid\": \"%s\", \"eid\": \"%s\"}", + mtbFile.patient.id, mtbFile.episode.id ), objectMapper.writeValueAsString(mtbFile) ) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt index 21e0f9a..364e296 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt @@ -19,7 +19,6 @@ package dev.dnpm.etl.processor.pseudonym -import ca.uhn.fhir.util.StringUtil import de.ukw.ccc.bwhc.dto.MtbFile import dev.dnpm.etl.processor.config.PseudonymizeConfigProperties @@ -29,9 +28,7 @@ class PseudonymizeService( ) { fun pseudonymize(mtbFile: MtbFile): MtbFile { - val patientPseudonym: String = if(configProperties.prefix.isNotEmpty()) - "${configProperties.prefix}_${generator.generate(mtbFile.patient.id)}" - else generator.generate(mtbFile.patient.id) + val patientPseudonym = "${configProperties.prefix}_${generator.generate(mtbFile.patient.id)}" mtbFile.episode.patient = patientPseudonym mtbFile.carePlans.forEach { it.patient = patientPseudonym } -- cgit v1.2.3