From c8a9b8618f6df8294090f809b39211fda9da25cb Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 20 Oct 2025 19:30:49 +0200 Subject: chore: bump version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index e9a2804..ff73da5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -13,7 +13,7 @@ plugins { } group = "dev.dnpm" -version = "0.11.1" +version = "0.11.2" var versions = mapOf( "mtb-dto" to "0.1.0-SNAPSHOT", -- cgit v1.2.3 From ca165a420ae739dab57633e01713c9e6fb3d5271 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 24 Oct 2025 13:42:49 +0200 Subject: feat: add logging for connection checks --- .../processor/monitoring/ConnectionCheckService.kt | 41 ++++++++++++++++------ 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt index 37cd5de..1343ee0 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt @@ -26,6 +26,7 @@ import dev.dnpm.etl.processor.config.RestTargetProperties import jakarta.annotation.PostConstruct import org.apache.kafka.clients.consumer.Consumer import org.apache.kafka.common.errors.TimeoutException +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Qualifier import org.springframework.http.* import org.springframework.scheduling.annotation.Scheduled @@ -83,6 +84,7 @@ class KafkaConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many ) : OutputConnectionCheckService { + private val logger = LoggerFactory.getLogger(javaClass) private var result = ConnectionCheckResult.KafkaConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @@ -95,7 +97,8 @@ class KafkaConnectionCheckService( Instant.now(), if (result.available == available) { result.lastChange } else { Instant.now() } ) - } catch (_: TimeoutException) { + } catch (ex: TimeoutException) { + logger.error("Connection-Timeout error: {}", ex.message) ConnectionCheckResult.KafkaConnectionCheckResult( false, Instant.now(), @@ -121,27 +124,33 @@ class RestConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many ) : OutputConnectionCheckService { + private val logger = LoggerFactory.getLogger(javaClass) private var result = ConnectionCheckResult.RestConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { result = try { - val available = restTemplate.getForEntity( + val statusCode = restTemplate.getForEntity( UriComponentsBuilder.fromUriString(restTargetProperties.uri.toString()) .pathSegment("mtb") .pathSegment("kaplan-meier") .pathSegment("config") .toUriString(), String::class.java - ).statusCode == HttpStatus.OK + ).statusCode + val available = statusCode == HttpStatus.OK + if (available.not()) { + logger.error("Invalid response code {}, expected HTTP status 200", statusCode) + } ConnectionCheckResult.RestConnectionCheckResult( available, Instant.now(), if (result.available == available) { result.lastChange } else { Instant.now() } ) - } catch (_: Exception) { + } catch (ex: Exception) { + logger.error("Connection-Check error: {}", ex.message) ConnectionCheckResult.RestConnectionCheckResult( false, Instant.now(), @@ -166,6 +175,7 @@ class GPasConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many ) : ConnectionCheckService { + private val logger = LoggerFactory.getLogger(javaClass) private var result = ConnectionCheckResult.GPasConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @@ -182,19 +192,24 @@ class GPasConnectionCheckService( headers.setBasicAuth(gPasConfigProperties.username, gPasConfigProperties.password) } - val available = restTemplate.exchange( + val statusCode = restTemplate.exchange( uri, HttpMethod.GET, HttpEntity(headers), Void::class.java - ).statusCode == HttpStatus.OK + ).statusCode + val available = statusCode == HttpStatus.OK + if (available.not()) { + logger.error("Invalid response code {}, expected HTTP status 200", statusCode) + } ConnectionCheckResult.GPasConnectionCheckResult( available, Instant.now(), if (result.available == available) { result.lastChange } else { Instant.now() } ) - } catch (_: Exception) { + } catch (ex: Exception) { + logger.error("Connection-Check error: {}", ex.message) ConnectionCheckResult.GPasConnectionCheckResult( false, Instant.now(), @@ -219,6 +234,7 @@ class GIcsConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many ) : ConnectionCheckService { + private val logger = LoggerFactory.getLogger(javaClass) private var result = ConnectionCheckResult.GIcsConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @@ -235,19 +251,24 @@ class GIcsConnectionCheckService( headers.setBasicAuth(gIcsConfigProperties.username, gIcsConfigProperties.password) } - val available = restTemplate.exchange( + val statusCode = restTemplate.exchange( uri, HttpMethod.GET, HttpEntity(headers), Void::class.java - ).statusCode == HttpStatus.OK + ).statusCode + val available = statusCode == HttpStatus.OK + if (available.not()) { + logger.error("Invalid response code {}, expected HTTP status 200", statusCode) + } ConnectionCheckResult.GIcsConnectionCheckResult( available, Instant.now(), if (result.available == available) { result.lastChange } else { Instant.now() } ) - } catch (_: Exception) { + } catch (ex: Exception) { + logger.error("Connection-Check error: {}", ex.message) ConnectionCheckResult.GIcsConnectionCheckResult( false, Instant.now(), -- cgit v1.2.3 From 9fdf360c0c7ba3a871fa0fb550d92a224f647572 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 24 Oct 2025 13:48:21 +0200 Subject: chore: bump version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index ff73da5..5053602 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -13,7 +13,7 @@ plugins { } group = "dev.dnpm" -version = "0.11.2" +version = "0.11.3" var versions = mapOf( "mtb-dto" to "0.1.0-SNAPSHOT", -- cgit v1.2.3 From eaadab88b80062432705bb5fbf295bde91143e76 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 31 Oct 2025 16:45:40 +0100 Subject: fix: usage and serialisation of null values (#174) --- .../dev/dnpm/etl/processor/config/JacksonConfig.kt | 2 + .../dev/dnpm/etl/processor/pseudonym/extensions.kt | 68 +++++++++++----------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt index fb03d66..2ba81b5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt @@ -1,6 +1,7 @@ package dev.dnpm.etl.processor.config import ca.uhn.fhir.context.FhirContext +import com.fasterxml.jackson.annotation.JsonInclude import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import com.fasterxml.jackson.databind.ObjectMapper @@ -24,4 +25,5 @@ class JacksonConfig { .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS).registerModule( JavaTimeModule() ) + .setSerializationInclusion(JsonInclude.Include.NON_NULL) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt index 01c781b..21d49c2 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt @@ -35,56 +35,56 @@ import org.apache.commons.codec.digest.DigestUtils infix fun Mtb.pseudonymizeWith(pseudonymizeService: PseudonymizeService) { val patientPseudonym = pseudonymizeService.patientPseudonym(PatientId(this.patient.id)).value - this.episodesOfCare?.forEach { it.patient.id = patientPseudonym } + this.episodesOfCare?.forEach { it.patient?.id = patientPseudonym } this.carePlans?.forEach { it.patient.id = patientPseudonym - it.rebiopsyRequests?.forEach { it.patient.id = patientPseudonym } - it.histologyReevaluationRequests?.forEach { it.patient.id = patientPseudonym } - it.medicationRecommendations.forEach { it.patient.id = patientPseudonym } - it.studyEnrollmentRecommendations?.forEach { it.patient.id = patientPseudonym } - it.procedureRecommendations?.forEach { it.patient.id = patientPseudonym } - it.geneticCounselingRecommendation.patient.id = patientPseudonym + it.rebiopsyRequests?.forEach { it.patient?.id = patientPseudonym } + it.histologyReevaluationRequests?.forEach { it.patient?.id = patientPseudonym } + it.medicationRecommendations?.forEach { it.patient?.id = patientPseudonym } + it.studyEnrollmentRecommendations?.forEach { it.patient?.id = patientPseudonym } + it.procedureRecommendations?.forEach { it.patient?.id = patientPseudonym } + it.geneticCounselingRecommendation?.patient?.id = patientPseudonym } - this.diagnoses?.forEach { it.patient.id = patientPseudonym } - this.guidelineTherapies?.forEach { it.patient.id = patientPseudonym } - this.guidelineProcedures?.forEach { it.patient.id = patientPseudonym } + this.diagnoses?.forEach { it.patient?.id = patientPseudonym } + this.guidelineTherapies?.forEach { it.patient?.id = patientPseudonym } + this.guidelineProcedures?.forEach { it.patient?.id = patientPseudonym } this.patient.id = patientPseudonym - this.claims?.forEach { it.patient.id = patientPseudonym } - this.claimResponses?.forEach { it.patient.id = patientPseudonym } - this.diagnoses?.forEach { it.patient.id = patientPseudonym } - this.familyMemberHistories?.forEach { it.patient.id = patientPseudonym } + this.claims?.forEach { it.patient?.id = patientPseudonym } + this.claimResponses?.forEach { it.patient?.id = patientPseudonym } + this.diagnoses?.forEach { it.patient?.id = patientPseudonym } + this.familyMemberHistories?.forEach { it.patient?.id = patientPseudonym } this.histologyReports?.forEach { it.patient.id = patientPseudonym it.results.tumorMorphology?.patient?.id = patientPseudonym it.results.tumorCellContent?.patient?.id = patientPseudonym } this.ngsReports?.forEach { - it.patient.id = patientPseudonym - it.results.simpleVariants?.forEach { it.patient.id = patientPseudonym } - it.results.copyNumberVariants?.forEach { it.patient.id = patientPseudonym } - it.results.dnaFusions?.forEach { it.patient.id = patientPseudonym } - it.results.rnaFusions?.forEach { it.patient.id = patientPseudonym } - it.results.tumorCellContent?.patient?.id = patientPseudonym - it.results.brcaness?.patient?.id = patientPseudonym - it.results.tmb?.patient?.id = patientPseudonym - it.results.hrdScore?.patient?.id = patientPseudonym + it.patient?.id = patientPseudonym + it.results?.simpleVariants?.forEach { it.patient?.id = patientPseudonym } + it.results?.copyNumberVariants?.forEach { it.patient?.id = patientPseudonym } + it.results?.dnaFusions?.forEach { it.patient?.id = patientPseudonym } + it.results?.rnaFusions?.forEach { it.patient?.id = patientPseudonym } + it.results?.tumorCellContent?.patient?.id = patientPseudonym + it.results?.brcaness?.patient?.id = patientPseudonym + it.results?.tmb?.patient?.id = patientPseudonym + it.results?.hrdScore?.patient?.id = patientPseudonym } this.ihcReports?.forEach { - it.patient.id = patientPseudonym - it.results.msiMmr?.forEach { it.patient.id = patientPseudonym } - it.results.proteinExpression?.forEach { it.patient.id = patientPseudonym } + it.patient?.id = patientPseudonym + it.results?.msiMmr?.forEach { it.patient?.id = patientPseudonym } + it.results?.proteinExpression?.forEach { it.patient?.id = patientPseudonym } } - this.responses?.forEach { it.patient.id = patientPseudonym } - this.specimens?.forEach { it.patient.id = patientPseudonym } - this.priorDiagnosticReports?.forEach { it.patient.id = patientPseudonym } - this.performanceStatus?.forEach { it.patient.id = patientPseudonym } + this.responses?.forEach { it.patient?.id = patientPseudonym } + this.specimens?.forEach { it.patient?.id = patientPseudonym } + this.priorDiagnosticReports?.forEach { it.patient?.id = patientPseudonym } + this.performanceStatus?.forEach { it.patient?.id = patientPseudonym } this.systemicTherapies?.forEach { it.history?.forEach { - it.patient.id = patientPseudonym + it.patient?.id = patientPseudonym } } this.followUps?.forEach { - it.patient.id = patientPseudonym + it.patient?.id = patientPseudonym } this.msiFindings?.forEach { it -> it.patient.id = patientPseudonym } @@ -254,7 +254,7 @@ infix fun Mtb.anonymizeContentWith(pseudonymizeService: PseudonymizeService) { this.ihcReports?.forEach { it -> it.id = it?.id?.let(::anonymize) it.specimen?.id = it.specimen?.id?.let(::anonymize) - it.results.proteinExpression.forEach { it -> it?.id = it.id.let(::anonymize) } + it.results?.proteinExpression?.forEach { it -> it?.id = it.id.let(::anonymize) } } this.msiFindings?.forEach { it -> @@ -310,5 +310,5 @@ fun Mtb.ensureMetaDataIsInitialized() { } infix fun Mtb.addGenomDeTan(pseudonymizeService: PseudonymizeService) { - this.metadata.transferTan = pseudonymizeService.genomDeTan(PatientId(this.patient.id)) + this.metadata?.transferTan = pseudonymizeService.genomDeTan(PatientId(this.patient.id)) } -- cgit v1.2.3 From dedcec4ad59462b97531e498e8e388688bd0b8b1 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Fri, 31 Oct 2025 16:54:33 +0100 Subject: fix: only send nested content with MTB record (#175) --- src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt | 2 +- .../kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index d45fc51..ef46c0a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -46,7 +46,7 @@ class KafkaMtbFileSender( ProducerRecord( kafkaProperties.outputTopic, key(request), - objectMapper.writeValueAsString(request), + objectMapper.writeValueAsString(request.content), ) record.headers().add("requestId", request.requestId.value.toByteArray()) when (request) { diff --git a/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt index d13d5e1..022b8dd 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSenderTest.kt @@ -224,8 +224,8 @@ class KafkaMtbFileSenderTest { } } - fun dnmpV2kafkaRecordData(requestId: RequestId): MtbRequest { - return DnpmV2MtbFileRequest(requestId, dnpmV2MtbFile()) + fun dnmpV2kafkaRecordData(requestId: RequestId): Mtb { + return DnpmV2MtbFileRequest(requestId, dnpmV2MtbFile()).content } data class TestData(val requestStatus: RequestStatus, val exception: Throwable? = null) -- cgit v1.2.3 From 6dfbadde23a7c920c4d4e474a060d2caf9c05246 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 27 Oct 2025 13:52:08 +0100 Subject: fix: use correct gPAS URI for connection test (#166) (cherry picked from commit af27399fcc8283ee2a2d1eab64280f200e85995d) --- .../processor/monitoring/ConnectionCheckService.kt | 3 +- .../monitoring/ConnectionCheckServiceTest.kt | 140 +++++++++++++++++++++ 2 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt index 1343ee0..bf8b8bd 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt @@ -183,8 +183,7 @@ class GPasConnectionCheckService( fun check() { result = try { val uri = UriComponentsBuilder.fromUriString( - gPasConfigProperties.uri?.replace("/\$pseudonymizeAllowCreate", "/metadata").toString() - ).build().toUri() + gPasConfigProperties.uri.toString()).path("/metadata").build().toUri() val headers = HttpHeaders() headers.contentType = MediaType.APPLICATION_JSON diff --git a/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt new file mode 100644 index 0000000..788ca6a --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckServiceTest.kt @@ -0,0 +1,140 @@ +package dev.dnpm.etl.processor.monitoring + +import dev.dnpm.etl.processor.config.GIcsConfigProperties +import dev.dnpm.etl.processor.config.GPasConfigProperties +import dev.dnpm.etl.processor.config.RestTargetProperties +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mock +import org.mockito.junit.jupiter.MockitoExtension +import org.springframework.http.HttpMethod +import org.springframework.http.MediaType +import org.springframework.test.web.client.MockRestServiceServer +import org.springframework.test.web.client.match.MockRestRequestMatchers.method +import org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo +import org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess +import org.springframework.web.client.RestTemplate +import reactor.core.publisher.Sinks + +@ExtendWith(MockitoExtension::class) +class ConnectionCheckServiceTest { + + @Nested + inner class RestConnectionCheckServiceTest { + + lateinit var mockRestServiceServer: MockRestServiceServer + lateinit var service: RestConnectionCheckService + + @BeforeEach + fun setUp( + @Mock sink: Sinks.Many + ) { + val restTemplate = RestTemplate() + this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate) + + val restTargetProperties = RestTargetProperties( + "http://localhost/api", + "user", + "password", + ) + + this.service = RestConnectionCheckService(restTemplate, restTargetProperties, sink) + } + + @Test + fun shouldSendRequestToCorrectUri() { + this.mockRestServiceServer + .expect(method(HttpMethod.GET)) + .andExpect(requestTo("http://localhost/api/mtb/kaplan-meier/config")) + .andRespond( + withSuccess("OK", MediaType.APPLICATION_JSON), + ) + + this.service.check() + + this.mockRestServiceServer.verify() + + } + } + + @Nested + inner class GPasConnectionCheckServiceTest { + + lateinit var mockRestServiceServer: MockRestServiceServer + lateinit var service: GPasConnectionCheckService + + @BeforeEach + fun setUp( + @Mock sink: Sinks.Many + ) { + val restTemplate = RestTemplate() + this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate) + + val gpasTargetProperties = GPasConfigProperties( + "http://localhost/gpas", + "patientDomain", + "genomDeTanDomain", + "username", + "password", + ) + + this.service = GPasConnectionCheckService(restTemplate, gpasTargetProperties, sink) + } + + @Test + fun shouldSendRequestToCorrectUri() { + this.mockRestServiceServer + .expect(method(HttpMethod.GET)) + .andExpect(requestTo("http://localhost/gpas/metadata")) + .andRespond( + withSuccess("OK", MediaType.APPLICATION_JSON), + ) + + this.service.check() + + this.mockRestServiceServer.verify() + + } + } + + @Nested + inner class GIcsConnectionCheckServiceTest { + + lateinit var mockRestServiceServer: MockRestServiceServer + lateinit var service: GIcsConnectionCheckService + + @BeforeEach + fun setUp( + @Mock sink: Sinks.Many + ) { + val restTemplate = RestTemplate() + this.mockRestServiceServer = MockRestServiceServer.createServer(restTemplate) + + val gicsTargetProperties = GIcsConfigProperties( + "http://localhost/gics", + "username", + "password", + ) + + this.service = GIcsConnectionCheckService(restTemplate, gicsTargetProperties, sink) + } + + @Test + fun shouldSendRequestToCorrectUri() { + this.mockRestServiceServer + .expect(method(HttpMethod.GET)) + .andExpect(requestTo("http://localhost/gics/metadata")) + .andRespond( + withSuccess("OK", MediaType.APPLICATION_JSON), + ) + + this.service.check() + + this.mockRestServiceServer.verify() + + } + } + +} \ No newline at end of file -- cgit v1.2.3 From c88788997d2d7487eb4460d7e5cd49d087454f2f Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Mon, 3 Nov 2025 11:43:19 +0100 Subject: fix: do not re-anonymize diagnosis ID (#179) (cherry picked from commit 30ad7c299cb5c6e70a934df885efe82501fe5172) --- .../dev/dnpm/etl/processor/pseudonym/extensions.kt | 36 ++++------- .../dnpm/etl/processor/pseudonym/ExtensionsTest.kt | 74 +++++++++++++++++++++- 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt index 21d49c2..8721cbe 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt @@ -125,35 +125,32 @@ infix fun Mtb.anonymizeContentWith(pseudonymizeService: PseudonymizeService) { this.carePlans?.onEach { carePlan -> carePlan?.apply { - id = id?.let { anonymize(it) } + this.id = id?.let { anonymize(it) } - diagnoses?.forEach { it -> it?.id = it.id?.let(::anonymize) } - geneticCounselingRecommendation?.apply { - id = geneticCounselingRecommendation.id?.let(::anonymize) + this.geneticCounselingRecommendation?.apply { + this.id = this.id?.let(::anonymize) } - rebiopsyRequests?.forEach { it -> + this.rebiopsyRequests?.forEach { it -> it.id = it.id?.let(::anonymize) it.tumorEntity?.id = it.tumorEntity?.id?.let(::anonymize) } - histologyReevaluationRequests?.forEach { it -> + this.histologyReevaluationRequests?.forEach { it -> it.id = it?.id?.let(::anonymize) it.specimen?.id = it.specimen?.id?.let(::anonymize) } - medicationRecommendations?.forEach { it -> + this.medicationRecommendations?.forEach { it -> it.id = it?.id?.let(::anonymize) it.supportingVariants?.forEach { it -> it.variant?.id = it.variant?.id?.let(::anonymize) } it.reason?.id = it.reason?.id?.let(::anonymize) } - reason?.id = reason?.id?.let(::anonymize) - studyEnrollmentRecommendations?.forEach { it -> + this.reason?.id = this.reason?.id?.let(::anonymize) + this.studyEnrollmentRecommendations?.forEach { it -> it?.reason?.id = it.reason?.id?.let(::anonymize) } - - procedureRecommendations?.forEach { it -> - + this.procedureRecommendations?.forEach { it -> it.id = it?.id?.let(::anonymize) it.supportingVariants?.forEach { it -> it.variant?.id = it.variant?.id?.let(::anonymize) @@ -161,16 +158,11 @@ infix fun Mtb.anonymizeContentWith(pseudonymizeService: PseudonymizeService) { it.reason?.id = it.reason?.id?.let(::anonymize) - studyEnrollmentRecommendations?.forEach { it -> - - it.id = it?.id?.let(::anonymize) - it.supportingVariants.forEach { it -> - it.variant?.id = it?.variant?.id?.let(::anonymize) - } - responses?.forEach { it -> - it.id = it?.id?.let(::anonymize) - it.id = it?.id?.let(::anonymize) - } + } + this.studyEnrollmentRecommendations?.forEach { it -> + it.id = it?.id?.let(::anonymize) + it.supportingVariants.forEach { it -> + it.variant?.id = it?.variant?.id?.let(::anonymize) } } } diff --git a/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt index 8460293..c302362 100644 --- a/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt +++ b/src/test/kotlin/dev/dnpm/etl/processor/pseudonym/ExtensionsTest.kt @@ -112,7 +112,7 @@ class ExtensionsTest { }.whenever(pseudonymizeService).prefix() val mtbFile = Mtb().apply { - this.patient = dev.pcvolkmer.mv64e.mtb.Patient().apply { + this.patient = Patient().apply { this.id = "PID" this.birthDate = Date.from(Instant.now()) this.gender = GenderCoding().apply { @@ -173,7 +173,7 @@ class ExtensionsTest { Math.random().toLong().toString() simpleVariant.fusionPartner5Prime?.transcriptId?.value = Math.random().toLong().toString() - simpleVariant.externalIds?.forEach { it -> + simpleVariant.externalIds?.forEach { it?.value = Math.random().toLong().toString() } } @@ -197,4 +197,74 @@ class ExtensionsTest { } } } + + @Test + fun shouldUseSameAnonymIdForDiagnosisAndDiagnosisReferences(@Mock pseudonymizeService: PseudonymizeService) { + + doAnswer { + it.arguments[0] + "PSEUDO-ID" + }.whenever(pseudonymizeService).patientPseudonym(anyValueClass()) + + doAnswer { + "TESTDOMAIN" + }.whenever(pseudonymizeService).prefix() + + val mtbFile = Mtb().apply { + this.patient = Patient().apply { + this.id = "PID" + this.birthDate = Date.from(Instant.now()) + this.gender = GenderCoding().apply { + this.code = GenderCodingCode.MALE + } + } + this.diagnoses = listOf( + MtbDiagnosis().apply { + this.id = "Diagnosis-1" + } + ) + this.episodesOfCare = listOf( + MtbEpisodeOfCare().apply { + this.id = "Episode-1" + this.diagnoses = listOf( + Reference().apply { + this.id = "Diagnosis-1" + } + ) + } + ) + this.guidelineTherapies = listOf( + MtbSystemicTherapy().apply { + this.id = "Systemic-Therapy-1" + this.reason = Reference().apply { + this.id = "Diagnosis-1" + } + } + ) + this.guidelineProcedures = listOf( + OncoProcedure().apply { + this.id = "Onco-Procedure-1" + this.reason = Reference().apply { + this.id = "Diagnosis-1" + } + } + ) + this.specimens = listOf( + TumorSpecimen().apply { + this.id = "Specimen-1" + this.diagnosis = Reference().apply { + this.id = "Diagnosis-1" + } + } + ) + } + + mtbFile.pseudonymizeWith(pseudonymizeService) + mtbFile.anonymizeContentWith(pseudonymizeService) + + assertThat(mtbFile.diagnoses.first().id).isEqualTo(mtbFile.episodesOfCare.first().diagnoses.first().id) + assertThat(mtbFile.diagnoses.first().id).isEqualTo(mtbFile.guidelineTherapies.first().reason.id) + assertThat(mtbFile.diagnoses.first().id).isEqualTo(mtbFile.guidelineProcedures.first().reason.id) + assertThat(mtbFile.diagnoses.first().id).isEqualTo(mtbFile.specimens.first().diagnosis.id) + } } -- cgit v1.2.3 From 160222b488580f19a68600ad09ef73984b8eead1 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Tue, 4 Nov 2025 11:39:05 +0100 Subject: fix: errors in response parsing (#181) (cherry picked from commit 24d9116e31a7d0afdfbf47aa96a0c8bd54c9508b) --- .../dev/dnpm/etl/processor/config/JacksonConfig.kt | 7 ++++- .../dnpm/etl/processor/monitoring/ReportService.kt | 14 +++++++-- .../etl/processor/monitoring/ReportServiceTest.kt | 36 ++++++++++++++++++++++ src/test/resources/dip-response.json | 36 ++++++++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 src/test/kotlin/dev/dnpm/etl/processor/monitoring/ReportServiceTest.kt create mode 100644 src/test/resources/dip-response.json diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt index 2ba81b5..282f69e 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/JacksonConfig.kt @@ -6,6 +6,7 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule @Configuration @@ -22,8 +23,12 @@ class JacksonConfig { @Bean fun objectMapper(): ObjectMapper = ObjectMapper().registerModule(FhirResourceModule()) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS).registerModule( + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .registerModule( JavaTimeModule() ) + .registerModule( + Jdk8Module() + ) .setSerializationInclusion(JsonInclude.Include.NON_NULL) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index e9ea489..dd5c44a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -21,13 +21,14 @@ package dev.dnpm.etl.processor.monitoring import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonValue import com.fasterxml.jackson.core.JsonParseException import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.ObjectMapper import dev.dnpm.etl.processor.monitoring.ReportService.Issue import dev.dnpm.etl.processor.monitoring.ReportService.Severity -import java.util.Optional +import java.util.* class ReportService( private val objectMapper: ObjectMapper @@ -55,12 +56,19 @@ class ReportService( @JsonIgnoreProperties(ignoreUnknown = true) - private data class DataQualityReport(val issues: List) + private data class DataQualityReport( + @param:JsonProperty(value = "issues") + val issues: List + ) @JsonIgnoreProperties(ignoreUnknown = true) data class Issue( + @param:JsonProperty(value = "severity") val severity: Severity, - @JsonAlias("details") val message: String, + @param:JsonProperty(value = "message") + @param:JsonAlias("details") + val message: String, + @param:JsonProperty(value = "path") val path: Optional = Optional.empty() ) diff --git a/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ReportServiceTest.kt b/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ReportServiceTest.kt new file mode 100644 index 0000000..4bf1321 --- /dev/null +++ b/src/test/kotlin/dev/dnpm/etl/processor/monitoring/ReportServiceTest.kt @@ -0,0 +1,36 @@ +package dev.dnpm.etl.processor.monitoring + +import dev.dnpm.etl.processor.config.JacksonConfig +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.* + +class ReportServiceTest { + + lateinit var service: ReportService + + @BeforeEach + fun setUp() { + val jacksonConfig = JacksonConfig() + service = ReportService(jacksonConfig.objectMapper()) + } + + @Test + fun shouldParseDataQualityReport() { + val dataQualityReport = Objects.requireNonNull(this.javaClass.classLoader.getResource("dip-response.json")) + .readText() + + val actual = service.deserialize(dataQualityReport) + + assertThat(actual).isNotNull + assertThat(actual).hasSize(6) + assertThat(actual[0].severity).isEqualTo(ReportService.Severity.FATAL) + assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR) + assertThat(actual[2].severity).isEqualTo(ReportService.Severity.WARNING) + assertThat(actual[3].severity).isEqualTo(ReportService.Severity.WARNING) + assertThat(actual[4].severity).isEqualTo(ReportService.Severity.WARNING) + assertThat(actual[5].severity).isEqualTo(ReportService.Severity.INFO) + } + +} \ No newline at end of file diff --git a/src/test/resources/dip-response.json b/src/test/resources/dip-response.json new file mode 100644 index 0000000..d1dacb6 --- /dev/null +++ b/src/test/resources/dip-response.json @@ -0,0 +1,36 @@ +{ + "patient": "TEST-PATIENT0123456789abcdef", + "issues": [ + { + "severity": "error", + "message": "MVH-Einschluss-Fallkonferenz darf nicht vor oder ohne Einwilligung zur Teilnahme stattgefunden haben", + "path": "/Datum der MVH-Einwilligung" + }, + { + "severity": "warning", + "message": "Fehlende Angabe 'Tumor-Grading'", + "path": "/Diagnose[0123456789abcdef]/Tumor-Grading" + }, + { + "severity": "fatal", + "message": "Nicht auflösbare Referenz-ID '0123456789abcdef' auf Objekt 'Diagnose'", + "path": "/Systemische-Therapie[0123456789abcdef]/Therapie-Grund (Diagnose)" + }, + { + "severity": "warning", + "message": "Fehlende Angabe", + "path": "/Kostenübernahme-Anträge" + }, + { + "severity": "info", + "message": "Sonstige Info", + "path": "/Kostenübernahme-Antworten" + }, + { + "severity": "warning", + "message": "Fehlende Angabe", + "path": "/MTB-Therapien" + } + ], + "createdAt": "2025-11-04T10:00:00.000000000Z" +} -- cgit v1.2.3