diff options
Diffstat (limited to 'src/main/kotlin/dev')
27 files changed, 580 insertions, 159 deletions
diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index d951c60..7c192c8 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -69,6 +69,9 @@ data class GPasConfigProperties( @ConfigurationProperties(RestTargetProperties.NAME) data class RestTargetProperties( val uri: String?, + val username: String?, + val password: String?, + val isBwhc: Boolean = false, ) { companion object { const val NAME = "app.rest" diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 0ae2c2f..5fc1120 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -20,21 +20,32 @@ package dev.dnpm.etl.processor.config import com.fasterxml.jackson.databind.ObjectMapper -import dev.dnpm.etl.processor.monitoring.* +import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult +import dev.dnpm.etl.processor.monitoring.ConnectionCheckService +import dev.dnpm.etl.processor.monitoring.GPasConnectionCheckService +import dev.dnpm.etl.processor.monitoring.ReportService import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService -import dev.dnpm.etl.processor.services.TokenRepository -import dev.dnpm.etl.processor.services.TokenService +import dev.dnpm.etl.processor.security.TokenRepository +import dev.dnpm.etl.processor.security.TokenService import dev.dnpm.etl.processor.services.Transformation import dev.dnpm.etl.processor.services.TransformationService +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager +import org.apache.hc.client5.http.socket.ConnectionSocketFactory +import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory +import org.apache.hc.core5.http.config.RegistryBuilder import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.data.jdbc.repository.config.AbstractJdbcConfiguration +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory import org.springframework.retry.RetryCallback import org.springframework.retry.RetryContext import org.springframework.retry.RetryListener @@ -46,6 +57,13 @@ import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.security.provisioning.InMemoryUserDetailsManager import org.springframework.web.client.RestTemplate import reactor.core.publisher.Sinks +import java.io.BufferedInputStream +import java.io.FileInputStream +import java.security.KeyStore +import java.security.cert.CertificateFactory +import java.security.cert.X509Certificate +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory import kotlin.time.Duration.Companion.seconds import kotlin.time.toJavaDuration @@ -70,8 +88,20 @@ class AppConfiguration { @ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS") @Bean - fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator { - return GpasPseudonymGenerator(configProperties, retryTemplate) + fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator { + try { + if (!configProperties.sslCaLocation.isNullOrBlank()) { + return GpasPseudonymGenerator( + configProperties, + retryTemplate, + createCustomGpasRestTemplate(configProperties) + ) + } + } catch (e: Exception) { + throw RuntimeException(e) + } + + return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate) } @ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "BUILDIN", matchIfMissing = true) @@ -83,8 +113,80 @@ class AppConfiguration { @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS") @ConditionalOnMissingBean @Bean - fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator { - return GpasPseudonymGenerator(configProperties, retryTemplate) + fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator { + try { + if (!configProperties.sslCaLocation.isNullOrBlank()) { + return GpasPseudonymGenerator( + configProperties, + retryTemplate, + createCustomGpasRestTemplate(configProperties) + ) + } + } catch (e: Exception) { + throw RuntimeException(e) + } + + return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate) + } + + private fun createCustomGpasRestTemplate(configProperties: GPasConfigProperties): RestTemplate { + fun getSslContext(certificateLocation: String): SSLContext? { + val ks = KeyStore.getInstance(KeyStore.getDefaultType()) + + val fis = FileInputStream(certificateLocation) + val ca = CertificateFactory.getInstance("X.509") + .generateCertificate(BufferedInputStream(fis)) as X509Certificate + + ks.load(null, null) + ks.setCertificateEntry(1.toString(), ca) + + val tmf = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm() + ) + tmf.init(ks) + + val sslContext = SSLContext.getInstance("TLS") + sslContext.init(null, tmf.trustManagers, null) + + return sslContext + } + + fun getCustomRestTemplate(customSslContext: SSLContext): RestTemplate { + val sslsf = SSLConnectionSocketFactory(customSslContext) + val socketFactoryRegistry = RegistryBuilder.create<ConnectionSocketFactory>() + .register("https", sslsf).register("http", PlainConnectionSocketFactory()).build() + + val connectionManager = BasicHttpClientConnectionManager( + socketFactoryRegistry + ) + val httpClient = HttpClients.custom() + .setConnectionManager(connectionManager).build() + + val requestFactory = HttpComponentsClientHttpRequestFactory( + httpClient + ) + return RestTemplate(requestFactory) + } + + try { + if (!configProperties.sslCaLocation.isNullOrBlank()) { + val customSslContext = getSslContext(configProperties.sslCaLocation) + logger.warn( + String.format( + "%s has been initialized with SSL certificate %s. This is deprecated in favor of including Root CA.", + this.javaClass.name, configProperties.sslCaLocation + ) + ) + + if (customSslContext != null) { + return getCustomRestTemplate(customSslContext) + } + } + } catch (e: Exception) { + throw RuntimeException(e) + } + + throw RuntimeException("Custom SSL configuration for gPAS not usable") } @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "BUILDIN") @@ -173,5 +275,9 @@ class AppConfiguration { return GPasConnectionCheckService(restTemplate, gPasConfigProperties, connectionCheckUpdateProducer) } + @Bean + fun jdbcConfiguration(): AbstractJdbcConfiguration { + return AppJdbcConfiguration() + } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppJdbcConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppJdbcConfiguration.kt new file mode 100644 index 0000000..898982c --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppJdbcConfiguration.kt @@ -0,0 +1,25 @@ +package dev.dnpm.etl.processor.config + +import dev.dnpm.etl.processor.Fingerprint +import org.springframework.context.annotation.Configuration +import org.springframework.core.convert.converter.Converter +import org.springframework.data.jdbc.repository.config.AbstractJdbcConfiguration + +@Configuration +class AppJdbcConfiguration : AbstractJdbcConfiguration() { + override fun userConverters(): MutableList<*> { + return mutableListOf(StringToFingerprintConverter(), FingerprintToStringConverter()) + } +} + +class StringToFingerprintConverter : Converter<String, Fingerprint> { + override fun convert(source: String): Fingerprint { + return Fingerprint(source) + } +} + +class FingerprintToStringConverter : Converter<Fingerprint, String> { + override fun convert(source: Fingerprint): String { + return source.value + } +}
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt index fc2676b..a393267 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppRestConfiguration.kt @@ -1,7 +1,7 @@ /* * This file is part of ETL-Processor * - * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published @@ -23,7 +23,8 @@ import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult import dev.dnpm.etl.processor.monitoring.ConnectionCheckService import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService import dev.dnpm.etl.processor.output.MtbFileSender -import dev.dnpm.etl.processor.output.RestMtbFileSender +import dev.dnpm.etl.processor.output.RestBwhcMtbFileSender +import dev.dnpm.etl.processor.output.RestDipMtbFileSender import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty @@ -54,8 +55,13 @@ class AppRestConfiguration { restTargetProperties: RestTargetProperties, retryTemplate: RetryTemplate ): MtbFileSender { - logger.info("Selected 'RestMtbFileSender'") - return RestMtbFileSender(restTemplate, restTargetProperties, retryTemplate) + if (restTargetProperties.isBwhc) { + logger.info("Selected 'RestBwhcMtbFileSender'") + return RestBwhcMtbFileSender(restTemplate, restTargetProperties, retryTemplate) + } + + logger.info("Selected 'RestDipMtbFileSender'") + return RestDipMtbFileSender(restTemplate, restTargetProperties, retryTemplate) } @Bean diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt index c377555..6b063bd 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt @@ -21,7 +21,7 @@ package dev.dnpm.etl.processor.config import dev.dnpm.etl.processor.security.UserRole import dev.dnpm.etl.processor.security.UserRoleRepository -import dev.dnpm.etl.processor.services.UserRoleService +import dev.dnpm.etl.processor.security.UserRoleService import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -89,7 +89,7 @@ class AppSecurityConfiguration( http { authorizeRequests { authorize("/configs/**", hasRole("ADMIN")) - authorize("/mtbfile/**", hasAnyRole("MTBFILE")) + authorize("/mtbfile/**", hasAnyRole("MTBFILE", "ADMIN", "USER")) authorize("/report/**", hasAnyRole("ADMIN", "USER")) authorize("*.css", permitAll) authorize("*.ico", permitAll) @@ -147,7 +147,7 @@ class AppSecurityConfiguration( http { authorizeRequests { authorize("/configs/**", hasRole("ADMIN")) - authorize("/mtbfile/**", hasAnyRole("MTBFILE")) + authorize("/mtbfile/**", hasAnyRole("MTBFILE", "ADMIN")) authorize("/report/**", hasRole("ADMIN")) authorize(anyRequest, permitAll) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt b/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt index de901ce..2aff8cb 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt @@ -22,6 +22,8 @@ package dev.dnpm.etl.processor.input import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.PatientId +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.services.RequestProcessor import org.apache.kafka.clients.consumer.ConsumerRecord import org.slf4j.LoggerFactory @@ -35,11 +37,12 @@ class KafkaInputListener( override fun onMessage(data: ConsumerRecord<String, String>) { val mtbFile = objectMapper.readValue(data.value(), MtbFile::class.java) + val patientId = PatientId(mtbFile.patient.id) val firstRequestIdHeader = data.headers().headers("requestId")?.firstOrNull() val requestId = if (null != firstRequestIdHeader) { - String(firstRequestIdHeader.value()) + RequestId(String(firstRequestIdHeader.value())) } else { - "" + RequestId("") } if (mtbFile.consent.status == Consent.Status.ACTIVE) { @@ -52,9 +55,9 @@ class KafkaInputListener( } else { logger.debug("Accepted MTB File and process deletion") if (requestId.isBlank()) { - requestProcessor.processDeletion(mtbFile.patient.id) + requestProcessor.processDeletion(patientId) } else { - requestProcessor.processDeletion(mtbFile.patient.id, requestId) + requestProcessor.processDeletion(patientId, requestId) } } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/input/MtbFileRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/input/MtbFileRestController.kt index 8259288..9e282c2 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/input/MtbFileRestController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/input/MtbFileRestController.kt @@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.input import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.PatientId import dev.dnpm.etl.processor.services.RequestProcessor import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity @@ -46,7 +47,8 @@ class MtbFileRestController( requestProcessor.processMtbFile(mtbFile) } else { logger.debug("Accepted MTB File and process deletion") - requestProcessor.processDeletion(mtbFile.patient.id) + val patientId = PatientId(mtbFile.patient.id) + requestProcessor.processDeletion(patientId) } return ResponseEntity.accepted().build() } @@ -54,7 +56,7 @@ class MtbFileRestController( @DeleteMapping(path = ["{patientId}"]) fun deleteData(@PathVariable patientId: String): ResponseEntity<Void> { logger.debug("Accepted patient ID to process deletion") - requestProcessor.processDeletion(patientId) + requestProcessor.processDeletion(PatientId(patientId)) return ResponseEntity.accepted().build() } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt index 81ad922..9d96654 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt @@ -26,22 +26,18 @@ import jakarta.annotation.PostConstruct import org.apache.kafka.clients.consumer.Consumer import org.apache.kafka.common.errors.TimeoutException import org.springframework.beans.factory.annotation.Qualifier -import org.springframework.http.HttpEntity -import org.springframework.http.HttpHeaders -import org.springframework.http.HttpMethod -import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.RequestEntity +import org.springframework.http.* import org.springframework.scheduling.annotation.Scheduled import org.springframework.web.client.RestTemplate import org.springframework.web.util.UriComponentsBuilder import reactor.core.publisher.Sinks +import java.time.Instant import kotlin.time.Duration.Companion.seconds import kotlin.time.toJavaDuration interface ConnectionCheckService { - fun connectionAvailable(): Boolean + fun connectionAvailable(): ConnectionCheckResult } @@ -51,9 +47,27 @@ sealed class ConnectionCheckResult { abstract val available: Boolean - data class KafkaConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult() - data class RestConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult() - data class GPasConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult() + abstract val timestamp: Instant + + abstract val lastChange: Instant + + data class KafkaConnectionCheckResult( + override val available: Boolean, + override val timestamp: Instant, + override val lastChange: Instant + ) : ConnectionCheckResult() + + data class RestConnectionCheckResult( + override val available: Boolean, + override val timestamp: Instant, + override val lastChange: Instant + ) : ConnectionCheckResult() + + data class GPasConnectionCheckResult( + override val available: Boolean, + override val timestamp: Instant, + override val lastChange: Instant + ) : ConnectionCheckResult() } class KafkaConnectionCheckService( @@ -62,25 +76,33 @@ class KafkaConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult> ) : OutputConnectionCheckService { - private var connectionAvailable: Boolean = false - + private var result = ConnectionCheckResult.KafkaConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { - connectionAvailable = try { - null != consumer.listTopics(5.seconds.toJavaDuration()) + result = try { + val available = null != consumer.listTopics(5.seconds.toJavaDuration()) + ConnectionCheckResult.KafkaConnectionCheckResult( + available, + Instant.now(), + if (result.available == available) { result.lastChange } else { Instant.now() } + ) } catch (e: TimeoutException) { - false + ConnectionCheckResult.KafkaConnectionCheckResult( + false, + Instant.now(), + if (!result.available) { result.lastChange } else { Instant.now() } + ) } connectionCheckUpdateProducer.emitNext( - ConnectionCheckResult.KafkaConnectionCheckResult(connectionAvailable), + result, Sinks.EmitFailureHandler.FAIL_FAST ) } - override fun connectionAvailable(): Boolean { - return this.connectionAvailable + override fun connectionAvailable(): ConnectionCheckResult.KafkaConnectionCheckResult { + return this.result } } @@ -92,27 +114,45 @@ class RestConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult> ) : OutputConnectionCheckService { - private var connectionAvailable: Boolean = false + private var result = ConnectionCheckResult.RestConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { - connectionAvailable = try { - restTemplate.getForEntity( - restTargetProperties.uri?.replace("/etl/api", "").toString(), + result = try { + val available = restTemplate.getForEntity( + if (restTargetProperties.isBwhc) { + UriComponentsBuilder.fromUriString(restTargetProperties.uri.toString()).path("").toUriString() + } else { + UriComponentsBuilder.fromUriString(restTargetProperties.uri.toString()) + .pathSegment("mtb") + .pathSegment("kaplan-meier") + .pathSegment("config") + .toUriString() + }, String::class.java ).statusCode == HttpStatus.OK + + ConnectionCheckResult.RestConnectionCheckResult( + available, + Instant.now(), + if (result.available == available) { result.lastChange } else { Instant.now() } + ) } catch (e: Exception) { - false + ConnectionCheckResult.RestConnectionCheckResult( + false, + Instant.now(), + if (!result.available) { result.lastChange } else { Instant.now() } + ) } connectionCheckUpdateProducer.emitNext( - ConnectionCheckResult.RestConnectionCheckResult(connectionAvailable), + result, Sinks.EmitFailureHandler.FAIL_FAST ) } - override fun connectionAvailable(): Boolean { - return this.connectionAvailable + override fun connectionAvailable(): ConnectionCheckResult.RestConnectionCheckResult { + return this.result } } @@ -123,40 +163,48 @@ class GPasConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult> ) : ConnectionCheckService { - private var connectionAvailable: Boolean = false + private var result = ConnectionCheckResult.GPasConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { - connectionAvailable = try { + result = try { val uri = UriComponentsBuilder.fromUriString( - gPasConfigProperties.uri?.replace("/\$pseudonymizeAllowCreate", "/\$pseudonymize").toString() - ) - .queryParam("target", gPasConfigProperties.target) - .queryParam("original", "???") - .build().toUri() + gPasConfigProperties.uri?.replace("/\$pseudonymizeAllowCreate", "/metadata").toString() + ).build().toUri() val headers = HttpHeaders() headers.contentType = MediaType.APPLICATION_JSON if (!gPasConfigProperties.username.isNullOrBlank() && !gPasConfigProperties.password.isNullOrBlank()) { headers.setBasicAuth(gPasConfigProperties.username, gPasConfigProperties.password) } - restTemplate.exchange( + + val available = restTemplate.exchange( uri, HttpMethod.GET, HttpEntity<Void>(headers), Void::class.java ).statusCode == HttpStatus.OK + + ConnectionCheckResult.GPasConnectionCheckResult( + available, + Instant.now(), + if (result.available == available) { result.lastChange } else { Instant.now() } + ) } catch (e: Exception) { - false + ConnectionCheckResult.GPasConnectionCheckResult( + false, + Instant.now(), + if (!result.available) { result.lastChange } else { Instant.now() } + ) } connectionCheckUpdateProducer.emitNext( - ConnectionCheckResult.GPasConnectionCheckResult(connectionAvailable), + result, Sinks.EmitFailureHandler.FAIL_FAST ) } - override fun connectionAvailable(): Boolean { - return this.connectionAvailable + override fun connectionAvailable(): ConnectionCheckResult.GPasConnectionCheckResult { + return this.result } }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index 97ecd05..062f749 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -19,6 +19,7 @@ package dev.dnpm.etl.processor.monitoring +import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonValue import com.fasterxml.jackson.core.JsonParseException @@ -54,7 +55,7 @@ class ReportService( private data class DataQualityReport(val issues: List<Issue>) @JsonIgnoreProperties(ignoreUnknown = true) - data class Issue(val severity: Severity, val message: String) + data class Issue(val severity: Severity, @JsonAlias("details") val message: String) enum class Severity(@JsonValue val value: String) { FATAL("fatal"), diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt index 028b4a3..36c9705 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt @@ -19,10 +19,12 @@ package dev.dnpm.etl.processor.monitoring +import dev.dnpm.etl.processor.* import org.springframework.data.annotation.Id import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.jdbc.repository.query.Query +import org.springframework.data.relational.core.mapping.Column import org.springframework.data.relational.core.mapping.Embedded import org.springframework.data.relational.core.mapping.Table import org.springframework.data.repository.CrudRepository @@ -30,26 +32,48 @@ import org.springframework.data.repository.PagingAndSortingRepository import java.time.Instant import java.util.* -typealias RequestId = UUID - @Table("request") data class Request( @Id val id: Long? = null, - val uuid: String = RequestId.randomUUID().toString(), - val patientId: String, - val pid: String, - val fingerprint: String, + val uuid: RequestId = randomRequestId(), + val patientPseudonym: PatientPseudonym, + val pid: PatientId, + @Column("fingerprint") + val fingerprint: Fingerprint, val type: RequestType, var status: RequestStatus, var processedAt: Instant = Instant.now(), @Embedded.Nullable var report: Report? = null -) +) { + constructor( + uuid: RequestId, + patientPseudonym: PatientPseudonym, + pid: PatientId, + fingerprint: Fingerprint, + type: RequestType, + status: RequestStatus + ) : + this(null, uuid, patientPseudonym, pid, fingerprint, type, status, Instant.now()) + + constructor( + uuid: RequestId, + patientPseudonym: PatientPseudonym, + pid: PatientId, + fingerprint: Fingerprint, + type: RequestType, + status: RequestStatus, + processedAt: Instant + ) : + this(null, uuid, patientPseudonym, pid, fingerprint, type, status, processedAt) +} +@JvmRecord data class Report( val description: String, val dataQualityReport: String = "" ) +@JvmRecord data class CountedState( val count: Int, val status: RequestStatus, @@ -57,17 +81,17 @@ data class CountedState( interface RequestRepository : CrudRepository<Request, Long>, PagingAndSortingRepository<Request, Long> { - fun findAllByPatientIdOrderByProcessedAtDesc(patientId: String): List<Request> + fun findAllByPatientPseudonymOrderByProcessedAtDesc(patientId: PatientPseudonym): List<Request> - fun findByUuidEquals(uuid: String): Optional<Request> + fun findByUuidEquals(uuid: RequestId): Optional<Request> - fun findRequestByPatientId(patientId: String, pageable: Pageable): Page<Request> + fun findRequestByPatientPseudonym(patientPseudonym: PatientPseudonym, pageable: Pageable): Page<Request> @Query("SELECT count(*) AS count, status FROM request WHERE type = 'MTB_FILE' GROUP BY status ORDER BY status, count DESC;") fun countStates(): List<CountedState> @Query("SELECT count(*) AS count, status FROM (" + - "SELECT status, rank() OVER (PARTITION BY patient_id ORDER BY processed_at DESC) AS rank FROM request " + + "SELECT status, rank() OVER (PARTITION BY patient_pseudonym ORDER BY processed_at DESC) AS rank FROM request " + "WHERE type = 'MTB_FILE' AND status NOT IN ('DUPLICATION') " + ") rank WHERE rank = 1 GROUP BY status ORDER BY status, count DESC;") fun findPatientUniqueStates(): List<CountedState> @@ -76,7 +100,7 @@ interface RequestRepository : CrudRepository<Request, Long>, PagingAndSortingRep fun countDeleteStates(): List<CountedState> @Query("SELECT count(*) AS count, status FROM (" + - "SELECT status, rank() OVER (PARTITION BY patient_id ORDER BY processed_at DESC) AS rank FROM request " + + "SELECT status, rank() OVER (PARTITION BY patient_pseudonym ORDER BY processed_at DESC) AS rank FROM request " + "WHERE type = 'DELETE'" + ") rank WHERE rank = 1 GROUP BY status ORDER BY status, count DESC;") fun findPatientUniqueDeleteStates(): List<CountedState> diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index fc5d617..4838689 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -22,6 +22,7 @@ package dev.dnpm.etl.processor.output import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.config.KafkaProperties import dev.dnpm.etl.processor.monitoring.RequestStatus import org.slf4j.LoggerFactory @@ -62,7 +63,7 @@ class KafkaMtbFileSender( val dummyMtbFile = MtbFile.builder() .withConsent( Consent.builder() - .withPatient(request.patientId) + .withPatient(request.patientId.value) .withStatus(Consent.Status.REJECTED) .build() ) @@ -98,8 +99,8 @@ class KafkaMtbFileSender( } private fun key(request: MtbFileSender.DeleteRequest): String { - return "{\"pid\": \"${request.patientId}\"}" + return "{\"pid\": \"${request.patientId.value}\"}" } - data class Data(val requestId: String, val content: MtbFile) + data class Data(val requestId: RequestId, val content: MtbFile) }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt index aca972b..8d994c5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt @@ -20,6 +20,8 @@ package dev.dnpm.etl.processor.output import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.RequestStatus import org.springframework.http.HttpStatusCode @@ -32,9 +34,9 @@ interface MtbFileSender { data class Response(val status: RequestStatus, val body: String = "") - data class MtbFileRequest(val requestId: String, val mtbFile: MtbFile) + data class MtbFileRequest(val requestId: RequestId, val mtbFile: MtbFile) - data class DeleteRequest(val requestId: String, val patientId: String) + data class DeleteRequest(val requestId: RequestId, val patientId: PatientPseudonym) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestBwhcMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestBwhcMtbFileSender.kt new file mode 100644 index 0000000..f4a58e8 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestBwhcMtbFileSender.kt @@ -0,0 +1,49 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.output + +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.config.RestTargetProperties +import org.springframework.retry.support.RetryTemplate +import org.springframework.web.client.RestTemplate +import org.springframework.web.util.UriComponentsBuilder + +class RestBwhcMtbFileSender( + restTemplate: RestTemplate, + private val restTargetProperties: RestTargetProperties, + retryTemplate: RetryTemplate +) : RestMtbFileSender(restTemplate, restTargetProperties, retryTemplate) { + + override fun sendUrl(): String { + return UriComponentsBuilder + .fromUriString(restTargetProperties.uri.toString()) + .pathSegment("MTBFile") + .toUriString() + } + + override fun deleteUrl(patientId: PatientPseudonym): String { + return UriComponentsBuilder + .fromUriString(restTargetProperties.uri.toString()) + .pathSegment("Patient") + .pathSegment(patientId.value) + .toUriString() + } + +}
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt new file mode 100644 index 0000000..42dbb30 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestDipMtbFileSender.kt @@ -0,0 +1,53 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2025 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor.output + +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.config.RestTargetProperties +import org.springframework.retry.support.RetryTemplate +import org.springframework.web.client.RestTemplate +import org.springframework.web.util.UriComponentsBuilder + +class RestDipMtbFileSender( + restTemplate: RestTemplate, + private val restTargetProperties: RestTargetProperties, + retryTemplate: RetryTemplate +) : RestMtbFileSender(restTemplate, restTargetProperties, retryTemplate) { + + override fun sendUrl(): String { + return UriComponentsBuilder + .fromUriString(restTargetProperties.uri.toString()) + .pathSegment("mtb") + .pathSegment("etl") + .pathSegment("patient-record") + .toUriString() + } + + override fun deleteUrl(patientId: PatientPseudonym): String { + return UriComponentsBuilder + .fromUriString(restTargetProperties.uri.toString()) + .pathSegment("mtb") + .pathSegment("etl") + .pathSegment("patient") + .pathSegment(patientId.value) + .toUriString() + } + +}
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt index e1aecb7..5ea42e3 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/RestMtbFileSender.kt @@ -21,15 +21,17 @@ package dev.dnpm.etl.processor.output import dev.dnpm.etl.processor.config.RestTargetProperties import dev.dnpm.etl.processor.monitoring.RequestStatus +import dev.dnpm.etl.processor.PatientPseudonym import org.slf4j.LoggerFactory import org.springframework.http.HttpEntity import org.springframework.http.HttpHeaders import org.springframework.http.MediaType import org.springframework.retry.support.RetryTemplate import org.springframework.web.client.RestClientException +import org.springframework.web.client.RestClientResponseException import org.springframework.web.client.RestTemplate -class RestMtbFileSender( +abstract class RestMtbFileSender( private val restTemplate: RestTemplate, private val restTargetProperties: RestTargetProperties, private val retryTemplate: RetryTemplate @@ -37,14 +39,17 @@ class RestMtbFileSender( private val logger = LoggerFactory.getLogger(RestMtbFileSender::class.java) + abstract fun sendUrl(): String + + abstract fun deleteUrl(patientId: PatientPseudonym): String + override fun send(request: MtbFileSender.MtbFileRequest): MtbFileSender.Response { try { return retryTemplate.execute<MtbFileSender.Response, Exception> { - val headers = HttpHeaders() - headers.contentType = MediaType.APPLICATION_JSON + val headers = getHttpHeaders() val entityReq = HttpEntity(request.mtbFile, headers) val response = restTemplate.postForEntity( - "${restTargetProperties.uri}/MTBFile", + sendUrl(), entityReq, String::class.java ) @@ -60,9 +65,10 @@ class RestMtbFileSender( } } catch (e: IllegalArgumentException) { logger.error("Not a valid URI to export to: '{}'", restTargetProperties.uri!!) - } catch (e: RestClientException) { + } catch (e: RestClientResponseException) { logger.info(restTargetProperties.uri!!.toString()) - logger.error("Cannot send data to remote system", e) + logger.error("Request data not accepted by remote system", e) + return MtbFileSender.Response(e.statusCode.asRequestStatus(), e.responseBodyAsString) } return MtbFileSender.Response(RequestStatus.ERROR, "Sonstiger Fehler bei der Übertragung") } @@ -70,11 +76,10 @@ class RestMtbFileSender( override fun send(request: MtbFileSender.DeleteRequest): MtbFileSender.Response { try { return retryTemplate.execute<MtbFileSender.Response, Exception> { - val headers = HttpHeaders() - headers.contentType = MediaType.APPLICATION_JSON + val headers = getHttpHeaders() val entityReq = HttpEntity(null, headers) restTemplate.delete( - "${restTargetProperties.uri}/Patient/${request.patientId}", + deleteUrl(request.patientId), entityReq, String::class.java ) @@ -94,4 +99,18 @@ class RestMtbFileSender( return this.restTargetProperties.uri.orEmpty() } + private fun getHttpHeaders(): HttpHeaders { + val username = restTargetProperties.username + val password = restTargetProperties.password + val headers = HttpHeaders() + headers.setContentType(MediaType.APPLICATION_JSON) + + if (username.isNullOrBlank() || password.isNullOrBlank()) { + return headers + } + + headers.setBasicAuth(username, password) + return headers + } + }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt index d18cd2c..e80f6ec 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/PseudonymizeService.kt @@ -19,6 +19,8 @@ package dev.dnpm.etl.processor.pseudonym +import dev.dnpm.etl.processor.PatientId +import dev.dnpm.etl.processor.PatientPseudonym import dev.dnpm.etl.processor.config.PseudonymizeConfigProperties class PseudonymizeService( @@ -26,10 +28,10 @@ class PseudonymizeService( private val configProperties: PseudonymizeConfigProperties ) { - fun patientPseudonym(patientId: String): String { + fun patientPseudonym(patientId: PatientId): PatientPseudonym { return when (generator) { - is GpasPseudonymGenerator -> generator.generate(patientId) - else -> "${configProperties.prefix}_${generator.generate(patientId)}" + is GpasPseudonymGenerator -> PatientPseudonym(generator.generate(patientId.value)) + else -> PatientPseudonym("${configProperties.prefix}_${generator.generate(patientId.value)}") } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt index ef25787..bf645f6 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/pseudonym/extensions.kt @@ -20,6 +20,7 @@ package dev.dnpm.etl.processor.pseudonym import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.PatientId import org.apache.commons.codec.digest.DigestUtils /** Replaces patient ID with generated patient pseudonym @@ -29,7 +30,7 @@ import org.apache.commons.codec.digest.DigestUtils * @return The MTB file containing patient pseudonymes */ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) { - val patientPseudonym = pseudonymizeService.patientPseudonym(this.patient.id) + val patientPseudonym = pseudonymizeService.patientPseudonym(PatientId(this.patient.id)).value this.episode?.patient = patientPseudonym this.carePlans?.forEach { it.patient = patientPseudonym } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/TokenService.kt b/src/main/kotlin/dev/dnpm/etl/processor/security/TokenService.kt index f084408..44b04e8 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/TokenService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/security/TokenService.kt @@ -17,7 +17,7 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -package dev.dnpm.etl.processor.services +package dev.dnpm.etl.processor.security import jakarta.annotation.PostConstruct import org.springframework.data.annotation.Id diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/UserRoleService.kt b/src/main/kotlin/dev/dnpm/etl/processor/security/UserRoleService.kt index 6649f7d..174f8a9 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/UserRoleService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/security/UserRoleService.kt @@ -17,11 +17,8 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -package dev.dnpm.etl.processor.services +package dev.dnpm.etl.processor.security -import dev.dnpm.etl.processor.security.Role -import dev.dnpm.etl.processor.security.UserRole -import dev.dnpm.etl.processor.security.UserRoleRepository import org.springframework.data.repository.findByIdOrNull import org.springframework.security.core.session.SessionRegistry import org.springframework.security.oauth2.core.oidc.user.OidcUser diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index bdf07cb..5b2c42a 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.* import dev.dnpm.etl.processor.config.AppConfigProperties import dev.dnpm.etl.processor.monitoring.Report import dev.dnpm.etl.processor.monitoring.Request @@ -49,25 +50,27 @@ class RequestProcessor( ) { fun processMtbFile(mtbFile: MtbFile) { - processMtbFile(mtbFile, UUID.randomUUID().toString()) + processMtbFile(mtbFile, randomRequestId()) } - fun processMtbFile(mtbFile: MtbFile, requestId: String) { - val pid = mtbFile.patient.id + fun processMtbFile(mtbFile: MtbFile, requestId: RequestId) { + val pid = PatientId(mtbFile.patient.id) mtbFile pseudonymizeWith pseudonymizeService mtbFile anonymizeContentWith pseudonymizeService val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile)) + val patientPseudonym = PatientPseudonym(request.mtbFile.patient.id) + requestService.save( Request( - uuid = requestId, - patientId = request.mtbFile.patient.id, - pid = pid, - fingerprint = fingerprint(request.mtbFile), - status = RequestStatus.UNKNOWN, - type = RequestType.MTB_FILE + requestId, + patientPseudonym, + pid, + fingerprint(request.mtbFile), + RequestType.MTB_FILE, + RequestStatus.UNKNOWN ) ) @@ -90,7 +93,7 @@ class RequestProcessor( Instant.now(), responseStatus.status, when (responseStatus.status) { - RequestStatus.WARNING -> Optional.of(responseStatus.body) + RequestStatus.ERROR, RequestStatus.WARNING -> Optional.of(responseStatus.body) else -> Optional.empty() } ) @@ -98,31 +101,33 @@ class RequestProcessor( } private fun isDuplication(pseudonymizedMtbFile: MtbFile): Boolean { + val patientPseudonym = PatientPseudonym(pseudonymizedMtbFile.patient.id) + val lastMtbFileRequestForPatient = - requestService.lastMtbFileRequestForPatientPseudonym(pseudonymizedMtbFile.patient.id) - val isLastRequestDeletion = requestService.isLastRequestWithKnownStatusDeletion(pseudonymizedMtbFile.patient.id) + requestService.lastMtbFileRequestForPatientPseudonym(patientPseudonym) + val isLastRequestDeletion = requestService.isLastRequestWithKnownStatusDeletion(patientPseudonym) return null != lastMtbFileRequestForPatient && !isLastRequestDeletion && lastMtbFileRequestForPatient.fingerprint == fingerprint(pseudonymizedMtbFile) } - fun processDeletion(patientId: String) { - processDeletion(patientId, UUID.randomUUID().toString()) + fun processDeletion(patientId: PatientId) { + processDeletion(patientId, randomRequestId()) } - fun processDeletion(patientId: String, requestId: String) { + fun processDeletion(patientId: PatientId, requestId: RequestId) { try { val patientPseudonym = pseudonymizeService.patientPseudonym(patientId) requestService.save( Request( - uuid = requestId, - patientId = patientPseudonym, - pid = patientId, - fingerprint = fingerprint(patientPseudonym), - status = RequestStatus.UNKNOWN, - type = RequestType.DELETE + requestId, + patientPseudonym, + patientId, + fingerprint(patientPseudonym.value), + RequestType.DELETE, + RequestStatus.UNKNOWN ) ) @@ -144,9 +149,9 @@ class RequestProcessor( requestService.save( Request( uuid = requestId, - patientId = "???", + patientPseudonym = emptyPatientPseudonym(), pid = patientId, - fingerprint = "", + fingerprint = Fingerprint.empty(), status = RequestStatus.ERROR, type = RequestType.DELETE, report = Report("Fehler bei der Pseudonymisierung") @@ -155,14 +160,16 @@ class RequestProcessor( } } - private fun fingerprint(mtbFile: MtbFile): String { + private fun fingerprint(mtbFile: MtbFile): Fingerprint { return fingerprint(objectMapper.writeValueAsString(mtbFile)) } - private fun fingerprint(s: String): String { - return Base32().encodeAsString(DigestUtils.sha256(s)) - .replace("=", "") - .lowercase() + private fun fingerprint(s: String): Fingerprint { + return Fingerprint( + Base32().encodeAsString(DigestUtils.sha256(s)) + .replace("=", "") + .lowercase() + ) } }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt index e0043d2..757b353 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt @@ -19,11 +19,13 @@ package dev.dnpm.etl.processor.services -import dev.dnpm.etl.processor.monitoring.Request -import dev.dnpm.etl.processor.monitoring.RequestRepository -import dev.dnpm.etl.processor.monitoring.RequestStatus -import dev.dnpm.etl.processor.monitoring.RequestType +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.RequestId +import dev.dnpm.etl.processor.monitoring.* +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable import org.springframework.stereotype.Service +import java.util.* @Service class RequestService( @@ -32,15 +34,32 @@ class RequestService( fun save(request: Request) = requestRepository.save(request) - fun allRequestsByPatientPseudonym(patientPseudonym: String) = requestRepository - .findAllByPatientIdOrderByProcessedAtDesc(patientPseudonym) + fun findAll(): Iterable<Request> = requestRepository.findAll() - fun lastMtbFileRequestForPatientPseudonym(patientPseudonym: String) = + fun findAll(pageable: Pageable): Page<Request> = requestRepository.findAll(pageable) + + fun findByUuid(uuid: RequestId): Optional<Request> = + requestRepository.findByUuidEquals(uuid) + + fun findRequestByPatientId(patientPseudonym: PatientPseudonym, pageable: Pageable): Page<Request> = requestRepository.findRequestByPatientPseudonym(patientPseudonym, pageable) + + fun allRequestsByPatientPseudonym(patientPseudonym: PatientPseudonym) = requestRepository + .findAllByPatientPseudonymOrderByProcessedAtDesc(patientPseudonym) + + fun lastMtbFileRequestForPatientPseudonym(patientPseudonym: PatientPseudonym) = Companion.lastMtbFileRequestForPatientPseudonym(allRequestsByPatientPseudonym(patientPseudonym)) - fun isLastRequestWithKnownStatusDeletion(patientPseudonym: String) = + fun isLastRequestWithKnownStatusDeletion(patientPseudonym: PatientPseudonym) = Companion.isLastRequestWithKnownStatusDeletion(allRequestsByPatientPseudonym(patientPseudonym)) + fun countStates(): Iterable<CountedState> = requestRepository.countStates() + + fun countDeleteStates(): Iterable<CountedState> = requestRepository.countDeleteStates() + + fun findPatientUniqueStates(): List<CountedState> = requestRepository.findPatientUniqueStates() + + fun findPatientUniqueDeleteStates(): List<CountedState> = requestRepository.findPatientUniqueDeleteStates() + companion object { fun lastMtbFileRequestForPatientPseudonym(allRequests: List<Request>) = allRequests diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt index 4048348..ecb2ec7 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt @@ -19,8 +19,8 @@ package dev.dnpm.etl.processor.services +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.Report -import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus import org.slf4j.LoggerFactory import org.springframework.context.event.EventListener @@ -31,7 +31,7 @@ import java.util.* @Service class ResponseProcessor( - private val requestRepository: RequestRepository, + private val requestService: RequestService, private val statisticsUpdateProducer: Sinks.Many<Any> ) { @@ -39,7 +39,7 @@ class ResponseProcessor( @EventListener(classes = [ResponseEvent::class]) fun handleResponseEvent(event: ResponseEvent) { - requestRepository.findByUuidEquals(event.requestUuid).ifPresentOrElse({ + requestService.findByUuid(event.requestUuid).ifPresentOrElse({ it.processedAt = event.timestamp it.status = event.status @@ -76,7 +76,7 @@ class ResponseProcessor( } } - requestRepository.save(it) + requestService.save(it) statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) }, { @@ -87,7 +87,7 @@ class ResponseProcessor( } data class ResponseEvent( - val requestUuid: String, + val requestUuid: RequestId, val timestamp: Instant, val status: RequestStatus, val body: Optional<String> = Optional.empty() diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index a29010f..12e824d 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -22,6 +22,7 @@ package dev.dnpm.etl.processor.services.kafka import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.output.asRequestStatus import dev.dnpm.etl.processor.services.ResponseEvent @@ -47,7 +48,7 @@ class KafkaResponseProcessor( Optional.empty() }.ifPresentOrElse({ responseBody -> val event = ResponseEvent( - responseBody.requestId, + RequestId(responseBody.requestId), Instant.ofEpochMilli(data.timestamp()), responseBody.statusCode.asRequestStatus(), when (responseBody.statusCode.asRequestStatus()) { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/types.kt b/src/main/kotlin/dev/dnpm/etl/processor/types.kt new file mode 100644 index 0000000..b2f13ef --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/types.kt @@ -0,0 +1,49 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor + +import java.util.* + +class Fingerprint(val value: String) { + override fun hashCode() = value.hashCode() + + override fun equals(other: Any?) = other is Fingerprint && other.value == value + + companion object { + fun empty() = Fingerprint("") + } +} + +@JvmInline +value class RequestId(val value: String) { + + fun isBlank() = value.isBlank() + +} + +fun randomRequestId() = RequestId(UUID.randomUUID().toString()) + +@JvmInline +value class PatientId(val value: String) + +@JvmInline +value class PatientPseudonym(val value: String) + +fun emptyPatientPseudonym() = PatientPseudonym("")
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt index eb9d541..25ec7cc 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt @@ -27,10 +27,10 @@ import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.security.Role import dev.dnpm.etl.processor.security.UserRole -import dev.dnpm.etl.processor.services.Token -import dev.dnpm.etl.processor.services.TokenService +import dev.dnpm.etl.processor.security.Token +import dev.dnpm.etl.processor.security.TokenService import dev.dnpm.etl.processor.services.TransformationService -import dev.dnpm.etl.processor.services.UserRoleService +import dev.dnpm.etl.processor.security.UserRoleService import org.springframework.beans.factory.annotation.Qualifier import org.springframework.http.MediaType import org.springframework.http.codec.ServerSentEvent @@ -56,7 +56,7 @@ class ConfigController( @GetMapping fun index(model: Model): String { val outputConnectionAvailable = - connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().first().connectionAvailable() + connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().firstOrNull()?.connectionAvailable() val gPasConnectionAvailable = connectionCheckServices.filterIsInstance<GPasConnectionCheckService>().firstOrNull()?.connectionAvailable() @@ -127,10 +127,11 @@ class ConfigController( } else { model.addAttribute("tokensEnabled", true) val result = tokenService.addToken(name) - if (result.isSuccess) { - model.addAttribute("newTokenValue", result.getOrDefault("")) + result.onSuccess { + model.addAttribute("newTokenValue", it) model.addAttribute("success", true) - } else { + } + result.onFailure { model.addAttribute("success", false) } model.addAttribute("tokens", tokenService.findAll()) @@ -182,6 +183,7 @@ class ConfigController( } @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE]) + @ResponseBody fun events(): Flux<ServerSentEvent<Any>> { return connectionCheckUpdateProducer.asFlux().map { val event = when (it) { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt index 6a256aa..54920b1 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt @@ -20,9 +20,10 @@ package dev.dnpm.etl.processor.web import dev.dnpm.etl.processor.NotFoundException +import dev.dnpm.etl.processor.PatientPseudonym +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.ReportService -import dev.dnpm.etl.processor.monitoring.RequestId -import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.services.RequestService import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.web.PageableDefault @@ -35,7 +36,7 @@ import org.springframework.web.bind.annotation.RequestMapping @Controller @RequestMapping(path = ["/"]) class HomeController( - private val requestRepository: RequestRepository, + private val requestService: RequestService, private val reportService: ReportService ) { @@ -44,20 +45,20 @@ class HomeController( @PageableDefault(page = 0, size = 20, sort = ["processedAt"], direction = Sort.Direction.DESC) pageable: Pageable, model: Model ): String { - val requests = requestRepository.findAll(pageable) + val requests = requestService.findAll(pageable) model.addAttribute("requests", requests) return "index" } - @GetMapping(path = ["patient/{patientId}"]) + @GetMapping(path = ["patient/{patientPseudonym}"]) fun byPatient( - @PathVariable patientId: String, + @PathVariable patientPseudonym: PatientPseudonym, @PageableDefault(page = 0, size = 20, sort = ["processedAt"], direction = Sort.Direction.DESC) pageable: Pageable, model: Model ): String { - val requests = requestRepository.findRequestByPatientId(patientId, pageable) - model.addAttribute("patientId", patientId) + val requests = requestService.findRequestByPatientId(patientPseudonym, pageable) + model.addAttribute("patientPseudonym", patientPseudonym.value) model.addAttribute("requests", requests) return "index" @@ -65,7 +66,7 @@ class HomeController( @GetMapping(path = ["/report/{id}"]) fun report(@PathVariable id: RequestId, model: Model): String { - val request = requestRepository.findByUuidEquals(id.toString()).orElse(null) ?: throw NotFoundException() + val request = requestService.findByUuid(id).orElse(null) ?: throw NotFoundException() model.addAttribute("request", request) model.addAttribute("issues", reportService.deserialize(request.report?.dataQualityReport)) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt index daa6af3..c034cb4 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt @@ -19,9 +19,9 @@ package dev.dnpm.etl.processor.web -import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.monitoring.RequestType +import dev.dnpm.etl.processor.services.RequestService import org.springframework.beans.factory.annotation.Qualifier import org.springframework.http.MediaType import org.springframework.http.codec.ServerSentEvent @@ -41,15 +41,15 @@ import java.time.temporal.ChronoUnit class StatisticsRestController( @Qualifier("statisticsUpdateProducer") private val statisticsUpdateProducer: Sinks.Many<Any>, - private val requestRepository: RequestRepository + private val requestService: RequestService ) { @GetMapping(path = ["requeststates"]) fun requestStates(@RequestParam(required = false, defaultValue = "false") delete: Boolean): List<NameValue> { val states = if (delete) { - requestRepository.countDeleteStates() + requestService.countDeleteStates() } else { - requestRepository.countStates() + requestService.countStates() } return states @@ -79,7 +79,7 @@ class StatisticsRestController( } val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("Europe/Berlin")) - val data = requestRepository.findAll() + val data = requestService.findAll() .filter { it.type == requestType } .filter { it.processedAt.isAfter(Instant.now().minus(30, ChronoUnit.DAYS)) } .groupBy { formatter.format(it.processedAt) } @@ -115,9 +115,9 @@ class StatisticsRestController( @GetMapping(path = ["requestpatientstates"]) fun requestPatientStates(@RequestParam(required = false, defaultValue = "false") delete: Boolean): List<NameValue> { val states = if (delete) { - requestRepository.findPatientUniqueDeleteStates() + requestService.findPatientUniqueDeleteStates() } else { - requestRepository.findPatientUniqueStates() + requestService.findPatientUniqueStates() } return states.map { |