diff options
Diffstat (limited to 'src/main/kotlin/dev')
19 files changed, 379 insertions, 111 deletions
diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 0ae2c2f..5fc1120 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -20,21 +20,32 @@ package dev.dnpm.etl.processor.config import com.fasterxml.jackson.databind.ObjectMapper -import dev.dnpm.etl.processor.monitoring.* +import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult +import dev.dnpm.etl.processor.monitoring.ConnectionCheckService +import dev.dnpm.etl.processor.monitoring.GPasConnectionCheckService +import dev.dnpm.etl.processor.monitoring.ReportService import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService -import dev.dnpm.etl.processor.services.TokenRepository -import dev.dnpm.etl.processor.services.TokenService +import dev.dnpm.etl.processor.security.TokenRepository +import dev.dnpm.etl.processor.security.TokenService import dev.dnpm.etl.processor.services.Transformation import dev.dnpm.etl.processor.services.TransformationService +import org.apache.hc.client5.http.impl.classic.HttpClients +import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager +import org.apache.hc.client5.http.socket.ConnectionSocketFactory +import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory +import org.apache.hc.core5.http.config.RegistryBuilder import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration +import org.springframework.data.jdbc.repository.config.AbstractJdbcConfiguration +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory import org.springframework.retry.RetryCallback import org.springframework.retry.RetryContext import org.springframework.retry.RetryListener @@ -46,6 +57,13 @@ import org.springframework.security.crypto.password.PasswordEncoder import org.springframework.security.provisioning.InMemoryUserDetailsManager import org.springframework.web.client.RestTemplate import reactor.core.publisher.Sinks +import java.io.BufferedInputStream +import java.io.FileInputStream +import java.security.KeyStore +import java.security.cert.CertificateFactory +import java.security.cert.X509Certificate +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory import kotlin.time.Duration.Companion.seconds import kotlin.time.toJavaDuration @@ -70,8 +88,20 @@ class AppConfiguration { @ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS") @Bean - fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator { - return GpasPseudonymGenerator(configProperties, retryTemplate) + fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator { + try { + if (!configProperties.sslCaLocation.isNullOrBlank()) { + return GpasPseudonymGenerator( + configProperties, + retryTemplate, + createCustomGpasRestTemplate(configProperties) + ) + } + } catch (e: Exception) { + throw RuntimeException(e) + } + + return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate) } @ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "BUILDIN", matchIfMissing = true) @@ -83,8 +113,80 @@ class AppConfiguration { @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS") @ConditionalOnMissingBean @Bean - fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator { - return GpasPseudonymGenerator(configProperties, retryTemplate) + fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate, restTemplate: RestTemplate): Generator { + try { + if (!configProperties.sslCaLocation.isNullOrBlank()) { + return GpasPseudonymGenerator( + configProperties, + retryTemplate, + createCustomGpasRestTemplate(configProperties) + ) + } + } catch (e: Exception) { + throw RuntimeException(e) + } + + return GpasPseudonymGenerator(configProperties, retryTemplate, restTemplate) + } + + private fun createCustomGpasRestTemplate(configProperties: GPasConfigProperties): RestTemplate { + fun getSslContext(certificateLocation: String): SSLContext? { + val ks = KeyStore.getInstance(KeyStore.getDefaultType()) + + val fis = FileInputStream(certificateLocation) + val ca = CertificateFactory.getInstance("X.509") + .generateCertificate(BufferedInputStream(fis)) as X509Certificate + + ks.load(null, null) + ks.setCertificateEntry(1.toString(), ca) + + val tmf = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm() + ) + tmf.init(ks) + + val sslContext = SSLContext.getInstance("TLS") + sslContext.init(null, tmf.trustManagers, null) + + return sslContext + } + + fun getCustomRestTemplate(customSslContext: SSLContext): RestTemplate { + val sslsf = SSLConnectionSocketFactory(customSslContext) + val socketFactoryRegistry = RegistryBuilder.create<ConnectionSocketFactory>() + .register("https", sslsf).register("http", PlainConnectionSocketFactory()).build() + + val connectionManager = BasicHttpClientConnectionManager( + socketFactoryRegistry + ) + val httpClient = HttpClients.custom() + .setConnectionManager(connectionManager).build() + + val requestFactory = HttpComponentsClientHttpRequestFactory( + httpClient + ) + return RestTemplate(requestFactory) + } + + try { + if (!configProperties.sslCaLocation.isNullOrBlank()) { + val customSslContext = getSslContext(configProperties.sslCaLocation) + logger.warn( + String.format( + "%s has been initialized with SSL certificate %s. This is deprecated in favor of including Root CA.", + this.javaClass.name, configProperties.sslCaLocation + ) + ) + + if (customSslContext != null) { + return getCustomRestTemplate(customSslContext) + } + } + } catch (e: Exception) { + throw RuntimeException(e) + } + + throw RuntimeException("Custom SSL configuration for gPAS not usable") } @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "BUILDIN") @@ -173,5 +275,9 @@ class AppConfiguration { return GPasConnectionCheckService(restTemplate, gPasConfigProperties, connectionCheckUpdateProducer) } + @Bean + fun jdbcConfiguration(): AbstractJdbcConfiguration { + return AppJdbcConfiguration() + } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppJdbcConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppJdbcConfiguration.kt new file mode 100644 index 0000000..898982c --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppJdbcConfiguration.kt @@ -0,0 +1,25 @@ +package dev.dnpm.etl.processor.config + +import dev.dnpm.etl.processor.Fingerprint +import org.springframework.context.annotation.Configuration +import org.springframework.core.convert.converter.Converter +import org.springframework.data.jdbc.repository.config.AbstractJdbcConfiguration + +@Configuration +class AppJdbcConfiguration : AbstractJdbcConfiguration() { + override fun userConverters(): MutableList<*> { + return mutableListOf(StringToFingerprintConverter(), FingerprintToStringConverter()) + } +} + +class StringToFingerprintConverter : Converter<String, Fingerprint> { + override fun convert(source: String): Fingerprint { + return Fingerprint(source) + } +} + +class FingerprintToStringConverter : Converter<Fingerprint, String> { + override fun convert(source: Fingerprint): String { + return source.value + } +}
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt index c377555..6b063bd 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppSecurityConfiguration.kt @@ -21,7 +21,7 @@ package dev.dnpm.etl.processor.config import dev.dnpm.etl.processor.security.UserRole import dev.dnpm.etl.processor.security.UserRoleRepository -import dev.dnpm.etl.processor.services.UserRoleService +import dev.dnpm.etl.processor.security.UserRoleService import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -89,7 +89,7 @@ class AppSecurityConfiguration( http { authorizeRequests { authorize("/configs/**", hasRole("ADMIN")) - authorize("/mtbfile/**", hasAnyRole("MTBFILE")) + authorize("/mtbfile/**", hasAnyRole("MTBFILE", "ADMIN", "USER")) authorize("/report/**", hasAnyRole("ADMIN", "USER")) authorize("*.css", permitAll) authorize("*.ico", permitAll) @@ -147,7 +147,7 @@ class AppSecurityConfiguration( http { authorizeRequests { authorize("/configs/**", hasRole("ADMIN")) - authorize("/mtbfile/**", hasAnyRole("MTBFILE")) + authorize("/mtbfile/**", hasAnyRole("MTBFILE", "ADMIN")) authorize("/report/**", hasRole("ADMIN")) authorize(anyRequest, permitAll) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt b/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt index de901ce..b72b1fd 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/input/KafkaInputListener.kt @@ -22,6 +22,7 @@ package dev.dnpm.etl.processor.input import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.services.RequestProcessor import org.apache.kafka.clients.consumer.ConsumerRecord import org.slf4j.LoggerFactory @@ -37,9 +38,9 @@ class KafkaInputListener( val mtbFile = objectMapper.readValue(data.value(), MtbFile::class.java) val firstRequestIdHeader = data.headers().headers("requestId")?.firstOrNull() val requestId = if (null != firstRequestIdHeader) { - String(firstRequestIdHeader.value()) + RequestId(String(firstRequestIdHeader.value())) } else { - "" + RequestId("") } if (mtbFile.consent.status == Consent.Status.ACTIVE) { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt index 81ad922..1afaa32 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ConnectionCheckService.kt @@ -26,22 +26,18 @@ import jakarta.annotation.PostConstruct import org.apache.kafka.clients.consumer.Consumer import org.apache.kafka.common.errors.TimeoutException import org.springframework.beans.factory.annotation.Qualifier -import org.springframework.http.HttpEntity -import org.springframework.http.HttpHeaders -import org.springframework.http.HttpMethod -import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.RequestEntity +import org.springframework.http.* import org.springframework.scheduling.annotation.Scheduled import org.springframework.web.client.RestTemplate import org.springframework.web.util.UriComponentsBuilder import reactor.core.publisher.Sinks +import java.time.Instant import kotlin.time.Duration.Companion.seconds import kotlin.time.toJavaDuration interface ConnectionCheckService { - fun connectionAvailable(): Boolean + fun connectionAvailable(): ConnectionCheckResult } @@ -51,9 +47,27 @@ sealed class ConnectionCheckResult { abstract val available: Boolean - data class KafkaConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult() - data class RestConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult() - data class GPasConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult() + abstract val timestamp: Instant + + abstract val lastChange: Instant + + data class KafkaConnectionCheckResult( + override val available: Boolean, + override val timestamp: Instant, + override val lastChange: Instant + ) : ConnectionCheckResult() + + data class RestConnectionCheckResult( + override val available: Boolean, + override val timestamp: Instant, + override val lastChange: Instant + ) : ConnectionCheckResult() + + data class GPasConnectionCheckResult( + override val available: Boolean, + override val timestamp: Instant, + override val lastChange: Instant + ) : ConnectionCheckResult() } class KafkaConnectionCheckService( @@ -62,25 +76,33 @@ class KafkaConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult> ) : OutputConnectionCheckService { - private var connectionAvailable: Boolean = false - + private var result = ConnectionCheckResult.KafkaConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { - connectionAvailable = try { - null != consumer.listTopics(5.seconds.toJavaDuration()) + result = try { + val available = null != consumer.listTopics(5.seconds.toJavaDuration()) + ConnectionCheckResult.KafkaConnectionCheckResult( + available, + Instant.now(), + if (result.available == available) { result.lastChange } else { Instant.now() } + ) } catch (e: TimeoutException) { - false + ConnectionCheckResult.KafkaConnectionCheckResult( + false, + Instant.now(), + if (!result.available) { result.lastChange } else { Instant.now() } + ) } connectionCheckUpdateProducer.emitNext( - ConnectionCheckResult.KafkaConnectionCheckResult(connectionAvailable), + result, Sinks.EmitFailureHandler.FAIL_FAST ) } - override fun connectionAvailable(): Boolean { - return this.connectionAvailable + override fun connectionAvailable(): ConnectionCheckResult.KafkaConnectionCheckResult { + return this.result } } @@ -92,27 +114,37 @@ class RestConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult> ) : OutputConnectionCheckService { - private var connectionAvailable: Boolean = false + private var result = ConnectionCheckResult.RestConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { - connectionAvailable = try { - restTemplate.getForEntity( + result = try { + val available = restTemplate.getForEntity( restTargetProperties.uri?.replace("/etl/api", "").toString(), String::class.java ).statusCode == HttpStatus.OK + + ConnectionCheckResult.RestConnectionCheckResult( + available, + Instant.now(), + if (result.available == available) { result.lastChange } else { Instant.now() } + ) } catch (e: Exception) { - false + ConnectionCheckResult.RestConnectionCheckResult( + false, + Instant.now(), + if (!result.available) { result.lastChange } else { Instant.now() } + ) } connectionCheckUpdateProducer.emitNext( - ConnectionCheckResult.RestConnectionCheckResult(connectionAvailable), + result, Sinks.EmitFailureHandler.FAIL_FAST ) } - override fun connectionAvailable(): Boolean { - return this.connectionAvailable + override fun connectionAvailable(): ConnectionCheckResult.RestConnectionCheckResult { + return this.result } } @@ -123,12 +155,12 @@ class GPasConnectionCheckService( private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult> ) : ConnectionCheckService { - private var connectionAvailable: Boolean = false + private var result = ConnectionCheckResult.GPasConnectionCheckResult(false, Instant.now(), Instant.now()) @PostConstruct @Scheduled(cron = "0 * * * * *") fun check() { - connectionAvailable = try { + result = try { val uri = UriComponentsBuilder.fromUriString( gPasConfigProperties.uri?.replace("/\$pseudonymizeAllowCreate", "/\$pseudonymize").toString() ) @@ -141,22 +173,33 @@ class GPasConnectionCheckService( if (!gPasConfigProperties.username.isNullOrBlank() && !gPasConfigProperties.password.isNullOrBlank()) { headers.setBasicAuth(gPasConfigProperties.username, gPasConfigProperties.password) } - restTemplate.exchange( + + val available = restTemplate.exchange( uri, HttpMethod.GET, HttpEntity<Void>(headers), Void::class.java ).statusCode == HttpStatus.OK + + ConnectionCheckResult.GPasConnectionCheckResult( + available, + Instant.now(), + if (result.available == available) { result.lastChange } else { Instant.now() } + ) } catch (e: Exception) { - false + ConnectionCheckResult.GPasConnectionCheckResult( + false, + Instant.now(), + if (!result.available) { result.lastChange } else { Instant.now() } + ) } connectionCheckUpdateProducer.emitNext( - ConnectionCheckResult.GPasConnectionCheckResult(connectionAvailable), + result, Sinks.EmitFailureHandler.FAIL_FAST ) } - override fun connectionAvailable(): Boolean { - return this.connectionAvailable + override fun connectionAvailable(): ConnectionCheckResult.GPasConnectionCheckResult { + return this.result } }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt index 97ecd05..062f749 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/ReportService.kt @@ -19,6 +19,7 @@ package dev.dnpm.etl.processor.monitoring +import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonValue import com.fasterxml.jackson.core.JsonParseException @@ -54,7 +55,7 @@ class ReportService( private data class DataQualityReport(val issues: List<Issue>) @JsonIgnoreProperties(ignoreUnknown = true) - data class Issue(val severity: Severity, val message: String) + data class Issue(val severity: Severity, @JsonAlias("details") val message: String) enum class Severity(@JsonValue val value: String) { FATAL("fatal"), diff --git a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt index 028b4a3..9efae4c 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/monitoring/Request.kt @@ -19,10 +19,14 @@ package dev.dnpm.etl.processor.monitoring +import dev.dnpm.etl.processor.Fingerprint +import dev.dnpm.etl.processor.randomRequestId +import dev.dnpm.etl.processor.RequestId import org.springframework.data.annotation.Id import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable import org.springframework.data.jdbc.repository.query.Query +import org.springframework.data.relational.core.mapping.Column import org.springframework.data.relational.core.mapping.Embedded import org.springframework.data.relational.core.mapping.Table import org.springframework.data.repository.CrudRepository @@ -30,26 +34,48 @@ import org.springframework.data.repository.PagingAndSortingRepository import java.time.Instant import java.util.* -typealias RequestId = UUID - @Table("request") data class Request( @Id val id: Long? = null, - val uuid: String = RequestId.randomUUID().toString(), + val uuid: RequestId = randomRequestId(), val patientId: String, val pid: String, - val fingerprint: String, + @Column("fingerprint") + val fingerprint: Fingerprint, val type: RequestType, var status: RequestStatus, var processedAt: Instant = Instant.now(), @Embedded.Nullable var report: Report? = null -) +) { + constructor( + uuid: RequestId, + patientId: String, + pid: String, + fingerprint: Fingerprint, + type: RequestType, + status: RequestStatus + ) : + this(null, uuid, patientId, pid, fingerprint, type, status, Instant.now()) + + constructor( + uuid: RequestId, + patientId: String, + pid: String, + fingerprint: Fingerprint, + type: RequestType, + status: RequestStatus, + processedAt: Instant + ) : + this(null, uuid, patientId, pid, fingerprint, type, status, processedAt) +} +@JvmRecord data class Report( val description: String, val dataQualityReport: String = "" ) +@JvmRecord data class CountedState( val count: Int, val status: RequestStatus, @@ -59,7 +85,7 @@ interface RequestRepository : CrudRepository<Request, Long>, PagingAndSortingRep fun findAllByPatientIdOrderByProcessedAtDesc(patientId: String): List<Request> - fun findByUuidEquals(uuid: String): Optional<Request> + fun findByUuidEquals(uuid: RequestId): Optional<Request> fun findRequestByPatientId(patientId: String, pageable: Pageable): Page<Request> diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt index fc5d617..7b777e8 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/KafkaMtbFileSender.kt @@ -22,6 +22,7 @@ package dev.dnpm.etl.processor.output import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.Consent import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.config.KafkaProperties import dev.dnpm.etl.processor.monitoring.RequestStatus import org.slf4j.LoggerFactory @@ -101,5 +102,5 @@ class KafkaMtbFileSender( return "{\"pid\": \"${request.patientId}\"}" } - data class Data(val requestId: String, val content: MtbFile) + data class Data(val requestId: RequestId, val content: MtbFile) }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt index aca972b..2670f2e 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/output/MtbFileSender.kt @@ -20,6 +20,7 @@ package dev.dnpm.etl.processor.output import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.RequestStatus import org.springframework.http.HttpStatusCode @@ -32,9 +33,9 @@ interface MtbFileSender { data class Response(val status: RequestStatus, val body: String = "") - data class MtbFileRequest(val requestId: String, val mtbFile: MtbFile) + data class MtbFileRequest(val requestId: RequestId, val mtbFile: MtbFile) - data class DeleteRequest(val requestId: String, val patientId: String) + data class DeleteRequest(val requestId: RequestId, val patientId: String) } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/TokenService.kt b/src/main/kotlin/dev/dnpm/etl/processor/security/TokenService.kt index f084408..44b04e8 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/TokenService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/security/TokenService.kt @@ -17,7 +17,7 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -package dev.dnpm.etl.processor.services +package dev.dnpm.etl.processor.security import jakarta.annotation.PostConstruct import org.springframework.data.annotation.Id diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/UserRoleService.kt b/src/main/kotlin/dev/dnpm/etl/processor/security/UserRoleService.kt index 6649f7d..174f8a9 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/UserRoleService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/security/UserRoleService.kt @@ -17,11 +17,8 @@ * along with this program. If not, see <https://www.gnu.org/licenses/>. */ -package dev.dnpm.etl.processor.services +package dev.dnpm.etl.processor.security -import dev.dnpm.etl.processor.security.Role -import dev.dnpm.etl.processor.security.UserRole -import dev.dnpm.etl.processor.security.UserRoleRepository import org.springframework.data.repository.findByIdOrNull import org.springframework.security.core.session.SessionRegistry import org.springframework.security.oauth2.core.oidc.user.OidcUser diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index bdf07cb..94598ae 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -21,6 +21,9 @@ package dev.dnpm.etl.processor.services import com.fasterxml.jackson.databind.ObjectMapper import de.ukw.ccc.bwhc.dto.MtbFile +import dev.dnpm.etl.processor.Fingerprint +import dev.dnpm.etl.processor.randomRequestId +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.config.AppConfigProperties import dev.dnpm.etl.processor.monitoring.Report import dev.dnpm.etl.processor.monitoring.Request @@ -49,10 +52,10 @@ class RequestProcessor( ) { fun processMtbFile(mtbFile: MtbFile) { - processMtbFile(mtbFile, UUID.randomUUID().toString()) + processMtbFile(mtbFile, randomRequestId()) } - fun processMtbFile(mtbFile: MtbFile, requestId: String) { + fun processMtbFile(mtbFile: MtbFile, requestId: RequestId) { val pid = mtbFile.patient.id mtbFile pseudonymizeWith pseudonymizeService @@ -62,12 +65,12 @@ class RequestProcessor( requestService.save( Request( - uuid = requestId, - patientId = request.mtbFile.patient.id, - pid = pid, - fingerprint = fingerprint(request.mtbFile), - status = RequestStatus.UNKNOWN, - type = RequestType.MTB_FILE + requestId, + request.mtbFile.patient.id, + pid, + fingerprint(request.mtbFile), + RequestType.MTB_FILE, + RequestStatus.UNKNOWN ) ) @@ -108,21 +111,21 @@ class RequestProcessor( } fun processDeletion(patientId: String) { - processDeletion(patientId, UUID.randomUUID().toString()) + processDeletion(patientId, randomRequestId()) } - fun processDeletion(patientId: String, requestId: String) { + fun processDeletion(patientId: String, requestId: RequestId) { try { val patientPseudonym = pseudonymizeService.patientPseudonym(patientId) requestService.save( Request( - uuid = requestId, - patientId = patientPseudonym, - pid = patientId, - fingerprint = fingerprint(patientPseudonym), - status = RequestStatus.UNKNOWN, - type = RequestType.DELETE + requestId, + patientPseudonym, + patientId, + fingerprint(patientPseudonym), + RequestType.DELETE, + RequestStatus.UNKNOWN ) ) @@ -146,7 +149,7 @@ class RequestProcessor( uuid = requestId, patientId = "???", pid = patientId, - fingerprint = "", + fingerprint = Fingerprint.empty(), status = RequestStatus.ERROR, type = RequestType.DELETE, report = Report("Fehler bei der Pseudonymisierung") @@ -155,14 +158,16 @@ class RequestProcessor( } } - private fun fingerprint(mtbFile: MtbFile): String { + private fun fingerprint(mtbFile: MtbFile): Fingerprint { return fingerprint(objectMapper.writeValueAsString(mtbFile)) } - private fun fingerprint(s: String): String { - return Base32().encodeAsString(DigestUtils.sha256(s)) - .replace("=", "") - .lowercase() + private fun fingerprint(s: String): Fingerprint { + return Fingerprint( + Base32().encodeAsString(DigestUtils.sha256(s)) + .replace("=", "") + .lowercase() + ) } }
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt index e0043d2..a2e8de3 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestService.kt @@ -19,11 +19,12 @@ package dev.dnpm.etl.processor.services -import dev.dnpm.etl.processor.monitoring.Request -import dev.dnpm.etl.processor.monitoring.RequestRepository -import dev.dnpm.etl.processor.monitoring.RequestStatus -import dev.dnpm.etl.processor.monitoring.RequestType +import dev.dnpm.etl.processor.RequestId +import dev.dnpm.etl.processor.monitoring.* +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable import org.springframework.stereotype.Service +import java.util.* @Service class RequestService( @@ -32,6 +33,15 @@ class RequestService( fun save(request: Request) = requestRepository.save(request) + fun findAll(): Iterable<Request> = requestRepository.findAll() + + fun findAll(pageable: Pageable): Page<Request> = requestRepository.findAll(pageable) + + fun findByUuid(uuid: RequestId): Optional<Request> = + requestRepository.findByUuidEquals(uuid) + + fun findRequestByPatientId(patientId: String, pageable: Pageable): Page<Request> = requestRepository.findRequestByPatientId(patientId, pageable) + fun allRequestsByPatientPseudonym(patientPseudonym: String) = requestRepository .findAllByPatientIdOrderByProcessedAtDesc(patientPseudonym) @@ -41,6 +51,14 @@ class RequestService( fun isLastRequestWithKnownStatusDeletion(patientPseudonym: String) = Companion.isLastRequestWithKnownStatusDeletion(allRequestsByPatientPseudonym(patientPseudonym)) + fun countStates(): Iterable<CountedState> = requestRepository.countStates() + + fun countDeleteStates(): Iterable<CountedState> = requestRepository.countDeleteStates() + + fun findPatientUniqueStates(): List<CountedState> = requestRepository.findPatientUniqueStates() + + fun findPatientUniqueDeleteStates(): List<CountedState> = requestRepository.findPatientUniqueDeleteStates() + companion object { fun lastMtbFileRequestForPatientPseudonym(allRequests: List<Request>) = allRequests diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt index 4048348..ecb2ec7 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/ResponseProcessor.kt @@ -19,8 +19,8 @@ package dev.dnpm.etl.processor.services +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.Report -import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus import org.slf4j.LoggerFactory import org.springframework.context.event.EventListener @@ -31,7 +31,7 @@ import java.util.* @Service class ResponseProcessor( - private val requestRepository: RequestRepository, + private val requestService: RequestService, private val statisticsUpdateProducer: Sinks.Many<Any> ) { @@ -39,7 +39,7 @@ class ResponseProcessor( @EventListener(classes = [ResponseEvent::class]) fun handleResponseEvent(event: ResponseEvent) { - requestRepository.findByUuidEquals(event.requestUuid).ifPresentOrElse({ + requestService.findByUuid(event.requestUuid).ifPresentOrElse({ it.processedAt = event.timestamp it.status = event.status @@ -76,7 +76,7 @@ class ResponseProcessor( } } - requestRepository.save(it) + requestService.save(it) statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST) }, { @@ -87,7 +87,7 @@ class ResponseProcessor( } data class ResponseEvent( - val requestUuid: String, + val requestUuid: RequestId, val timestamp: Instant, val status: RequestStatus, val body: Optional<String> = Optional.empty() diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt index a29010f..12e824d 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/kafka/KafkaResponseProcessor.kt @@ -22,6 +22,7 @@ package dev.dnpm.etl.processor.services.kafka import com.fasterxml.jackson.annotation.JsonAlias import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.ObjectMapper +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.output.asRequestStatus import dev.dnpm.etl.processor.services.ResponseEvent @@ -47,7 +48,7 @@ class KafkaResponseProcessor( Optional.empty() }.ifPresentOrElse({ responseBody -> val event = ResponseEvent( - responseBody.requestId, + RequestId(responseBody.requestId), Instant.ofEpochMilli(data.timestamp()), responseBody.statusCode.asRequestStatus(), when (responseBody.statusCode.asRequestStatus()) { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/types.kt b/src/main/kotlin/dev/dnpm/etl/processor/types.kt new file mode 100644 index 0000000..b41a550 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/types.kt @@ -0,0 +1,41 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <https://www.gnu.org/licenses/>. + */ + +package dev.dnpm.etl.processor + +import java.util.* + +class Fingerprint(val value: String) { + override fun hashCode() = value.hashCode() + + override fun equals(other: Any?) = other is Fingerprint && other.value == value + + companion object { + fun empty() = Fingerprint("") + } +} + +@JvmInline +value class RequestId(val value: String) { + + fun isBlank() = value.isBlank() + +} + +fun randomRequestId() = RequestId(UUID.randomUUID().toString())
\ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt index eb9d541..25ec7cc 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/ConfigController.kt @@ -27,10 +27,10 @@ import dev.dnpm.etl.processor.output.MtbFileSender import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.security.Role import dev.dnpm.etl.processor.security.UserRole -import dev.dnpm.etl.processor.services.Token -import dev.dnpm.etl.processor.services.TokenService +import dev.dnpm.etl.processor.security.Token +import dev.dnpm.etl.processor.security.TokenService import dev.dnpm.etl.processor.services.TransformationService -import dev.dnpm.etl.processor.services.UserRoleService +import dev.dnpm.etl.processor.security.UserRoleService import org.springframework.beans.factory.annotation.Qualifier import org.springframework.http.MediaType import org.springframework.http.codec.ServerSentEvent @@ -56,7 +56,7 @@ class ConfigController( @GetMapping fun index(model: Model): String { val outputConnectionAvailable = - connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().first().connectionAvailable() + connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().firstOrNull()?.connectionAvailable() val gPasConnectionAvailable = connectionCheckServices.filterIsInstance<GPasConnectionCheckService>().firstOrNull()?.connectionAvailable() @@ -127,10 +127,11 @@ class ConfigController( } else { model.addAttribute("tokensEnabled", true) val result = tokenService.addToken(name) - if (result.isSuccess) { - model.addAttribute("newTokenValue", result.getOrDefault("")) + result.onSuccess { + model.addAttribute("newTokenValue", it) model.addAttribute("success", true) - } else { + } + result.onFailure { model.addAttribute("success", false) } model.addAttribute("tokens", tokenService.findAll()) @@ -182,6 +183,7 @@ class ConfigController( } @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE]) + @ResponseBody fun events(): Flux<ServerSentEvent<Any>> { return connectionCheckUpdateProducer.asFlux().map { val event = when (it) { diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt index 6a256aa..ac003d3 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/HomeController.kt @@ -20,9 +20,9 @@ package dev.dnpm.etl.processor.web import dev.dnpm.etl.processor.NotFoundException +import dev.dnpm.etl.processor.RequestId import dev.dnpm.etl.processor.monitoring.ReportService -import dev.dnpm.etl.processor.monitoring.RequestId -import dev.dnpm.etl.processor.monitoring.RequestRepository +import dev.dnpm.etl.processor.services.RequestService import org.springframework.data.domain.Pageable import org.springframework.data.domain.Sort import org.springframework.data.web.PageableDefault @@ -35,7 +35,7 @@ import org.springframework.web.bind.annotation.RequestMapping @Controller @RequestMapping(path = ["/"]) class HomeController( - private val requestRepository: RequestRepository, + private val requestService: RequestService, private val reportService: ReportService ) { @@ -44,7 +44,7 @@ class HomeController( @PageableDefault(page = 0, size = 20, sort = ["processedAt"], direction = Sort.Direction.DESC) pageable: Pageable, model: Model ): String { - val requests = requestRepository.findAll(pageable) + val requests = requestService.findAll(pageable) model.addAttribute("requests", requests) return "index" @@ -56,7 +56,7 @@ class HomeController( @PageableDefault(page = 0, size = 20, sort = ["processedAt"], direction = Sort.Direction.DESC) pageable: Pageable, model: Model ): String { - val requests = requestRepository.findRequestByPatientId(patientId, pageable) + val requests = requestService.findRequestByPatientId(patientId, pageable) model.addAttribute("patientId", patientId) model.addAttribute("requests", requests) @@ -65,7 +65,7 @@ class HomeController( @GetMapping(path = ["/report/{id}"]) fun report(@PathVariable id: RequestId, model: Model): String { - val request = requestRepository.findByUuidEquals(id.toString()).orElse(null) ?: throw NotFoundException() + val request = requestService.findByUuid(id).orElse(null) ?: throw NotFoundException() model.addAttribute("request", request) model.addAttribute("issues", reportService.deserialize(request.report?.dataQualityReport)) diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt index daa6af3..c034cb4 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt @@ -19,9 +19,9 @@ package dev.dnpm.etl.processor.web -import dev.dnpm.etl.processor.monitoring.RequestRepository import dev.dnpm.etl.processor.monitoring.RequestStatus import dev.dnpm.etl.processor.monitoring.RequestType +import dev.dnpm.etl.processor.services.RequestService import org.springframework.beans.factory.annotation.Qualifier import org.springframework.http.MediaType import org.springframework.http.codec.ServerSentEvent @@ -41,15 +41,15 @@ import java.time.temporal.ChronoUnit class StatisticsRestController( @Qualifier("statisticsUpdateProducer") private val statisticsUpdateProducer: Sinks.Many<Any>, - private val requestRepository: RequestRepository + private val requestService: RequestService ) { @GetMapping(path = ["requeststates"]) fun requestStates(@RequestParam(required = false, defaultValue = "false") delete: Boolean): List<NameValue> { val states = if (delete) { - requestRepository.countDeleteStates() + requestService.countDeleteStates() } else { - requestRepository.countStates() + requestService.countStates() } return states @@ -79,7 +79,7 @@ class StatisticsRestController( } val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("Europe/Berlin")) - val data = requestRepository.findAll() + val data = requestService.findAll() .filter { it.type == requestType } .filter { it.processedAt.isAfter(Instant.now().minus(30, ChronoUnit.DAYS)) } .groupBy { formatter.format(it.processedAt) } @@ -115,9 +115,9 @@ class StatisticsRestController( @GetMapping(path = ["requestpatientstates"]) fun requestPatientStates(@RequestParam(required = false, defaultValue = "false") delete: Boolean): List<NameValue> { val states = if (delete) { - requestRepository.findPatientUniqueDeleteStates() + requestService.findPatientUniqueDeleteStates() } else { - requestRepository.findPatientUniqueStates() + requestService.findPatientUniqueStates() } return states.map { |