From 7440fe1e23e730fd526a814cfde7cc86e105cf70 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 5 Oct 2023 10:51:49 +0200 Subject: Issue #12: Basic implementation of transformation service --- .../processor/services/TransformationService.kt | 62 ++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt (limited to 'src/main') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt new file mode 100644 index 0000000..9be0216 --- /dev/null +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt @@ -0,0 +1,62 @@ +package dev.dnpm.etl.processor.services + +import com.fasterxml.jackson.databind.ObjectMapper +import com.jayway.jsonpath.JsonPath +import com.jayway.jsonpath.PathNotFoundException +import de.ukw.ccc.bwhc.dto.MtbFile + +class TransformationService(private val objectMapper: ObjectMapper) { + fun transform(mtbFile: MtbFile, vararg transformations: Transformation): MtbFile { + var json = objectMapper.writeValueAsString(mtbFile) + + transformations.forEach { transformation -> + val jsonPath = JsonPath.parse(json) + + try { + val before = transformation.path.substringBeforeLast(".") + val last = transformation.path.substringAfterLast(".") + + val existingValue = if (transformation.existingValue is Number) transformation.existingValue else transformation.existingValue.toString() + val newValue = if (transformation.newValue is Number) transformation.newValue else transformation.newValue.toString() + + jsonPath.set("$.$before.[?]$last", newValue, { + it.item(HashMap::class.java)[last] == existingValue + }) + } catch (e: PathNotFoundException) { + // Ignore + } + + json = jsonPath.jsonString() + } + + return objectMapper.readValue(json, MtbFile::class.java) + } + +} + +class Transformation private constructor(internal val path: String) { + + lateinit var existingValue: Any + private set + lateinit var newValue: Any + private set + + infix fun from(value: Any): Transformation { + this.existingValue = value + return this + } + + infix fun to(value: Any): Transformation { + this.newValue = value + return this + } + + companion object { + + fun of(path: String): Transformation { + return Transformation(path) + } + + } + +} -- cgit v1.2.3 From 1e1db1c4d9cf0810056287c8895b1662c16daf2c Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 5 Oct 2023 11:37:10 +0200 Subject: Issue #12: Add application config for transformation configuration --- .../dnpm/etl/processor/config/AppConfigProperties.kt | 11 +++++++++-- .../dev/dnpm/etl/processor/config/AppConfiguration.kt | 8 ++++++++ .../etl/processor/services/TransformationService.kt | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) (limited to 'src/main') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt index 06e730b..6b85603 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfigProperties.kt @@ -24,7 +24,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(AppConfigProperties.NAME) data class AppConfigProperties( var bwhcUri: String?, - var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN + var generator: PseudonymGenerator = PseudonymGenerator.BUILDIN, + var transformations: List = listOf() ) { companion object { const val NAME = "app" @@ -78,4 +79,10 @@ data class KafkaTargetProperties( enum class PseudonymGenerator { BUILDIN, GPAS -} \ No newline at end of file +} + +data class TransformationProperties( + val path: String, + val from: String, + val to: String +) \ No newline at end of file diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 6b15fc0..ccc4b78 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -25,6 +25,7 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService +import dev.dnpm.etl.processor.services.Transformation import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean @@ -71,5 +72,12 @@ class AppConfiguration { return Sinks.many().multicast().directBestEffort() } + @Bean + fun transformations(configProperties: AppConfigProperties): List { + return configProperties.transformations.map { + Transformation.of(it.path) from it.from to it.to + } + } + } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt index 9be0216..f33900d 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt @@ -1,3 +1,22 @@ +/* + * This file is part of ETL-Processor + * + * Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + package dev.dnpm.etl.processor.services import com.fasterxml.jackson.databind.ObjectMapper -- cgit v1.2.3 From 4196664060e879c6159d7986a145f28be3c1798a Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 5 Oct 2023 12:09:56 +0200 Subject: Issue #12: Transform MTBFile objects by using transformation rules --- .../kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt | 10 +++++++--- .../kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt | 3 ++- .../dev/dnpm/etl/processor/services/TransformationService.kt | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'src/main') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index ccc4b78..70ae0f5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -26,6 +26,7 @@ import dev.dnpm.etl.processor.pseudonym.Generator import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService import dev.dnpm.etl.processor.services.Transformation +import dev.dnpm.etl.processor.services.TransformationService import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean @@ -73,10 +74,13 @@ class AppConfiguration { } @Bean - fun transformations(configProperties: AppConfigProperties): List { - return configProperties.transformations.map { + fun transformationService( + objectMapper: ObjectMapper, + configProperties: AppConfigProperties + ): TransformationService { + return TransformationService(objectMapper, configProperties.transformations.map { Transformation.of(it.path) from it.from to it.to - } + }) } } diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt index 3cd912c..fd9a3f5 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/RequestProcessor.kt @@ -38,6 +38,7 @@ import java.util.* @Service class RequestProcessor( private val pseudonymizeService: PseudonymizeService, + private val transformationService: TransformationService, private val sender: MtbFileSender, private val requestService: RequestService, private val objectMapper: ObjectMapper, @@ -50,7 +51,7 @@ class RequestProcessor( mtbFile pseudonymizeWith pseudonymizeService - val request = MtbFileSender.MtbFileRequest(requestId, mtbFile) + val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile)) requestService.save( Request( diff --git a/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt index f33900d..26de550 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/services/TransformationService.kt @@ -24,8 +24,8 @@ import com.jayway.jsonpath.JsonPath import com.jayway.jsonpath.PathNotFoundException import de.ukw.ccc.bwhc.dto.MtbFile -class TransformationService(private val objectMapper: ObjectMapper) { - fun transform(mtbFile: MtbFile, vararg transformations: Transformation): MtbFile { +class TransformationService(private val objectMapper: ObjectMapper, private val transformations: List) { + fun transform(mtbFile: MtbFile): MtbFile { var json = objectMapper.writeValueAsString(mtbFile) transformations.forEach { transformation -> -- cgit v1.2.3 From eb24995ed910e33e9af1de227f6c78f429f55b82 Mon Sep 17 00:00:00 2001 From: Paul-Christian Volkmer Date: Thu, 5 Oct 2023 12:35:29 +0200 Subject: Issue #12: Log transformation count applied on application start --- src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/main') diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt index 70ae0f5..c8e86fb 100644 --- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt +++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt @@ -27,6 +27,7 @@ import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator import dev.dnpm.etl.processor.pseudonym.PseudonymizeService import dev.dnpm.etl.processor.services.Transformation import dev.dnpm.etl.processor.services.TransformationService +import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean @@ -43,6 +44,8 @@ import reactor.core.publisher.Sinks ) class AppConfiguration { + private val logger = LoggerFactory.getLogger(AppConfiguration::class.java) + @ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS") @Bean fun gpasPseudonymGenerator(configProperties: GPasConfigProperties): Generator { @@ -78,6 +81,7 @@ class AppConfiguration { objectMapper: ObjectMapper, configProperties: AppConfigProperties ): TransformationService { + logger.info("Apply ${configProperties.transformations.size} transformation rules") return TransformationService(objectMapper, configProperties.transformations.map { Transformation.of(it.path) from it.from to it.to }) -- cgit v1.2.3