(Near) realtime update of statistics charts

6 files changed, 202 insertions, 100 deletions

diff --git a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt
index ed3be5d..e4a97ee 100644
--- a/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt
+++ b/src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt
@@ -27,12 +27,16 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
 import dev.dnpm.etl.processor.pseudonym.Generator
 import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
 import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
+import org.reactivestreams.Publisher
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.kafka.core.KafkaTemplate
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Sinks
 import java.net.URI
+import java.time.Duration
 
 @Configuration
 @EnableConfigurationProperties(
@@ -78,5 +82,10 @@ class AppConfiguration {
         return KafkaMtbFileSender(kafkaTemplate, objectMapper)
     }
 
+    @Bean
+    fun statisticsUpdateProducer(): Sinks.Many<Any> {
+        return Sinks.many().multicast().directBestEffort()
+    }
+
 }
 
diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt
index 835f3de..9cbb52a 100644
--- a/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt
+++ b/src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt
@@ -34,13 +34,15 @@ import org.springframework.http.ResponseEntity
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RestController
+import reactor.core.publisher.Sinks
 
 @RestController
 class MtbFileController(
     private val pseudonymizeService: PseudonymizeService,
     private val senders: List<MtbFileSender>,
     private val requestRepository: RequestRepository,
-    private val objectMapper: ObjectMapper
+    private val objectMapper: ObjectMapper,
+    private val statisticsUpdateProducer: Sinks.Many<Any>
 ) {
 
     private val logger = LoggerFactory.getLogger(MtbFileController::class.java)
@@ -63,6 +65,7 @@ class MtbFileController(
                     report = Report("Duplikat erkannt - keine Daten weitergeleitet")
                 )
             )
+            statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST)
             return ResponseEntity.noContent().build()
         }
 
@@ -110,6 +113,8 @@ class MtbFileController(
             )
         )
 
+        statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST)
+
         return if (requestStatus == RequestStatus.ERROR) {
             ResponseEntity.unprocessableEntity().build()
         } else {
diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt
index 05b84ff..adc1e2b 100644
--- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt
+++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt
@@ -20,15 +20,18 @@
 package dev.dnpm.etl.processor.web
 
 import org.springframework.stereotype.Controller
+import org.springframework.ui.Model
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RequestMapping
+import java.time.Instant
 
 @Controller
 @RequestMapping(path = ["/statistics"])
 class StatisticsController {
 
     @GetMapping
-    fun index(): String {
+    fun index(model: Model): String {
+        model.addAttribute("now", Instant.now())
         return "statistics"
     }
 
diff --git a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt
index 8d5cb0e..2741fd3 100644
--- a/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt
+++ b/src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt
@@ -21,9 +21,15 @@ package dev.dnpm.etl.processor.web
 
 import dev.dnpm.etl.processor.monitoring.RequestRepository
 import dev.dnpm.etl.processor.monitoring.RequestStatus
+import org.reactivestreams.Publisher
+import org.springframework.http.MediaType
+import org.springframework.http.codec.ServerSentEvent
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.RequestMapping
 import org.springframework.web.bind.annotation.RestController
+import reactor.core.publisher.Flux
+import reactor.core.publisher.Sinks
+import reactor.kotlin.core.publisher.toFlux
 import java.time.Instant
 import java.time.Month
 import java.time.ZoneId
@@ -34,6 +40,7 @@ import java.time.temporal.TemporalUnit
 @RestController
 @RequestMapping(path = ["/statistics"])
 class StatisticsRestController(
+    private val statisticsUpdateProducer: Sinks.Many<Any>,
     private val requestRepository: RequestRepository
 ) {
 
@@ -68,13 +75,15 @@ class StatisticsRestController(
                     .toMap()
                 Pair(
                     it.key.toString(),
-                    DateNameValues(it.key.toString(), NameValues(
-                        error = requestList[RequestStatus.ERROR] ?: 0,
-                        warning = requestList[RequestStatus.WARNING] ?: 0,
-                        success = requestList[RequestStatus.SUCCESS] ?: 0,
-                        duplication = requestList[RequestStatus.DUPLICATION] ?: 0,
-                        unknown = requestList[RequestStatus.UNKNOWN] ?: 0,
-                    ))
+                    DateNameValues(
+                        it.key.toString(), NameValues(
+                            error = requestList[RequestStatus.ERROR] ?: 0,
+                            warning = requestList[RequestStatus.WARNING] ?: 0,
+                            success = requestList[RequestStatus.SUCCESS] ?: 0,
+                            duplication = requestList[RequestStatus.DUPLICATION] ?: 0,
+                            unknown = requestList[RequestStatus.UNKNOWN] ?: 0,
+                        )
+                    )
                 )
             }.toMap()
 
@@ -86,10 +95,33 @@ class StatisticsRestController(
             .sortedBy { it.date }
     }
 
+    @GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
+    fun updater(): Flux<ServerSentEvent<Any>> {
+        return statisticsUpdateProducer.asFlux().flatMap {
+            Flux.fromIterable(
+                listOf(
+                    ServerSentEvent.builder<Any>()
+                        .event("requeststates").id("none").data(this.requestStates())
+                        .build(),
+                    ServerSentEvent.builder<Any>()
+                        .event("requestslastmonth").id("none").data(this.requestsLastMonth())
+                        .build()
+                )
+            )
+
+        }
+    }
+
 }
 
 data class NameValue(val name: String, val value: Int, val color: String)
 
 data class DateNameValues(val date: String, val nameValues: NameValues)
 
-data class NameValues(val error: Int = 0, val warning: Int = 0, val success: Int = 0, val duplication: Int = 0, val unknown: Int = 0)
\ No newline at end of file
+data class NameValues(
+    val error: Int = 0,
+    val warning: Int = 0,
+    val success: Int = 0,
+    val duplication: Int = 0,
+    val unknown: Int = 0
+)
\ No newline at end of file
diff --git a/src/main/resources/static/scripts.js b/src/main/resources/static/scripts.js
index 8f08ee5..2285167 100644
--- a/src/main/resources/static/scripts.js
+++ b/src/main/resources/static/scripts.js
@@ -13,101 +13,144 @@ window.onload = () => {
     });
 };
 
-function drawPieChart(url, elemId, title) {
-    fetch(url)
-        .then(resp => resp.json())
-        .then(data => {
-            let chartDom = document.getElementById(elemId);
-            let chart = echarts.init(chartDom);
-            let option= {
-                title: {
-                    text: title,
-                    left: 'center'
-                },
-                tooltip: {
-                    trigger: 'item'
-                },
-                color: data.map(i => i.color),
-                series: [
-                    {
-                        type: 'pie',
-                        radius: ['40%', '70%'],
-                        avoidLabelOverlap: false,
-                        label: {
-                            show: false,
-                            position: 'center'
-                        },
-                        labelLine: {
-                            show: false
-                        },
-                        data: data
-                    }
-                ]
-            };
+function drawPieChart(url, elemId, title, data) {
+    if (data) {
+        update(elemId, data);
+    } else {
+        fetch(url)
+            .then(resp => resp.json())
+            .then(d => {
+                draw(elemId, title, d);
+                update(elemId, d);
+            });
+    }
+
+    function update(elemId, data) {
+        let chartDom = document.getElementById(elemId);
+        let chart = echarts.init(chartDom);
+
+        let option = {
+            color: data.map(i => i.color),
+            animationDuration: 250,
+            animationDurationUpdate: 250,
+            series: [
+                {
+                    type: 'pie',
+                    radius: ['40%', '70%'],
+                    avoidLabelOverlap: false,
+                    label: {
+                        show: false,
+                        position: 'center'
+                    },
+                    labelLine: {
+                        show: false
+                    },
+                    data: data
+                }
+            ]
+        };
 
-            option && chart.setOption(option);
-        });
+        option && chart.setOption(option);
+    }
 
+    function draw(elemId, title, data) {
+        let chartDom = document.getElementById(elemId);
+        let chart = echarts.init(chartDom);
+        let option= {
+            title: {
+                text: title,
+                left: 'center'
+            },
+            tooltip: {
+                trigger: 'item'
+            },
+            color: data.map(i => i.color),
+            animationDuration: 250,
+            animationDurationUpdate: 250
+        };
+
+        option && chart.setOption(option);
+    }
 }
 
-function drawBarChart(url, elemId, title) {
-    fetch(url)
-        .then(resp => resp.json())
-        .then(data => {
-            let chartDom = document.getElementById(elemId);
-            let chart = echarts.init(chartDom);
-            let option= {
-                title: {
-                    text: title,
-                    left: 'center'
+function drawBarChart(url, elemId, title, data) {
+    if (data) {
+        update(elemId, data);
+    } else {
+        fetch(url)
+            .then(resp => resp.json())
+            .then(data => {
+                draw(elemId, title, data);
+                update(elemId, data);
+            });
+    }
+
+    function update(elemId, data) {
+        let chartDom = document.getElementById(elemId);
+        let chart = echarts.init(chartDom);
+
+        let option = {
+            series: [
+                {
+                    name: 'UNKNOWN',
+                    type: 'bar',
+                    stack: 'total',
+                    data: data.map(i => i.nameValues.unknown)
                 },
-                xAxis: {
-                    type: 'category',
-                    data: data.map(i => dateFormat.format(Date.parse(i.date)))
+                {
+                    name: 'ERROR',
+                    type: 'bar',
+                    stack: 'total',
+                    data: data.map(i => i.nameValues.error)
                 },
-                yAxis: {
-                    type: 'value',
-                    minInterval: 2,
+                {
+                    name: 'WARNING',
+                    type: 'bar',
+                    stack: 'total',
+                    data: data.map(i => i.nameValues.warning)
                 },
-                tooltip: {
-                    trigger: 'item'
+                {
+                    name: 'SUCCESS',
+                    type: 'bar',
+                    stack: 'total',
+                    data: data.map(i => i.nameValues.success)
                 },
-                animation: false,
-                color: ['slategray', 'red', 'darkorange', 'green', 'slategray'],
-                series: [
-                    {
-                        name: 'UNKNOWN',
-                        type: 'bar',
-                        stack: 'total',
-                        data: data.map(i => i.nameValues.unknown)
-                    },
-                    {
-                        name: 'ERROR',
-                        type: 'bar',
-                        stack: 'total',
-                        data: data.map(i => i.nameValues.error)
-                    },
-                    {
-                        name: 'WARNING',
-                        type: 'bar',
-                        stack: 'total',
-                        data: data.map(i => i.nameValues.warning)
-                    },
-                    {
-                        name: 'SUCCESS',
-                        type: 'bar',
-                        stack: 'total',
-                        data: data.map(i => i.nameValues.success)
-                    },
-                    {
-                        name: 'DUPLICATION',
-                        type: 'bar',
-                        stack: 'total',
-                        data: data.map(i => i.nameValues.duplication)
-                    }
-                ]
-            };
+                {
+                    name: 'DUPLICATION',
+                    type: 'bar',
+                    stack: 'total',
+                    data: data.map(i => i.nameValues.duplication)
+                }
+            ]
+        };
+
+        option && chart.setOption(option);
+    }
+
+    function draw(elemId, title, data) {
+        let chartDom = document.getElementById(elemId);
+        let chart = echarts.init(chartDom);
+        let option= {
+            title: {
+                text: title,
+                left: 'center'
+            },
+            xAxis: {
+                type: 'category',
+                data: data.map(i => dateFormat.format(Date.parse(i.date)))
+            },
+            yAxis: {
+                type: 'value',
+                minInterval: 1
+            },
+            tooltip: {
+                trigger: 'item'
+            },
+            color: ['slategray', 'red', 'darkorange', 'green', 'slategray'],
+            animationDuration: 250,
+            animationDurationUpdate: 250
+        };
 
-            option && chart.setOption(option);
-        });
+        option && chart.setOption(option);
+    }
 }
\ No newline at end of file
diff --git a/src/main/resources/templates/statistics.html b/src/main/resources/templates/statistics.html
index 3f2a11b..007303e 100644
--- a/src/main/resources/templates/statistics.html
+++ b/src/main/resources/templates/statistics.html
@@ -17,8 +17,18 @@
     <script th:src="@{/echarts.min.js}"></script>
     <script th:src="@{/scripts.js}"></script>
     <script>
-        drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung der Anfragen');
-        drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen des letzten Monats');
+        window.onload = () => {
+            drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung aller Anfragen');
+            drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen der letzten 30 Tage');
+
+            const eventSource = new EventSource('statistics/events');
+            eventSource.addEventListener('requeststates', event => {
+                drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung aller Anfragen', JSON.parse(event.data));
+            });
+            eventSource.addEventListener('requestslastmonth', event => {
+                drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen des letzten Monats', JSON.parse(event.data));
+            });
+        }
     </script>
 </body>
 </html>
\ No newline at end of file