From af6752a5763ff56eacf3b77d2e4c8ca0b1828fe0 Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Tue, 7 May 2024 17:28:44 +0200
Subject: [PATCH 1/2] =?UTF-8?q?Rafra=C3=AEchir=20la=20vue=20mat=C3=A9riali?=
=?UTF-8?q?s=C3=A9e=20et=20vider=20le=20cache.=20fixes=20#10?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
pom.xml | 2 +-
sql/schema.functions.sql | 1 +
.../seasonhandler/MainConfiguration.java | 39 +++++++-
.../seasonhandler/cmd/MainCommand.java | 11 +++
.../seasonhandler/di/DiHelper.java | 2 +-
.../jms/IntegrationDoneProducer.java | 43 +++++++++
.../jms/IntegrationDoneReceiver.java | 92 +++++++++++++++++++
.../jms/SimulationDoneReceiver.java | 7 ++
.../config-evaluation-not-exists.properties | 2 +
.../config-good-with-stages.properties | 2 +
src/test/resources/config-good.properties | 2 +
.../config-invalid-stages.properties | 2 +
.../config-simulation-not-exists.properties | 2 +
13 files changed, 200 insertions(+), 7 deletions(-)
create mode 100644 src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneProducer.java
create mode 100644 src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneReceiver.java
diff --git a/pom.xml b/pom.xml
index 80eddfc..17c2ff7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>fr.agrometinfo</groupId>
<artifactId>season-handler</artifactId>
- <version>2.0.0-alpha-3</version>
+ <version>2.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>AgroMetInfo SEASON handler</name>
<description>SEASON handler for AgroMetInfo</description>
diff --git a/sql/schema.functions.sql b/sql/schema.functions.sql
index 36b8b17..a49bbea 100644
--- a/sql/schema.functions.sql
+++ b/sql/schema.functions.sql
@@ -25,6 +25,7 @@ ON CONFLICT ON CONSTRAINT "UK_dailyvalue" DO
UPDATE SET computedvalue=EXCLUDED.computedvalue, comparedvalue=EXCLUDED.comparedvalue, created=CURRENT_TIMESTAMP;
$$, tablename);
EXECUTE sql;
+ REFRESH MATERIALIZED VIEW v_pra_dailyvalue;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
diff --git a/src/main/java/fr/agrometinfo/seasonhandler/MainConfiguration.java b/src/main/java/fr/agrometinfo/seasonhandler/MainConfiguration.java
index d7e9ce1..b817343 100644
--- a/src/main/java/fr/agrometinfo/seasonhandler/MainConfiguration.java
+++ b/src/main/java/fr/agrometinfo/seasonhandler/MainConfiguration.java
@@ -68,6 +68,10 @@ public class MainConfiguration {
* Properties keys for config.properties.
*/
public enum Property {
+ /**
+ * REST URL to empty cache in the AgroMetInfo web app.
+ */
+ EMPTY_CACHE_REST_URL("rest.empty.cache.rest.url", true),
/**
* File path for evaluation to run.
*/
@@ -138,6 +142,12 @@ public class MainConfiguration {
*/
private final I18n i18n = DiHelper.getI18N();
+ /**
+ * REST URLs to empty cache in the AgroMetInfo web app.
+ */
+ @Getter
+ private final List<String> emptyCacheRestUrls = new ArrayList<>();
+
/**
* Properties from configuration file.
*/
@@ -241,6 +251,18 @@ public class MainConfiguration {
return map;
}
+ /**
+ * @param property a property
+ * @return all suffixes of configured keys
+ */
+ private List<String> getKeySuffixes(final Property property) {
+ return properties.stringPropertyNames().stream() //
+ .filter(p -> p.startsWith(property.getKey())) //
+ .map(p -> p.replace(property.getKey() + ".", "")) //
+ .sorted() //
+ .toList();
+ }
+
/**
* Make some checks on the file and load {@link SimulationProperties}.
*
@@ -315,9 +337,20 @@ public class MainConfiguration {
return Optional.of(i18n.format("error.config.missingkeys", String.join(",", missingKeys)));
}
initPersistenceUnit();
+ initEmptyCacheRestUrls();
return initEvaluations();
}
+ /**
+ * Get all URL to empty cache in AgroMetInfo web app.
+ */
+ private void initEmptyCacheRestUrls() {
+ getKeySuffixes(Property.EMPTY_CACHE_REST_URL).stream() //
+ .map(suffix -> get(Property.SIMULATION_PATH, suffix)) //
+ .filter(v -> v != null && !v.isBlank()) //
+ .forEach(emptyCacheRestUrls::add);
+ }
+
/**
* Make some checks on the file and load {@link Evaluation}.
*
@@ -368,11 +401,7 @@ public class MainConfiguration {
LOGGER.traceEntry();
evaluations.clear();
stages.clear();
- final List<String> suffixes = properties.stringPropertyNames().stream() //
- .filter(p -> p.startsWith(Property.SIMULATION_PATH.getKey())) //
- .map(p -> p.replace(Property.SIMULATION_PATH.getKey() + ".", "")) //
- .sorted() //
- .toList();
+ final List<String> suffixes = getKeySuffixes(Property.SIMULATION_PATH);
for (final String suffix : suffixes) {
LOGGER.trace("Suffix : {}", suffix);
Optional<String> res = initEvaluation(suffix);
diff --git a/src/main/java/fr/agrometinfo/seasonhandler/cmd/MainCommand.java b/src/main/java/fr/agrometinfo/seasonhandler/cmd/MainCommand.java
index 4d47a50..a1da118 100644
--- a/src/main/java/fr/agrometinfo/seasonhandler/cmd/MainCommand.java
+++ b/src/main/java/fr/agrometinfo/seasonhandler/cmd/MainCommand.java
@@ -36,6 +36,8 @@ import org.apache.logging.log4j.core.config.Configurator;
import fr.agrometinfo.seasonhandler.MainConfiguration;
import fr.agrometinfo.seasonhandler.di.DiHelper;
+import fr.agrometinfo.seasonhandler.jms.IntegrationDoneProducer;
+import fr.agrometinfo.seasonhandler.jms.IntegrationDoneReceiver;
import fr.agrometinfo.seasonhandler.jms.SafranReceiver;
import fr.agrometinfo.seasonhandler.jms.SimulationDoneReceiver;
import fr.inrae.agroclim.indicators.resources.I18n;
@@ -127,12 +129,21 @@ public final class MainCommand implements Callable<Integer> {
diHelper.inject(safranReceiver);
safranReceiver.run();
+ final Destination integrationDoneQueue = (Destination) initialContext.lookup("integrationDoneQueueLookup");
+ final IntegrationDoneProducer integrationDoneSender = new IntegrationDoneProducer(jmsContext,
+ integrationDoneQueue);
+
final Destination simulationDoneQueue = (Destination) initialContext.lookup("simulationDoneQueueLookup");
final SimulationDoneReceiver simulationDoneReceiver = new SimulationDoneReceiver(jmsContext,
simulationDoneQueue);
+ simulationDoneReceiver.setIntegrationDoneProducer(integrationDoneSender);
diHelper.inject(simulationDoneReceiver);
simulationDoneReceiver.run();
+ final IntegrationDoneReceiver integrationDoneReceiver = new IntegrationDoneReceiver(jmsContext,
+ integrationDoneQueue);
+ integrationDoneReceiver.run();
+
jmsContext.start();
// Sleep forever, waiting for ourselves to finish
Thread.currentThread().join();
diff --git a/src/main/java/fr/agrometinfo/seasonhandler/di/DiHelper.java b/src/main/java/fr/agrometinfo/seasonhandler/di/DiHelper.java
index c831897..8ddd7b0 100644
--- a/src/main/java/fr/agrometinfo/seasonhandler/di/DiHelper.java
+++ b/src/main/java/fr/agrometinfo/seasonhandler/di/DiHelper.java
@@ -163,7 +163,7 @@ public final class DiHelper {
* @param pm JPA persistence manager
* @return DAO for {@link SimulationSoilDao}.
*/
- private SimulationSoilDao proviceSimulationSoilDao(PersistenceManager pm) {
+ private SimulationSoilDao proviceSimulationSoilDao(final PersistenceManager pm) {
return new SimulationSoilDaoHibernate(pm, config.getDatabaseRoles());
}
diff --git a/src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneProducer.java b/src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneProducer.java
new file mode 100644
index 0000000..c0a4373
--- /dev/null
+++ b/src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneProducer.java
@@ -0,0 +1,43 @@
+package fr.agrometinfo.seasonhandler.jms;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import javax.jms.Destination;
+import javax.jms.JMSContext;
+import javax.jms.JMSProducer;
+import lombok.extern.log4j.Log4j2;
+
+/**
+ * JMS message sender to notify when data integration into AgroMetInfo database is done.
+ *
+ * @author Olivier Maury
+ */
+@Log4j2
+public final class IntegrationDoneProducer {
+
+ /**
+ * The queue used by the SEASON-handler to notify itself.
+ */
+ private final Destination destination;
+
+ /**
+ * JMS producer.
+ */
+ private final JMSProducer producer;
+
+ /**
+ * Constructor.
+ *
+ * @param context JMS Context
+ * @param queue queue to listen
+ */
+ public IntegrationDoneProducer(final JMSContext context, final Destination queue) {
+ producer = context.createProducer();
+ destination = queue;
+ }
+
+ public void send() {
+ var body = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
+ producer.send(destination, body);
+ }
+}
diff --git a/src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneReceiver.java b/src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneReceiver.java
new file mode 100644
index 0000000..6dfacda
--- /dev/null
+++ b/src/main/java/fr/agrometinfo/seasonhandler/jms/IntegrationDoneReceiver.java
@@ -0,0 +1,92 @@
+package fr.agrometinfo.seasonhandler.jms;
+
+import fr.agrometinfo.seasonhandler.MainConfiguration;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.LocalDateTime;
+import javax.jms.Destination;
+import javax.jms.JMSConsumer;
+import javax.jms.JMSContext;
+import javax.jms.JMSException;
+import javax.jms.Message;
+import javax.jms.MessageListener;
+import javax.ws.rs.core.Response;
+import lombok.Setter;
+import lombok.extern.log4j.Log4j2;
+import org.apache.activemq.artemis.jms.client.ActiveMQMessage;
+
+/**
+ * JMS message receiver to empty caches in the web apps.
+ *
+ * @author Olivier Maury
+ */
+@Log4j2
+public final class IntegrationDoneReceiver implements MessageListener, Runnable {
+
+ /**
+ * Consumer of JMS messages.
+ */
+ private final JMSConsumer consumer;
+ /**
+ * The last time the REST end point was called.
+ */
+ private LocalDateTime lastDateTime;
+ /**
+ * Application configuration.
+ */
+ @Setter
+ private MainConfiguration config;
+
+ /**
+ * Constructor.
+ *
+ * @param context JMS Context
+ * @param destination queue to listen
+ */
+ public IntegrationDoneReceiver(final JMSContext context, final Destination destination) {
+ LOGGER.traceEntry();
+ consumer = context.createConsumer(destination);
+ }
+
+ @Override
+ public void onMessage(final Message message) {
+ LOGGER.traceEntry("Message received: {}", message);
+ if (message instanceof final ActiveMQMessage msg) {
+ try {
+ final String body = msg.getBody(String.class);
+ LOGGER.info("Body from received message: {}", body);
+ var parsed = LocalDateTime.parse(body);
+ if (parsed != null && parsed.isAfter(lastDateTime)) {
+ // empty cache
+ HttpClient client = HttpClient.newHttpClient();
+ config.getEmptyCacheRestUrls().forEach(url -> {
+ HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).build();
+ try {
+ HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
+ if (response.statusCode() != Response.Status.OK.getStatusCode()) {
+ LOGGER.error("AgroMetInfo did not response OK at {}: {}", url, response);
+ }
+ } catch (IOException | InterruptedException ex) {
+ LOGGER.error("Something went wrong when calling AgroMetInfo REST end point {}!", url, ex);
+ }
+ });
+ lastDateTime = LocalDateTime.now();
+ }
+ msg.acknowledge();
+ } catch (final JMSException ex) {
+ LOGGER.fatal(ex);
+ }
+ } else {
+ LOGGER.error("Ignoring this message as it is not ActiveMQMessage: {}", message);
+ }
+ }
+
+ @Override
+ public void run() {
+ consumer.setMessageListener(this);
+ }
+
+}
diff --git a/src/main/java/fr/agrometinfo/seasonhandler/jms/SimulationDoneReceiver.java b/src/main/java/fr/agrometinfo/seasonhandler/jms/SimulationDoneReceiver.java
index e37d1ae..71dd4af 100644
--- a/src/main/java/fr/agrometinfo/seasonhandler/jms/SimulationDoneReceiver.java
+++ b/src/main/java/fr/agrometinfo/seasonhandler/jms/SimulationDoneReceiver.java
@@ -72,6 +72,12 @@ public final class SimulationDoneReceiver implements MessageListener, Runnable {
@Setter
private SimulationSoilDao simulationSoilDao;
+ /**
+ * JMS producer when integration is done.
+ */
+ @Setter
+ private IntegrationDoneProducer integrationDoneProducer;
+
/**
* Constructor.
*
@@ -109,6 +115,7 @@ public final class SimulationDoneReceiver implements MessageListener, Runnable {
msg.acknowledge();
amiSimulationDao.setEnded(simulationId);
LOGGER.info("Simulation results in {}.{} handled", schemaName, tableName);
+ integrationDoneProducer.send();
} catch (final JMSException ex) {
LOGGER.fatal(ex);
}
diff --git a/src/test/resources/config-evaluation-not-exists.properties b/src/test/resources/config-evaluation-not-exists.properties
index 5825bab..fc126bf 100644
--- a/src/test/resources/config-evaluation-not-exists.properties
+++ b/src/test/resources/config-evaluation-not-exists.properties
@@ -1,5 +1,7 @@
## AgroMetInfo-SEASON-handler configuration
+rest.empty.cache.rest.url.0 = http://localhost:8080/www-server/rs/application/empty_cache
+
simulation.path.0 = src/test/resources/simulation-evaluation-does-not-exist.properties
## JMS configuration
diff --git a/src/test/resources/config-good-with-stages.properties b/src/test/resources/config-good-with-stages.properties
index 2e44f29..a250405 100644
--- a/src/test/resources/config-good-with-stages.properties
+++ b/src/test/resources/config-good-with-stages.properties
@@ -1,5 +1,7 @@
## AgroMetInfo-SEASON-handler configuration
+rest.empty.cache.rest.url.0 = http://localhost:8080/www-server/rs/application/empty_cache
+
simulation.path.0 = src/test/resources/simulation-good-with-stages.properties
## JMS configuration
diff --git a/src/test/resources/config-good.properties b/src/test/resources/config-good.properties
index de0e3c8..d29a073 100644
--- a/src/test/resources/config-good.properties
+++ b/src/test/resources/config-good.properties
@@ -1,5 +1,7 @@
## AgroMetInfo-SEASON-handler configuration
+rest.empty.cache.rest.url.0 = http://localhost:8080/www-server/rs/application/empty_cache
+
simulation.path.0 = src/test/resources/simulation-good.properties
## JMS configuration
diff --git a/src/test/resources/config-invalid-stages.properties b/src/test/resources/config-invalid-stages.properties
index 361b97a..5171ce7 100644
--- a/src/test/resources/config-invalid-stages.properties
+++ b/src/test/resources/config-invalid-stages.properties
@@ -1,5 +1,7 @@
## AgroMetInfo-SEASON-handler configuration
+rest.empty.cache.rest.url.0 = http://localhost:8080/www-server/rs/application/empty_cache
+
simulation.path.0 = src/test/resources/simulation-invalid-stages.xml
## JMS configuration
diff --git a/src/test/resources/config-simulation-not-exists.properties b/src/test/resources/config-simulation-not-exists.properties
index 61d8b05..8378427 100644
--- a/src/test/resources/config-simulation-not-exists.properties
+++ b/src/test/resources/config-simulation-not-exists.properties
@@ -1,5 +1,7 @@
## AgroMetInfo-SEASON-handler configuration
+rest.empty.cache.rest.url.0 = http://localhost:8080/www-server/rs/application/empty_cache
+
simulation.path.0 = src/test/resources/simulation-does-not-exist.properties
## JMS configuration
--
GitLab
From d8e90ec7082e4deeb12046ec22e375b0d2a3221c Mon Sep 17 00:00:00 2001
From: Olivier Maury <Olivier.Maury@inrae.fr>
Date: Tue, 7 May 2024 17:38:39 +0200
Subject: [PATCH 2/2] Doc
---
src/site/markdown/usage.md | 4 ++++
src/site/resources/images/seq-jms.puml | 10 ++++++++--
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/src/site/markdown/usage.md b/src/site/markdown/usage.md
index ca40c95..7bf7276 100644
--- a/src/site/markdown/usage.md
+++ b/src/site/markdown/usage.md
@@ -37,6 +37,10 @@ A valid example is available in the test sources at `src/test/resources/config-g
```properties
# AgroMetInfo-SEASON-handler configuration
+## AgroMetInfo REST end point
+
+rest.empty.cache.rest.url.0 = http://localhost:8080/www-server/rs/application/empty_cache
+
## SEASON simulations
simulation.path.0 = src/test/resources/simulation-good-with-stages.properties
diff --git a/src/site/resources/images/seq-jms.puml b/src/site/resources/images/seq-jms.puml
index be63fdf..056ed7a 100644
--- a/src/site/resources/images/seq-jms.puml
+++ b/src/site/resources/images/seq-jms.puml
@@ -8,21 +8,24 @@ database "SAFRAN database" as safrandb
queue "agrometinfo-new-safran-data\nqueue" as queuesafran
participant "SEASON-Handler" as handler
queue "simulation-done\ntopic" as topicsimulationdone
+queue "integration-done\nqueue" as queueintegrationdone
database "SEASON database" as seasondb
queue "treatment queue" as queuetreatment
participant "SEASON-cli" as seasoncli
+participant "AgroMetInfo-www" as webapp
== Initialization ==
handler -> queuesafran : subscribe
handler -> topicsimulationdone : subscribe
+handler -> queueintegrationdone : subscribe
seasoncli -> queuetreatment : subscribe
== Process SAFRAN ==
-mf -> agroclim : send SAFRAN data
+mf -> agroclim : SAFRAN data from data.gouv
agroclim -> safrandb : integrate SAFRAN data
agroclim -> queuesafran : publish date of last integrated data
@@ -46,5 +49,8 @@ destroy seasoncli
== Process results ==
topicsimulationdone -> handler : notify end of calculation
-handler -> seasondb : TODO: handle results from SEASON database
+handler -> seasondb : copy results from SEASON database
+handler -> queueintegrationdone : notify end of integration
+queueintegrationdone -> handler : notify end of integration
+handler -> webapp : call empty cache
@enduml
--
GitLab