From 1fcac8885fa581ec4b7000d94b4a6b92bb0569fd Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Fri, 18 Feb 2022 12:11:42 +0100
Subject: [PATCH 01/10] =?UTF-8?q?Verification=20des=20lignes=20dupliqu?=
 =?UTF-8?q?=C3=A9es=20dans=20le=20chargement=20des=20references?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Dans le cas des references récursives, on vérifie aussi que le parent existe (ce qui n'était pas le cas car le checker contenait toutes les références du parent.
---
 ...ckerOnOneVariableComponentLineChecker.java |   2 +-
 .../inra/oresing/checker/DateLineChecker.java |   1 +
 .../fr/inra/oresing/checker/FloatChecker.java |   2 +-
 .../oresing/checker/GroovyLineChecker.java    |   2 +-
 .../inra/oresing/checker/IntegerChecker.java  |   2 +-
 .../InvalidDatasetContentException.java       |   2 +-
 .../oresing/checker/ReferenceLineChecker.java |   1 +
 .../checker/RegularExpressionChecker.java     |   2 +-
 .../fr/inra/oresing/model/ReferenceDatum.java |  12 +-
 .../rest/ConfigurationParsingResult.java      |   1 +
 .../fr/inra/oresing/rest/OreSiService.java    | 159 ++++++--------
 .../DateValidationCheckResult.java            |   4 +-
 .../DefaultValidationCheckResult.java         |   3 +-
 .../DuplicationLineValidationCheckResult.java |  52 +++++
 ...issingParentLineValidationCheckResult.java |  38 ++++
 .../ReferenceValidationCheckResult.java       |   3 +-
 .../java/fr/inra/oresing/rest/Fixtures.java   |  14 ++
 .../inra/oresing/rest/OreSiResourcesTest.java | 194 +++++++++++++++++-
 .../data/duplication/duplication.yaml         |  72 +++++++
 .../resources/data/duplication/typezone.csv   |   3 +
 .../data/duplication/typezoneduplique.csv     |   5 +
 .../resources/data/duplication/zone_etude.csv |   3 +
 .../zone_etude_dupliqu\303\251.csv"           |   4 +
 .../duplication/zone_etude_missing_parent.csv |   4 +
 ui2/src/locales/en.json                       |   4 +
 ui2/src/locales/fr.json                       |   4 +
 26 files changed, 488 insertions(+), 105 deletions(-)
 rename src/main/java/fr/inra/oresing/{checker => rest/validationcheckresults}/DateValidationCheckResult.java (94%)
 rename src/main/java/fr/inra/oresing/rest/{ => validationcheckresults}/DefaultValidationCheckResult.java (90%)
 create mode 100644 src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
 create mode 100644 src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java
 rename src/main/java/fr/inra/oresing/{checker => rest/validationcheckresults}/ReferenceValidationCheckResult.java (90%)
 create mode 100644 src/test/resources/data/duplication/duplication.yaml
 create mode 100644 src/test/resources/data/duplication/typezone.csv
 create mode 100644 src/test/resources/data/duplication/typezoneduplique.csv
 create mode 100644 src/test/resources/data/duplication/zone_etude.csv
 create mode 100644 "src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv"
 create mode 100644 src/test/resources/data/duplication/zone_etude_missing_parent.csv

diff --git a/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java b/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java
index 336f62251..c4e80bc2d 100644
--- a/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/CheckerOnOneVariableComponentLineChecker.java
@@ -5,7 +5,7 @@ import fr.inra.oresing.model.Datum;
 import fr.inra.oresing.model.ReferenceColumn;
 import fr.inra.oresing.model.ReferenceDatum;
 import fr.inra.oresing.model.VariableComponentKey;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import fr.inra.oresing.rest.ValidationCheckResult;
 import fr.inra.oresing.transformer.LineTransformer;
 import org.assertj.core.util.Strings;
diff --git a/src/main/java/fr/inra/oresing/checker/DateLineChecker.java b/src/main/java/fr/inra/oresing/checker/DateLineChecker.java
index 95134a330..03b531702 100644
--- a/src/main/java/fr/inra/oresing/checker/DateLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/DateLineChecker.java
@@ -3,6 +3,7 @@ package fr.inra.oresing.checker;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
 import fr.inra.oresing.rest.ValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DateValidationCheckResult;
 import fr.inra.oresing.transformer.LineTransformer;
 
 import java.time.format.DateTimeFormatter;
diff --git a/src/main/java/fr/inra/oresing/checker/FloatChecker.java b/src/main/java/fr/inra/oresing/checker/FloatChecker.java
index 16aed69b9..107f9b343 100644
--- a/src/main/java/fr/inra/oresing/checker/FloatChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/FloatChecker.java
@@ -2,7 +2,7 @@ package fr.inra.oresing.checker;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import fr.inra.oresing.rest.ValidationCheckResult;
 import fr.inra.oresing.transformer.LineTransformer;
 
diff --git a/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java b/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java
index fb29c552e..fa2444058 100644
--- a/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/GroovyLineChecker.java
@@ -6,7 +6,7 @@ import fr.inra.oresing.groovy.GroovyExpression;
 import fr.inra.oresing.model.Datum;
 import fr.inra.oresing.model.ReferenceDatum;
 import fr.inra.oresing.model.SomethingThatCanProvideEvaluationContext;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import fr.inra.oresing.rest.ValidationCheckResult;
 
 import java.util.Optional;
diff --git a/src/main/java/fr/inra/oresing/checker/IntegerChecker.java b/src/main/java/fr/inra/oresing/checker/IntegerChecker.java
index 48c54f84d..b38d9718f 100644
--- a/src/main/java/fr/inra/oresing/checker/IntegerChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/IntegerChecker.java
@@ -2,7 +2,7 @@ package fr.inra.oresing.checker;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import fr.inra.oresing.rest.ValidationCheckResult;
 import fr.inra.oresing.transformer.LineTransformer;
 
diff --git a/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java b/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java
index c783e9079..a730d3876 100644
--- a/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java
+++ b/src/main/java/fr/inra/oresing/checker/InvalidDatasetContentException.java
@@ -3,7 +3,7 @@ package fr.inra.oresing.checker;
 import com.google.common.collect.*;
 import fr.inra.oresing.OreSiTechnicalException;
 import fr.inra.oresing.rest.CsvRowValidationCheckResult;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import fr.inra.oresing.rest.ValidationCheckResult;
 
 import java.util.List;
diff --git a/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java b/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java
index 5d7f30b2a..76b9a2fa7 100644
--- a/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/ReferenceLineChecker.java
@@ -2,6 +2,7 @@ package fr.inra.oresing.checker;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.rest.validationcheckresults.ReferenceValidationCheckResult;
 import fr.inra.oresing.transformer.LineTransformer;
 
 import java.util.UUID;
diff --git a/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java b/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java
index 3ff89361c..3c19f0874 100644
--- a/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java
+++ b/src/main/java/fr/inra/oresing/checker/RegularExpressionChecker.java
@@ -2,7 +2,7 @@ package fr.inra.oresing.checker;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.collect.ImmutableMap;
-import fr.inra.oresing.rest.DefaultValidationCheckResult;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import fr.inra.oresing.rest.ValidationCheckResult;
 import fr.inra.oresing.transformer.LineTransformer;
 
diff --git a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
index 8ac944ece..3893cd6f7 100644
--- a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
+++ b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
@@ -6,7 +6,7 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 
 public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext {
-
+    private long lineNumber;
     private final Map<ReferenceColumn, String> values;
 
     public ReferenceDatum() {
@@ -34,6 +34,14 @@ public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext
         return map;
     }
 
+    public long getLineNumber() {
+        return lineNumber;
+    }
+
+    public void setLineNumber(long lineNumber) {
+        this.lineNumber = lineNumber;
+    }
+
     public String put(ReferenceColumn string, String value) {
         return values.put(string, value);
     }
@@ -46,4 +54,4 @@ public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext
     public ImmutableMap<String, Object> getEvaluationContext() {
         return ImmutableMap.of("datum", asMap());
     }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java b/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java
index b75f8d5f7..535726fe3 100644
--- a/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java
+++ b/src/main/java/fr/inra/oresing/rest/ConfigurationParsingResult.java
@@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableSet;
 import fr.inra.oresing.groovy.GroovyExpression;
 import fr.inra.oresing.model.Configuration;
 import fr.inra.oresing.model.VariableComponentKey;
+import fr.inra.oresing.rest.validationcheckresults.DefaultValidationCheckResult;
 import lombok.Value;
 
 import javax.annotation.Nullable;
diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index e58048116..6b8ac80b4 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -3,66 +3,23 @@ package fr.inra.oresing.rest;
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Predicate;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMultiset;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MoreCollectors;
-import com.google.common.collect.Ordering;
-import com.google.common.collect.SetMultimap;
-import com.google.common.collect.Sets;
+import com.google.common.collect.*;
 import com.google.common.primitives.Ints;
 import fr.inra.oresing.OreSiTechnicalException;
-import fr.inra.oresing.checker.CheckerFactory;
-import fr.inra.oresing.checker.DateLineChecker;
-import fr.inra.oresing.checker.DateValidationCheckResult;
-import fr.inra.oresing.checker.FloatChecker;
-import fr.inra.oresing.checker.IntegerChecker;
-import fr.inra.oresing.checker.InvalidDatasetContentException;
-import fr.inra.oresing.checker.LineChecker;
-import fr.inra.oresing.checker.ReferenceLineChecker;
-import fr.inra.oresing.checker.ReferenceLineCheckerConfiguration;
-import fr.inra.oresing.checker.ReferenceValidationCheckResult;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.*;
 import fr.inra.oresing.groovy.CommonExpression;
 import fr.inra.oresing.groovy.Expression;
 import fr.inra.oresing.groovy.GroovyContextHelper;
 import fr.inra.oresing.groovy.StringGroovyExpression;
-import fr.inra.oresing.model.Application;
-import fr.inra.oresing.model.Authorization;
-import fr.inra.oresing.model.BinaryFile;
-import fr.inra.oresing.model.BinaryFileDataset;
-import fr.inra.oresing.model.Configuration;
-import fr.inra.oresing.model.Data;
-import fr.inra.oresing.model.Datum;
-import fr.inra.oresing.model.LocalDateTimeRange;
-import fr.inra.oresing.model.ReferenceColumn;
-import fr.inra.oresing.model.ReferenceDatum;
-import fr.inra.oresing.model.ReferenceValue;
-import fr.inra.oresing.model.VariableComponentKey;
+import fr.inra.oresing.model.*;
 import fr.inra.oresing.model.internationalization.Internationalization;
 import fr.inra.oresing.model.internationalization.InternationalizationDisplay;
 import fr.inra.oresing.model.internationalization.InternationalizationReferenceMap;
-import fr.inra.oresing.persistence.AuthenticationService;
-import fr.inra.oresing.persistence.BinaryFileInfos;
-import fr.inra.oresing.persistence.DataRepository;
-import fr.inra.oresing.persistence.DataRow;
-import fr.inra.oresing.persistence.Ltree;
-import fr.inra.oresing.persistence.OreSiRepository;
-import fr.inra.oresing.persistence.ReferenceValueRepository;
-import fr.inra.oresing.persistence.SqlPolicy;
-import fr.inra.oresing.persistence.SqlSchema;
-import fr.inra.oresing.persistence.SqlSchemaForApplication;
-import fr.inra.oresing.persistence.SqlService;
+import fr.inra.oresing.persistence.*;
 import fr.inra.oresing.persistence.roles.OreSiRightOnApplicationRole;
 import fr.inra.oresing.persistence.roles.OreSiUserRole;
+import fr.inra.oresing.rest.validationcheckresults.*;
 import fr.inra.oresing.transformer.TransformerFactory;
 import lombok.Value;
 import lombok.extern.slf4j.Slf4j;
@@ -94,20 +51,7 @@ import java.time.LocalDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -367,11 +311,13 @@ public class OreSiService {
     }
 
     public UUID addReference(Application app, String refType, MultipartFile file) throws IOException {
+        ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue();
         authenticationService.setRoleForClient();
         UUID fileId = storeFile(app, file);
 
         Configuration conf = app.getConfiguration();
         Configuration.ReferenceDescription ref = conf.getReferences().get(refType);
+        final ImmutableMap<String, UUID> storedReferences = referenceValueRepository.getReferenceIdPerKeys(refType);
 
         ImmutableSet<LineChecker> lineCheckers = checkerFactory.getReferenceValidationLineCheckers(app, refType);
         Optional<ReferenceLineChecker> selfLineChecker = lineCheckers.stream()
@@ -387,6 +333,7 @@ public class OreSiService {
         Function<Ltree, Ltree> getHierarchicalReferenceFn;
         Map<Ltree, Ltree> buildedHierarchicalKeys = new HashMap<>();
         Map<Ltree, Ltree> parentreferenceMap = new HashMap<>();
+        Map<Ltree, List<Long>> hierarchicalKeys = new HashMap<>();
         if (toUpdateCompositeReference.isPresent()) {
             Configuration.CompositeReferenceDescription compositeReferenceDescription = toUpdateCompositeReference.get();
             boolean root = Iterables.get(compositeReferenceDescription.getComponents(), 0).getReference().equals(refType);
@@ -410,7 +357,6 @@ public class OreSiService {
             getHierarchicalReferenceFn = (reference) -> reference;
         }
 
-        ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue();
 
         CSVFormat csvFormat = CSVFormat.DEFAULT
                 .withDelimiter(ref.getSeparator())
@@ -423,6 +369,7 @@ public class OreSiService {
             Function<CSVRecord, ReferenceDatum> csvRecordToLineAsMapFn = line -> {
                 Iterator<String> currentHeader = columns.iterator();
                 ReferenceDatum referenceDatum = new ReferenceDatum();
+                referenceDatum.setLineNumber(line.getRecordNumber());
                 line.forEach(value -> {
                     String header = currentHeader.next();
                     ReferenceColumn referenceColumn = new ReferenceColumn(header);
@@ -434,9 +381,9 @@ public class OreSiService {
             List<CsvRowValidationCheckResult> rowErrors = new LinkedList<>();
             Stream<CSVRecord> recordStream = Streams.stream(csvParser);
             if (isRecursive) {
-                recordStream = addMissingReferences(recordStream, selfLineChecker, recursiveComponentDescription, columns, ref, parentreferenceMap);
+                recordStream = addMissingReferences(recordStream, selfLineChecker, recursiveComponentDescription, columns, ref, parentreferenceMap, rowErrors, refType);
+                InvalidDatasetContentException.checkErrorsIsEmpty(rowErrors);
             }
-            List<Ltree> hierarchicalKeys = new LinkedList<>();
             Optional<InternationalizationReferenceMap> internationalizationReferenceMap = Optional.ofNullable(conf)
                     .map(configuration -> conf.getInternationalization())
                     .map(inter -> inter.getReferences())
@@ -464,10 +411,10 @@ public class OreSiService {
                                             .add(referenceId);
                                 }
                             } else {
-                                rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, csvParser.getCurrentLineNumber()));
+                                rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, referenceDatum.getLineNumber()));
                             }
                         });
-                        ReferenceValue e = new ReferenceValue();
+                        final ReferenceValue e = new ReferenceValue();
                         Ltree naturalKey;
                         if (ref.getKeyColumns().isEmpty()) {
                             UUID technicalId = e.getId();
@@ -488,7 +435,9 @@ public class OreSiService {
                                     .map(referenceLineChecker -> referenceLineChecker.getReferenceValues())
                                     .map(values -> values.get(naturalKey.getSql()))
                                     .filter(key -> key != null)
-                                    .ifPresent(key -> e.setId(key));
+                                    .ifPresent(key -> {
+                                        e.setId(key);
+                                    });
                             Ltree parentKey = parentreferenceMap.getOrDefault(recursiveNaturalKey, null);
                             while (parentKey != null) {
                                 recursiveNaturalKey = Ltree.join(parentKey, recursiveNaturalKey);
@@ -510,6 +459,15 @@ public class OreSiService {
                                 getHierarchicalReferenceFn.apply(selfHierarchicalReference);
                         referenceDatum.putAll(InternationalizationDisplay.getDisplays(displayPattern, displayColumns, referenceDatum));
                         buildedHierarchicalKeys.put(naturalKey, hierarchicalKey);
+
+                        /**
+                         * on remplace l'id par celle en base si elle existe
+                         * a noter que pour les references récursives on récupère l'id depuis  referenceLineChecker.getReferenceValues() ce qui revient au même
+                         */
+
+                        if (storedReferences.containsKey(hierarchicalKey)) {
+                            e.setId(storedReferences.get(hierarchicalKey));
+                        }
                         e.setBinaryFile(fileId);
                         e.setReferenceType(refType);
                         e.setHierarchicalKey(hierarchicalKey);
@@ -518,22 +476,22 @@ public class OreSiService {
                         e.setNaturalKey(naturalKey);
                         e.setApplication(app.getId());
                         e.setRefValues(referenceDatum.asMap());
-                        return e;
-                    })
-                    .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql()))
-                    .map(e -> {
-                        if (hierarchicalKeys.contains(e.getHierarchicalKey())) {
-                            /*envoyer un message de warning : le refType avec la clef e.getNaturalKey existe en plusieurs exemplaires
-                            dans le fichier. Seule la première ligne est enregistrée
-                             */
-//                            ValidationCheckResult validationCheckResult = new ValidationCheckResult()
-//                            rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, csvParser.getCurrentLineNumber()));
+                        if (hierarchicalKeys.containsKey(e.getHierarchicalKey())) {
+                            ValidationCheckResult validationCheckResult = new DuplicationLineValidationCheckResult(DuplicationLineValidationCheckResult.FileType.REFERENCES, refType, ValidationLevel.ERROR, e.getHierarchicalKey(), referenceDatum.getLineNumber(), hierarchicalKeys.get(e.getHierarchicalKey()));
+                            rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, referenceDatum.getLineNumber()));
+                            hierarchicalKeys
+                                    .computeIfAbsent(e.getHierarchicalKey(), k -> new LinkedList<>())
+                                    .add(referenceDatum.getLineNumber());
+                            return null;
                         } else {
-                            hierarchicalKeys.add(e.getHierarchicalKey());
+                            hierarchicalKeys
+                                    .computeIfAbsent(e.getHierarchicalKey(), k -> new LinkedList<>())
+                                    .add(referenceDatum.getLineNumber());
+                            return e;
                         }
-                        return e;
                     })
-                    .filter(e -> e != null);
+                    .filter(e -> e != null)
+                    .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql()));
             referenceValueRepository.storeAll(referenceValuesStream);
             InvalidDatasetContentException.checkErrorsIsEmpty(rowErrors);
         }
@@ -541,11 +499,12 @@ public class OreSiService {
         return fileId;
     }
 
-    private Stream<CSVRecord> addMissingReferences(Stream<CSVRecord> recordStream, Optional<ReferenceLineChecker> selfLineChecker, Optional<Configuration.CompositeReferenceComponentDescription> recursiveComponentDescription, ImmutableList<String> columns, Configuration.ReferenceDescription ref, Map<Ltree, Ltree> referenceMap) {
+    private Stream<CSVRecord> addMissingReferences(Stream<CSVRecord> recordStream, Optional<ReferenceLineChecker> selfLineChecker, Optional<Configuration.CompositeReferenceComponentDescription> recursiveComponentDescription, ImmutableList<String> columns, Configuration.ReferenceDescription ref, Map<Ltree, Ltree> referenceMap, List<CsvRowValidationCheckResult> rowErrors, String refType) {
         Integer parentRecursiveIndex = recursiveComponentDescription
                 .map(rcd -> rcd.getParentRecursiveKey())
                 .map(rck -> columns.indexOf(rck))
                 .orElse(null);
+        Map<String, List<Long>> missingParentReferences = new HashMap<>();
         if (parentRecursiveIndex == null || parentRecursiveIndex < 0) {
             return recordStream;
         }
@@ -556,31 +515,45 @@ public class OreSiService {
         List<CSVRecord> collect = recordStream
                 .peek(csvrecord -> {
                     String s = csvrecord.get(parentRecursiveIndex);
+                    String naturalKey = ref.getKeyColumns()
+                            .stream()
+                            .map(kc -> columns.indexOf(kc))
+                            .map(k -> Ltree.escapeToLabel(csvrecord.get(k)))
+                            .collect(Collectors.joining("__"));
+                    if (!referenceUUIDs.containsKey(naturalKey)) {
+                        referenceUUIDs.put(naturalKey, UUID.randomUUID());
+                    }
                     if (!Strings.isNullOrEmpty(s)) {
-                        String naturalKey;
                         try {
                             s = Ltree.escapeToLabel(s);
-                            naturalKey = ref.getKeyColumns()
-                                    .stream()
-                                    .map(kc -> columns.indexOf(kc))
-                                    .map(k -> Ltree.escapeToLabel(csvrecord.get(k)))
-                                    .collect(Collectors.joining("__"));
                         } catch (IllegalArgumentException e) {
                             return;
                         }
                         referenceMap.put(Ltree.fromSql(naturalKey), Ltree.fromUnescapedString(s));
                         if (!referenceUUIDs.containsKey(s)) {
-                            referenceUUIDs.put(s, UUID.randomUUID());
-                        }
-                        if (!referenceUUIDs.containsKey(naturalKey)) {
-                            referenceUUIDs.put(naturalKey, UUID.randomUUID());
+                            final UUID uuid = UUID.randomUUID();
+                            referenceUUIDs.put(s, uuid);
+                            missingParentReferences
+                                    .computeIfAbsent(s, k -> new LinkedList<>())
+                                    .add(csvrecord.getRecordNumber());
                         }
                     }
+                    missingParentReferences.remove(naturalKey);
                     return;
                 })
                 .collect(Collectors.toList());
         selfLineChecker
                 .ifPresent(slc -> slc.setReferenceValues(ImmutableMap.copyOf(referenceUUIDs)));
+
+        if (!missingParentReferences.isEmpty()) {
+            missingParentReferences.entrySet().stream()
+                    .forEach(entry -> {
+                        final String missingParentReference = entry.getKey();
+                        entry.getValue().stream().forEach(
+                                lineNumber -> rowErrors.add(new CsvRowValidationCheckResult(new MissingParentLineValidationCheckResult(lineNumber, refType, missingParentReference, referenceUUIDs.keySet()), lineNumber))
+                        );
+                    });
+        }
         return collect.stream();
     }
 
diff --git a/src/main/java/fr/inra/oresing/checker/DateValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DateValidationCheckResult.java
similarity index 94%
rename from src/main/java/fr/inra/oresing/checker/DateValidationCheckResult.java
rename to src/main/java/fr/inra/oresing/rest/validationcheckresults/DateValidationCheckResult.java
index 66d533df7..8bf361a4f 100644
--- a/src/main/java/fr/inra/oresing/checker/DateValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DateValidationCheckResult.java
@@ -1,7 +1,9 @@
-package fr.inra.oresing.checker;
+package fr.inra.oresing.rest.validationcheckresults;
 
 import com.google.common.collect.ImmutableMap;
 import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.CheckerTarget;
+import fr.inra.oresing.checker.DateLineChecker;
 import fr.inra.oresing.rest.ValidationCheckResult;
 import lombok.Value;
 
diff --git a/src/main/java/fr/inra/oresing/rest/DefaultValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DefaultValidationCheckResult.java
similarity index 90%
rename from src/main/java/fr/inra/oresing/rest/DefaultValidationCheckResult.java
rename to src/main/java/fr/inra/oresing/rest/validationcheckresults/DefaultValidationCheckResult.java
index cbb109f95..66f227a2d 100644
--- a/src/main/java/fr/inra/oresing/rest/DefaultValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DefaultValidationCheckResult.java
@@ -1,7 +1,8 @@
-package fr.inra.oresing.rest;
+package fr.inra.oresing.rest.validationcheckresults;
 
 import com.google.common.collect.ImmutableMap;
 import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.rest.ValidationCheckResult;
 import lombok.Value;
 
 import java.util.Map;
diff --git a/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
new file mode 100644
index 000000000..28c92b7f6
--- /dev/null
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
@@ -0,0 +1,52 @@
+package fr.inra.oresing.rest.validationcheckresults;
+
+import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.persistence.Ltree;
+import fr.inra.oresing.rest.ValidationCheckResult;
+
+import java.util.List;
+import java.util.Map;
+
+public class DuplicationLineValidationCheckResult implements ValidationCheckResult {
+
+    public static String MESSAGE_FOR_REFERENCES ="duplicatedLineInReference";
+    public static String MESSAGE_FOR_DATATYPES ="duplicatedLineInDatatype";
+    ValidationLevel level;
+    String message;
+
+    Map<String, Object> messageParams;
+
+    public DuplicationLineValidationCheckResult(FileType filetype, String file, ValidationLevel level, Ltree hierarchicalKey, long currentLineNumber, List<Long> otherLines) {
+        this.level = level;
+        this.message = FileType.DATATYPE.message;
+        this.messageParams = ImmutableMap.of(
+                    "file", file,
+                "lineNumber", currentLineNumber,
+                "otherLines", otherLines,
+                "duplicateKey", hierarchicalKey.getSql()
+        );
+    }
+
+    @Override
+    public ValidationLevel getLevel() {
+        return level;
+    }
+
+    @Override
+    public String getMessage() {
+        return message;
+    }
+
+    @Override
+    public Map<String, Object> getMessageParams() {
+        return this.messageParams;
+    }
+    public enum FileType{
+        DATATYPE(MESSAGE_FOR_DATATYPES),REFERENCES(MESSAGE_FOR_REFERENCES);
+        String message;
+        FileType(String message) {
+            this.message = message;
+        }
+    };
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java
new file mode 100644
index 000000000..a4723a3ef
--- /dev/null
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/MissingParentLineValidationCheckResult.java
@@ -0,0 +1,38 @@
+package fr.inra.oresing.rest.validationcheckresults;
+
+import com.google.common.collect.ImmutableMap;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.rest.ValidationCheckResult;
+
+import java.util.Map;
+import java.util.Set;
+
+public class MissingParentLineValidationCheckResult implements ValidationCheckResult {
+    public static String MISSING_PARENT_LINE_IN_RECURSIVE_REFERENCE ="missingParentLineInRecursiveReference";
+    Map<String, Object> messageParams;
+
+    public MissingParentLineValidationCheckResult(long lineNumber, String refType, String missingReferencesKey, Set<String> knownReferences ) {
+        super();
+        this.messageParams = ImmutableMap.of(
+                "lineNumber", lineNumber,
+                "references", refType,
+                "missingReferencesKey", missingReferencesKey,
+                "knownReferences", knownReferences
+        );
+    }
+
+    @Override
+    public ValidationLevel getLevel() {
+        return ValidationLevel.ERROR;
+    }
+
+    @Override
+    public String getMessage() {
+        return MISSING_PARENT_LINE_IN_RECURSIVE_REFERENCE;
+    }
+
+    @Override
+    public Map<String, Object> getMessageParams() {
+        return this.messageParams;
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/checker/ReferenceValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/ReferenceValidationCheckResult.java
similarity index 90%
rename from src/main/java/fr/inra/oresing/checker/ReferenceValidationCheckResult.java
rename to src/main/java/fr/inra/oresing/rest/validationcheckresults/ReferenceValidationCheckResult.java
index a565f4331..1af149b85 100644
--- a/src/main/java/fr/inra/oresing/checker/ReferenceValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/ReferenceValidationCheckResult.java
@@ -1,7 +1,8 @@
-package fr.inra.oresing.checker;
+package fr.inra.oresing.rest.validationcheckresults;
 
 import com.google.common.collect.ImmutableMap;
 import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.CheckerTarget;
 import fr.inra.oresing.rest.ValidationCheckResult;
 import lombok.Value;
 
diff --git a/src/test/java/fr/inra/oresing/rest/Fixtures.java b/src/test/java/fr/inra/oresing/rest/Fixtures.java
index 377615bba..6099f2d0f 100644
--- a/src/test/java/fr/inra/oresing/rest/Fixtures.java
+++ b/src/test/java/fr/inra/oresing/rest/Fixtures.java
@@ -468,6 +468,20 @@ public class Fixtures {
         return "/data/pros/donnees_prelevement_pro.csv";
     }
 
+    public String getDuplicatedApplicationConfigurationResourceName() {
+        return "/data/duplication/duplication.yaml";
+    }
+
+    public Map<String, String> getDuplicatedReferentielFiles() {
+        Map<String, String> referentielFiles = new LinkedHashMap<>();
+        referentielFiles.put("typezonewithoutduplication", "/data/duplication/typezone.csv");
+        referentielFiles.put("typezonewithduplication", "/data/duplication/typezoneduplique.csv");
+        referentielFiles.put("zonewithoutduplication", "/data/duplication/zone_etude.csv");
+        referentielFiles.put("zonewithduplication", "/data/duplication/zone_etude_dupliqué.csv");
+        referentielFiles.put("zonewithmissingparent", "/data/duplication/zone_etude_missing_parent.csv");
+        return referentielFiles;
+    }
+
     public String getPhysicoChimieSolsProDataResourceName() {
         return "/data/pros/physico_chimie_sols.csv";
     }
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index 354666ea7..a02c1a9d4 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -6,6 +6,8 @@ import com.google.common.io.Resources;
 import com.jayway.jsonpath.JsonPath;
 import fr.inra.oresing.OreSiNg;
 import fr.inra.oresing.OreSiTechnicalException;
+import fr.inra.oresing.ValidationLevel;
+import fr.inra.oresing.checker.InvalidDatasetContentException;
 import fr.inra.oresing.persistence.AuthenticationService;
 import fr.inra.oresing.persistence.JsonRowMapper;
 import lombok.extern.slf4j.Slf4j;
@@ -13,6 +15,7 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.hamcrest.Matchers;
 import org.hamcrest.core.Is;
+import org.hamcrest.core.IsEqual;
 import org.hamcrest.core.IsNull;
 import org.json.JSONArray;
 import org.junit.Assert;
@@ -36,6 +39,7 @@ import org.springframework.test.web.servlet.MockMvc;
 import org.springframework.test.web.servlet.ResultActions;
 import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
 import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
+import org.springframework.web.util.NestedServletException;
 
 import javax.servlet.http.Cookie;
 import java.io.IOException;
@@ -897,6 +901,194 @@ public class OreSiResourcesTest {
         }
     }
 
+    @Test
+    public void addDuplicatedTest() throws Exception {
+        authenticationService.addUserRightCreateApplication(userId);
+        try (InputStream configurationFile = fixtures.getClass().getResourceAsStream(fixtures.getDuplicatedApplicationConfigurationResourceName())) {
+            MockMultipartFile configuration = new MockMultipartFile("file", "duplicated.yaml", "text/plain", configurationFile);
+            mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated")
+                            .file(configuration)
+                            .cookie(authCookie))
+                    .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
+        }
+        String message;
+
+        //on charge le fichier de type zone d'étude
+        final String typezonewithoutduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("typezonewithoutduplication");
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(typezonewithoutduplicationDuplication)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "type_zone_etude.csv", "text/plain", refStream);
+            message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andReturn().getResponse().getContentAsString();
+        }
+
+        // on vérifie le nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+
+
+        //on recharge le fichier de type zone d'étude
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(typezonewithoutduplicationDuplication)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "type_zone_etude2.csv", "text/plain", refStream);
+            message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andReturn().getResponse().getContentAsString();
+        }
+
+        //il doit toujours y avoir le même nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+
+
+        //on charge le fichier de zone type d'étude avec une duplication
+        final String typezonewithduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("typezonewithduplication");
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(typezonewithduplicationDuplication)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "type_zone_etude_duplicate.csv", "text/plain", refStream);
+            final ResultActions error = mockMvc
+                    .perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie));
+            Assert.fail();
+        } catch (NestedServletException e) {
+            Assert.assertTrue(e.getCause() instanceof InvalidDatasetContentException);
+            final InvalidDatasetContentException invalidDatasetContentException = (InvalidDatasetContentException) e.getCause();
+            final List<CsvRowValidationCheckResult> errors = invalidDatasetContentException.getErrors();
+            Assert.assertEquals(1, errors.size());
+            Assert.assertEquals(4, errors.get(0).getLineNumber());
+            final ValidationCheckResult validationCheckResult = errors.get(0).getValidationCheckResult();
+            Assert.assertEquals(ValidationLevel.ERROR, validationCheckResult.getLevel());
+            Assert.assertEquals("duplicatedLineInDatatype", validationCheckResult.getMessage());
+            final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
+            Assert.assertEquals("types_de_zones_etudes", messageParams.get("file"));
+            Assert.assertEquals(4L, messageParams.get("lineNumber"));
+            Assert.assertArrayEquals(new Long[]{3L, 4L}, ((List) messageParams.get("otherLines")).toArray());
+            Assert.assertEquals("zone20", messageParams.get("duplicateKey"));
+        }
+
+        //il doit toujours y avoir le même nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "types_de_zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+/*
+on test le dépôt d'un fichier récursif
+ */
+
+
+//on charge le fichier de zone d'étude
+        final String zonewithoutduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("zonewithoutduplication");
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithoutduplicationDuplication)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude.csv", "text/plain", refStream);
+            message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andReturn().getResponse().getContentAsString();
+        }
+
+        // on vérifie le nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+
+
+        //on recharge le fichier de zone d'étude
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithoutduplicationDuplication)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude2.csv", "text/plain", refStream);
+            message = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andReturn().getResponse().getContentAsString();
+        }
+
+        //il doit toujours y avoir le même nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+
+
+        //on charge le fichier de zone d'étudeavec une duplication
+        final String zonewithduplicationDuplication = fixtures.getDuplicatedReferentielFiles().get("zonewithduplication");
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithduplicationDuplication)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude_duplicated.csv", "text/plain", refStream);
+            final ResultActions error = mockMvc
+                    .perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie));
+            Assert.fail();
+        } catch (NestedServletException e) {
+            Assert.assertTrue(e.getCause() instanceof InvalidDatasetContentException);
+            final InvalidDatasetContentException invalidDatasetContentException = (InvalidDatasetContentException) e.getCause();
+            final List<CsvRowValidationCheckResult> errors = invalidDatasetContentException.getErrors();
+            Assert.assertEquals(1, errors.size());
+            Assert.assertEquals(4, errors.get(0).getLineNumber());
+            final ValidationCheckResult validationCheckResult = errors.get(0).getValidationCheckResult();
+            Assert.assertEquals(ValidationLevel.ERROR, validationCheckResult.getLevel());
+            Assert.assertEquals("duplicatedLineInDatatype", validationCheckResult.getMessage());
+            final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
+            Assert.assertEquals("zones_etudes", messageParams.get("file"));
+            Assert.assertEquals(4L, messageParams.get("lineNumber"));
+            Assert.assertArrayEquals(new Long[]{2L, 4L}, ((List) messageParams.get("otherLines")).toArray());
+            Assert.assertEquals("site1", messageParams.get("duplicateKey"));
+        }
+
+        //il doit toujours y avoir le même nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+
+        //on charge le fichier de zone d'étudeavec une duplication
+        final String zonewithmissingParent = fixtures.getDuplicatedReferentielFiles().get("zonewithmissingparent");
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(zonewithmissingParent)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "zone_etude_missing_parent.csv", "text/plain", refStream);
+            final ResultActions error = mockMvc
+                    .perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                            .file(refFile)
+                            .cookie(authCookie));
+            Assert.fail();
+        } catch (NestedServletException e) {
+            Assert.assertTrue(e.getCause() instanceof InvalidDatasetContentException);
+            final InvalidDatasetContentException invalidDatasetContentException = (InvalidDatasetContentException) e.getCause();
+            final List<CsvRowValidationCheckResult> errors = invalidDatasetContentException.getErrors();
+            Assert.assertEquals(1, errors.size());
+            Assert.assertEquals(3, errors.get(0).getLineNumber());
+            final ValidationCheckResult validationCheckResult = errors.get(0).getValidationCheckResult();
+            Assert.assertEquals(ValidationLevel.ERROR, validationCheckResult.getLevel());
+            Assert.assertEquals("missingParentLineInRecursiveReference", validationCheckResult.getMessage());
+            final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
+            Assert.assertEquals("zones_etudes", messageParams.get("references"));
+            Assert.assertEquals(3L, messageParams.get("lineNumber"));
+            Assert.assertEquals("site3", messageParams.get("missingReferencesKey"));
+            Assert.assertTrue(Set.of("site3","site1.site2","site1","site2").containsAll((Set) messageParams.get("knownReferences")));
+        }
+
+        //il doit toujours y avoir le même nombre de ligne
+        message = mockMvc.perform(MockMvcRequestBuilders.get("/api/v1/applications/duplicated/references/{refType}", "zones_etudes")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
+                .andReturn().getResponse().getContentAsString();
+
+    }
+
     @Test
     public void addApplicationOLAC() throws Exception {
         authenticationService.addUserRightCreateApplication(userId);
@@ -1010,7 +1202,7 @@ public class OreSiResourcesTest {
         for (Map.Entry<String, String> entry : fixtures.getFluxMeteoForetEssaiDataResourceName().entrySet()) {
             try (InputStream refStream = fixtures.getClass().getResourceAsStream(entry.getValue())) {
                 MockMultipartFile refFile = new MockMultipartFile("file", "flux_meteo_dataResult.csv", "text/plain", refStream);
-                mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/foret/data/"+entry.getKey())
+                mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/foret/data/" + entry.getKey())
                                 .file(refFile)
                                 .cookie(authCookie))
                         .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
diff --git a/src/test/resources/data/duplication/duplication.yaml b/src/test/resources/data/duplication/duplication.yaml
new file mode 100644
index 000000000..b48a4d280
--- /dev/null
+++ b/src/test/resources/data/duplication/duplication.yaml
@@ -0,0 +1,72 @@
+version: 0
+application:
+  name: duplication
+  version: 1
+compositeReferences:
+  localizations:
+    components:
+      - reference: zones_etudes
+        parentRecursiveKey: parent
+references:
+  types_de_zones_etudes:
+    keyColumns: [nom]
+    columns:
+      nom:
+  zones_etudes:
+    validations:
+      parent_ref:
+        description: référence au parent
+        checker:
+          name: Reference
+          params:
+            refType: zones_etudes
+            columns: parent
+            required: false
+            codify: true
+    keyColumns: [nom]
+    columns:
+      nom:
+      parent:
+dataTypes:
+  dty:
+    authorization:
+      dataGroups:
+        reference:
+          data:
+            - localization
+            - Date
+          label: "Reference"
+      authorizationScopes:
+        authorization_zoneEtude:
+          component: zones_etudes
+          variable: localization
+      timeScope:
+        component: day
+        variable: Date
+    data:
+      Date:
+        components:
+          day:
+            checker:
+              name: Date
+              params:
+                pattern: dd/MM/yyyy
+      localization:
+        components:
+          zones_etudes:
+            checker:
+              name: Reference
+              params:
+                refType: zones_etudes
+    format:
+      headerLine: 2
+      firstRowLine: 3
+      columns:
+        - header: "site"
+          boundTo:
+            variable: localization
+            component: zones_etudes
+        - header: "date"
+          boundTo:
+            variable: Date
+            component: day
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/typezone.csv b/src/test/resources/data/duplication/typezone.csv
new file mode 100644
index 000000000..7372a66a4
--- /dev/null
+++ b/src/test/resources/data/duplication/typezone.csv
@@ -0,0 +1,3 @@
+nom;
+zone1;
+zone2;
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/typezoneduplique.csv b/src/test/resources/data/duplication/typezoneduplique.csv
new file mode 100644
index 000000000..a9be259d5
--- /dev/null
+++ b/src/test/resources/data/duplication/typezoneduplique.csv
@@ -0,0 +1,5 @@
+nom;
+zone10;
+zone20;
+zone20;
+zone30;
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/zone_etude.csv b/src/test/resources/data/duplication/zone_etude.csv
new file mode 100644
index 000000000..10a42bca8
--- /dev/null
+++ b/src/test/resources/data/duplication/zone_etude.csv
@@ -0,0 +1,3 @@
+nom;parent;
+site1;;
+site2;site1;
\ No newline at end of file
diff --git "a/src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv" "b/src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv"
new file mode 100644
index 000000000..cc4f7bf7b
--- /dev/null
+++ "b/src/test/resources/data/duplication/zone_etude_dupliqu\303\251.csv"
@@ -0,0 +1,4 @@
+nom;parent;
+site1;;
+site2;site1;
+site1;;
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/zone_etude_missing_parent.csv b/src/test/resources/data/duplication/zone_etude_missing_parent.csv
new file mode 100644
index 000000000..b0bdde17e
--- /dev/null
+++ b/src/test/resources/data/duplication/zone_etude_missing_parent.csv
@@ -0,0 +1,4 @@
+nom;parent;
+site1;;
+site1;site3;
+site2;site1;
\ No newline at end of file
diff --git a/ui2/src/locales/en.json b/ui2/src/locales/en.json
index 54ea173b4..d9b197851 100644
--- a/ui2/src/locales/en.json
+++ b/ui2/src/locales/en.json
@@ -1,4 +1,8 @@
 {
+    "duplicated": {
+        "duplicateLineInReference": "In the references file {file}, line ${lineNumber} has the same identifier ${duplicateKey} as lines ${otherLines}",
+        "duplicateLineInDatatype": "In data file {file}, line ${lineNumber} has the same identifier ${duplicateKey} as lines ${otherLines}"
+    },
     "titles":{
         "login-page":"Welcome to SI-ORE",
         "applications-page":"My applications",
diff --git a/ui2/src/locales/fr.json b/ui2/src/locales/fr.json
index e82d10e87..00e4242c0 100644
--- a/ui2/src/locales/fr.json
+++ b/ui2/src/locales/fr.json
@@ -1,4 +1,8 @@
 {
+    "duplicated": {
+        "duplicatedLineInReference": "Dans le fichier de references {file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}",
+        "duplicatedLineInDatatype": "Dans le fichier de data {file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}"
+    },
     "titles": {
         "login-page": "Bienvenue sur SI-ORE",
         "applications-page": "Mes applications",
-- 
GitLab


From 8277c30d831eeada3d29c41132f859726700784f Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Fri, 18 Feb 2022 15:26:35 +0100
Subject: [PATCH 02/10] Suppression des noeuds null dans la construction du
 path

---
 src/main/java/fr/inra/oresing/rest/OreSiService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 6b8ac80b4..0269462b5 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -518,7 +518,8 @@ public class OreSiService {
                     String naturalKey = ref.getKeyColumns()
                             .stream()
                             .map(kc -> columns.indexOf(kc))
-                            .map(k -> Ltree.escapeToLabel(csvrecord.get(k)))
+                            .map(k -> Strings.isNullOrEmpty(csvrecord.get(k))?null:Ltree.escapeToLabel(csvrecord.get(k)))
+                            .filter(k->k!=null)
                             .collect(Collectors.joining("__"));
                     if (!referenceUUIDs.containsKey(naturalKey)) {
                         referenceUUIDs.put(naturalKey, UUID.randomUUID());
-- 
GitLab


From 346a42b6f355423ed90c6e19db7fe09ac600ad94 Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Fri, 18 Feb 2022 16:47:23 +0100
Subject: [PATCH 03/10] Ajout des messages pour les erreurs de duplication et
 de non existence de la ligne parent

---
 ui2/src/locales/en.json | 9 ++++-----
 ui2/src/locales/fr.json | 9 ++++-----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/ui2/src/locales/en.json b/ui2/src/locales/en.json
index d9b197851..afa7f4175 100644
--- a/ui2/src/locales/en.json
+++ b/ui2/src/locales/en.json
@@ -1,8 +1,4 @@
 {
-    "duplicated": {
-        "duplicateLineInReference": "In the references file {file}, line ${lineNumber} has the same identifier ${duplicateKey} as lines ${otherLines}",
-        "duplicateLineInDatatype": "In data file {file}, line ${lineNumber} has the same identifier ${duplicateKey} as lines ${otherLines}"
-    },
     "titles":{
         "login-page":"Welcome to SI-ORE",
         "applications-page":"My applications",
@@ -147,7 +143,10 @@
         "requiredValueWithColumn": "For column: <code> {target} </code> the value cannot be zero.",
         "timerangeoutofinterval":"The date <code> {value} </code> is incompatible with the date range of the deposit. It must be between <code> {from} </code> and <code> {to} </code>. ",
         "badauthorizationscopeforrepository":"Authorization <code> {authorization} </code>) must be <code> {expectedValue} / code> and not <code> {givenValue} </code>",
-        "overlappingpublishedversion":"There is a deposited version in the deposit dates have an overlapping period with the deposited version. <code> {overlapingFiles] </code>"
+        "overlappingpublishedversion":"There is a deposited version in the deposit dates have an overlapping period with the deposited version. <code> {overlapingFiles] </code>",
+        "duplicateLineInReference": "In the repository file ${file}, line ${lineNumber} has the same id ${duplicateKey} as lines ${otherLines}",
+        "duplicateLineInDatatype": "In data file ${file}, line ${lineNumber} has the same identifier ${duplicateKey} as lines ${otherLines}",
+        "missingParentLineInRecursiveReference": "In repository file ${references} the id value ${missingReferencesKey} for parent does not exist. Accepted values ${knownReferences}"
     },
     "referencesManagement":{
         "actions":"Actions",
diff --git a/ui2/src/locales/fr.json b/ui2/src/locales/fr.json
index 00e4242c0..5d7b3633a 100644
--- a/ui2/src/locales/fr.json
+++ b/ui2/src/locales/fr.json
@@ -1,8 +1,4 @@
 {
-    "duplicated": {
-        "duplicatedLineInReference": "Dans le fichier de references {file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}",
-        "duplicatedLineInDatatype": "Dans le fichier de data {file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}"
-    },
     "titles": {
         "login-page": "Bienvenue sur SI-ORE",
         "applications-page": "Mes applications",
@@ -148,7 +144,10 @@
         "requiredValueWithColumn": "Pour la colonne : <code>{target}</code> la valeur ne peut pas être nulle.",
         "timerangeoutofinterval": "La date  <code>{value}</code> est incompatible avec l'intervale de dates du dépôt. Elle doit être comprise entre <code>{from}</code> et <code>{to}</code>.",
         "badauthorizationscopeforrepository": "L'autorisation <code>{authorization}</code>) doit être <code>{expectedValue}/code> et non <code>{givenValue}</code>",
-        "overlappingpublishedversion": "Il existe une version déposée dans les dates de dépôt ont une période chevauchante avec la version déposée. <code>{overlapingFiles]</code>"
+        "overlappingpublishedversion": "Il existe une version déposée dans les dates de dépôt ont une période chevauchante avec la version déposée. <code>{overlapingFiles]</code>",
+        "duplicatedLineInReference": "Dans le fichier du référentiel ${file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}",
+        "duplicatedLineInDatatype": "Dans le fichier de données ${file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}",
+        "missingParentLineInRecursiveReference": "Dans le fichier du référentiel ${references} la valeur d'identifiant ${missingReferencesKey} pour le  parent n'existe pas. Valeurs acceptées ${knownReferences}"
     },
     "referencesManagement": {
         "actions": "Actions",
-- 
GitLab


From 6e9fccc7dfb4cc5cf0a6b1f57c92af8103a656eb Mon Sep 17 00:00:00 2001
From: Philippe Tcherniatinsky <philippe.tcherniatinsky@inra.fr>
Date: Mon, 21 Feb 2022 14:53:59 +0100
Subject: [PATCH 04/10] Update fr.json (pas de $)

---
 ui2/src/locales/fr.json | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ui2/src/locales/fr.json b/ui2/src/locales/fr.json
index 5d7b3633a..64979f0f5 100644
--- a/ui2/src/locales/fr.json
+++ b/ui2/src/locales/fr.json
@@ -145,9 +145,9 @@
         "timerangeoutofinterval": "La date  <code>{value}</code> est incompatible avec l'intervale de dates du dépôt. Elle doit être comprise entre <code>{from}</code> et <code>{to}</code>.",
         "badauthorizationscopeforrepository": "L'autorisation <code>{authorization}</code>) doit être <code>{expectedValue}/code> et non <code>{givenValue}</code>",
         "overlappingpublishedversion": "Il existe une version déposée dans les dates de dépôt ont une période chevauchante avec la version déposée. <code>{overlapingFiles]</code>",
-        "duplicatedLineInReference": "Dans le fichier du référentiel ${file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}",
-        "duplicatedLineInDatatype": "Dans le fichier de données ${file}, la ligne ${lineNumber} a le même identifiant ${duplicateKey} que les lignes ${otherLines}",
-        "missingParentLineInRecursiveReference": "Dans le fichier du référentiel ${references} la valeur d'identifiant ${missingReferencesKey} pour le  parent n'existe pas. Valeurs acceptées ${knownReferences}"
+        "duplicatedLineInReference": "Dans le fichier du référentiel {file}, la ligne {lineNumber} a le même identifiant {duplicateKey} que les lignes {otherLines}",
+        "duplicatedLineInDatatype": "Dans le fichier de données {file}, la ligne {lineNumber} a le même identifiant {duplicateKey} que les lignes {otherLines}",
+        "missingParentLineInRecursiveReference": "Dans le fichier du référentiel {references} la valeur d'identifiant {missingReferencesKey} pour le  parent n'existe pas. Valeurs acceptées {knownReferences}"
     },
     "referencesManagement": {
         "actions": "Actions",
@@ -231,4 +231,4 @@
         "slash": "/",
         "regEx": ".*"
     }
-}
\ No newline at end of file
+}
-- 
GitLab


From b4140b7c018e6e1f5b7f90e82ec5b0887a99001e Mon Sep 17 00:00:00 2001
From: Philippe Tcherniatinsky <philippe.tcherniatinsky@inra.fr>
Date: Mon, 21 Feb 2022 14:54:48 +0100
Subject: [PATCH 05/10] Update en.json (pas de $)

---
 ui2/src/locales/en.json | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ui2/src/locales/en.json b/ui2/src/locales/en.json
index afa7f4175..4d6182efa 100644
--- a/ui2/src/locales/en.json
+++ b/ui2/src/locales/en.json
@@ -144,9 +144,9 @@
         "timerangeoutofinterval":"The date <code> {value} </code> is incompatible with the date range of the deposit. It must be between <code> {from} </code> and <code> {to} </code>. ",
         "badauthorizationscopeforrepository":"Authorization <code> {authorization} </code>) must be <code> {expectedValue} / code> and not <code> {givenValue} </code>",
         "overlappingpublishedversion":"There is a deposited version in the deposit dates have an overlapping period with the deposited version. <code> {overlapingFiles] </code>",
-        "duplicateLineInReference": "In the repository file ${file}, line ${lineNumber} has the same id ${duplicateKey} as lines ${otherLines}",
-        "duplicateLineInDatatype": "In data file ${file}, line ${lineNumber} has the same identifier ${duplicateKey} as lines ${otherLines}",
-        "missingParentLineInRecursiveReference": "In repository file ${references} the id value ${missingReferencesKey} for parent does not exist. Accepted values ${knownReferences}"
+        "duplicateLineInReference": "In the repository file {file}, line {lineNumber} has the same id {duplicateKey} as lines {otherLines}",
+        "duplicateLineInDatatype": "In data file {file}, line {lineNumber} has the same identifier {duplicateKey} as lines {otherLines}",
+        "missingParentLineInRecursiveReference": "In repository file {references} the id value {missingReferencesKey} for parent does not exist. Accepted values ${knownReferences}"
     },
     "referencesManagement":{
         "actions":"Actions",
@@ -231,4 +231,4 @@
         "regEx": ".*",
         "star": "*"
     }
-}
\ No newline at end of file
+}
-- 
GitLab


From efbde6da23ab20556c144ade6d0f42a184f3d013 Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Mon, 21 Feb 2022 14:59:37 +0100
Subject: [PATCH 06/10] =?UTF-8?q?Utilise=20Multimap=20plut=C3=B4t=20que=20?=
 =?UTF-8?q?Map=20de=20collection?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/java/fr/inra/oresing/rest/OreSiService.java | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 31d35be9a..0013a7fab 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -333,7 +333,7 @@ public class OreSiService {
         Function<Ltree, Ltree> getHierarchicalReferenceFn;
         Map<Ltree, Ltree> buildedHierarchicalKeys = new HashMap<>();
         Map<Ltree, Ltree> parentreferenceMap = new HashMap<>();
-        Map<Ltree, List<Long>> hierarchicalKeys = new HashMap<>();
+        ListMultimap<Ltree, Long> hierarchicalKeys = LinkedListMultimap.create();
         if (toUpdateCompositeReference.isPresent()) {
             Configuration.CompositeReferenceDescription compositeReferenceDescription = toUpdateCompositeReference.get();
             boolean root = Iterables.get(compositeReferenceDescription.getComponents(), 0).getReference().equals(refType);
@@ -480,14 +480,10 @@ public class OreSiService {
                         if (hierarchicalKeys.containsKey(e.getHierarchicalKey())) {
                             ValidationCheckResult validationCheckResult = new DuplicationLineValidationCheckResult(DuplicationLineValidationCheckResult.FileType.REFERENCES, refType, ValidationLevel.ERROR, e.getHierarchicalKey(), referenceDatum.getLineNumber(), hierarchicalKeys.get(e.getHierarchicalKey()));
                             rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, referenceDatum.getLineNumber()));
-                            hierarchicalKeys
-                                    .computeIfAbsent(e.getHierarchicalKey(), k -> new LinkedList<>())
-                                    .add(referenceDatum.getLineNumber());
+                            hierarchicalKeys.put(e.getHierarchicalKey(), referenceDatum.getLineNumber());
                             return null;
                         } else {
-                            hierarchicalKeys
-                                    .computeIfAbsent(e.getHierarchicalKey(), k -> new LinkedList<>())
-                                    .add(referenceDatum.getLineNumber());
+                            hierarchicalKeys.put(e.getHierarchicalKey(), referenceDatum.getLineNumber());
                             return e;
                         }
                     })
-- 
GitLab


From 8ae97a7876fa6fa7293d08257b4c35e2a0443fce Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Mon, 21 Feb 2022 15:22:30 +0100
Subject: [PATCH 07/10] =?UTF-8?q?=C3=89vite=20d'ajouter=20un=20chamsp=20da?=
 =?UTF-8?q?ns=20ReferenceDatum=20car=20il=20ne=20sera=20pas=20valu=C3=A9?=
 =?UTF-8?q?=20hors=20contexte=20CSV?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../fr/inra/oresing/model/ReferenceDatum.java |  9 -------
 .../fr/inra/oresing/rest/OreSiService.java    | 27 ++++++++++++-------
 .../DuplicationLineValidationCheckResult.java |  2 +-
 .../inra/oresing/rest/OreSiResourcesTest.java |  8 +++---
 4 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
index 3893cd6f7..8c912fa63 100644
--- a/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
+++ b/src/main/java/fr/inra/oresing/model/ReferenceDatum.java
@@ -6,7 +6,6 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 
 public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext {
-    private long lineNumber;
     private final Map<ReferenceColumn, String> values;
 
     public ReferenceDatum() {
@@ -34,14 +33,6 @@ public class ReferenceDatum implements SomethingThatCanProvideEvaluationContext
         return map;
     }
 
-    public long getLineNumber() {
-        return lineNumber;
-    }
-
-    public void setLineNumber(long lineNumber) {
-        this.lineNumber = lineNumber;
-    }
-
     public String put(ReferenceColumn string, String value) {
         return values.put(string, value);
     }
diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 0013a7fab..91065571d 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -333,7 +333,7 @@ public class OreSiService {
         Function<Ltree, Ltree> getHierarchicalReferenceFn;
         Map<Ltree, Ltree> buildedHierarchicalKeys = new HashMap<>();
         Map<Ltree, Ltree> parentreferenceMap = new HashMap<>();
-        ListMultimap<Ltree, Long> hierarchicalKeys = LinkedListMultimap.create();
+        ListMultimap<Ltree, Integer> hierarchicalKeys = LinkedListMultimap.create();
         if (toUpdateCompositeReference.isPresent()) {
             Configuration.CompositeReferenceDescription compositeReferenceDescription = toUpdateCompositeReference.get();
             boolean root = Iterables.get(compositeReferenceDescription.getComponents(), 0).getReference().equals(refType);
@@ -366,16 +366,16 @@ public class OreSiService {
             Iterator<CSVRecord> linesIterator = csvParser.iterator();
             CSVRecord headerRow = linesIterator.next();
             ImmutableList<String> columns = Streams.stream(headerRow).collect(ImmutableList.toImmutableList());
-            Function<CSVRecord, ReferenceDatum> csvRecordToLineAsMapFn = line -> {
+            Function<CSVRecord, RowWithReferenceDatum> csvRecordToLineAsMapFn = line -> {
                 Iterator<String> currentHeader = columns.iterator();
                 ReferenceDatum referenceDatum = new ReferenceDatum();
-                referenceDatum.setLineNumber(line.getRecordNumber());
                 line.forEach(value -> {
                     String header = currentHeader.next();
                     ReferenceColumn referenceColumn = new ReferenceColumn(header);
                     referenceDatum.put(referenceColumn, value);
                 });
-                return referenceDatum;
+                int lineNumber = Ints.checkedCast(line.getRecordNumber());
+                return new RowWithReferenceDatum(lineNumber, referenceDatum);
             };
 
             List<CsvRowValidationCheckResult> rowErrors = new LinkedList<>();
@@ -396,7 +396,8 @@ public class OreSiService {
                     .map(internationalizationDisplay -> internationalizationDisplay.getPattern());
             Stream<ReferenceValue> referenceValuesStream = recordStream
                     .map(csvRecordToLineAsMapFn)
-                    .map(referenceDatum -> {
+                    .map(rowWithReferenceDatum -> {
+                        ReferenceDatum referenceDatum = rowWithReferenceDatum.getReferenceDatum();
                         Map<String, Set<UUID>> refsLinkedTo = new LinkedHashMap<>();
                         lineCheckers.forEach(lineChecker -> {
                             ValidationCheckResult validationCheckResult = lineChecker.checkReference(referenceDatum);
@@ -411,7 +412,7 @@ public class OreSiService {
                                             .add(referenceId);
                                 }
                             } else {
-                                rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, referenceDatum.getLineNumber()));
+                                rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, rowWithReferenceDatum.getLineNumber()));
                             }
                         });
                         final ReferenceValue e = new ReferenceValue();
@@ -478,12 +479,12 @@ public class OreSiService {
                         e.setApplication(app.getId());
                         e.setRefValues(referenceDatum.asMap());
                         if (hierarchicalKeys.containsKey(e.getHierarchicalKey())) {
-                            ValidationCheckResult validationCheckResult = new DuplicationLineValidationCheckResult(DuplicationLineValidationCheckResult.FileType.REFERENCES, refType, ValidationLevel.ERROR, e.getHierarchicalKey(), referenceDatum.getLineNumber(), hierarchicalKeys.get(e.getHierarchicalKey()));
-                            rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, referenceDatum.getLineNumber()));
-                            hierarchicalKeys.put(e.getHierarchicalKey(), referenceDatum.getLineNumber());
+                            ValidationCheckResult validationCheckResult = new DuplicationLineValidationCheckResult(DuplicationLineValidationCheckResult.FileType.REFERENCES, refType, ValidationLevel.ERROR, e.getHierarchicalKey(), rowWithReferenceDatum.getLineNumber(), hierarchicalKeys.get(e.getHierarchicalKey()));
+                            rowErrors.add(new CsvRowValidationCheckResult(validationCheckResult, rowWithReferenceDatum.getLineNumber()));
+                            hierarchicalKeys.put(e.getHierarchicalKey(), rowWithReferenceDatum.getLineNumber());
                             return null;
                         } else {
-                            hierarchicalKeys.put(e.getHierarchicalKey(), referenceDatum.getLineNumber());
+                            hierarchicalKeys.put(e.getHierarchicalKey(), rowWithReferenceDatum.getLineNumber());
                             return e;
                         }
                     })
@@ -1498,6 +1499,12 @@ public class OreSiService {
         Datum datum;
     }
 
+    @Value
+    private static class RowWithReferenceDatum {
+        int lineNumber;
+        ReferenceDatum referenceDatum;
+    }
+
     @Value
     private static class ParsedCsvRow {
         int lineNumber;
diff --git a/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
index 28c92b7f6..75f3606e5 100644
--- a/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
+++ b/src/main/java/fr/inra/oresing/rest/validationcheckresults/DuplicationLineValidationCheckResult.java
@@ -17,7 +17,7 @@ public class DuplicationLineValidationCheckResult implements ValidationCheckResu
 
     Map<String, Object> messageParams;
 
-    public DuplicationLineValidationCheckResult(FileType filetype, String file, ValidationLevel level, Ltree hierarchicalKey, long currentLineNumber, List<Long> otherLines) {
+    public DuplicationLineValidationCheckResult(FileType filetype, String file, ValidationLevel level, Ltree hierarchicalKey, int currentLineNumber, List<Integer> otherLines) {
         this.level = level;
         this.message = FileType.DATATYPE.message;
         this.messageParams = ImmutableMap.of(
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index a02c1a9d4..6444a54ef 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -970,8 +970,8 @@ public class OreSiResourcesTest {
             Assert.assertEquals("duplicatedLineInDatatype", validationCheckResult.getMessage());
             final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
             Assert.assertEquals("types_de_zones_etudes", messageParams.get("file"));
-            Assert.assertEquals(4L, messageParams.get("lineNumber"));
-            Assert.assertArrayEquals(new Long[]{3L, 4L}, ((List) messageParams.get("otherLines")).toArray());
+            Assert.assertEquals(4, messageParams.get("lineNumber"));
+            Assert.assertArrayEquals(new Integer[]{3, 4}, ((List) messageParams.get("otherLines")).toArray());
             Assert.assertEquals("zone20", messageParams.get("duplicateKey"));
         }
 
@@ -1043,8 +1043,8 @@ on test le dépôt d'un fichier récursif
             Assert.assertEquals("duplicatedLineInDatatype", validationCheckResult.getMessage());
             final Map<String, Object> messageParams = validationCheckResult.getMessageParams();
             Assert.assertEquals("zones_etudes", messageParams.get("file"));
-            Assert.assertEquals(4L, messageParams.get("lineNumber"));
-            Assert.assertArrayEquals(new Long[]{2L, 4L}, ((List) messageParams.get("otherLines")).toArray());
+            Assert.assertEquals(4, messageParams.get("lineNumber"));
+            Assert.assertArrayEquals(new Integer[]{2, 4}, ((List) messageParams.get("otherLines")).toArray());
             Assert.assertEquals("site1", messageParams.get("duplicateKey"));
         }
 
-- 
GitLab


From 9373327ecdf16435856cf00a8ad0d23f5d9c6af8 Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Mon, 21 Feb 2022 17:21:05 +0100
Subject: [PATCH 08/10] =?UTF-8?q?Ajout=20de=20test=20pour=20pr=C3=A9parer?=
 =?UTF-8?q?=20la=20duplication=20des=20lignes=20de=20datas?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On peut voir que le redépôt du fichier (partie commentée) crée des lignes en doublon (8 lignes au lieu de 4)
C'est forcément un bug mais la résolution est plus complexe qu'une simple unicité sur authorization.

J'ouvre une demande pour cela et je vais voir comment on peut la résoudre
---
 .../java/fr/inra/oresing/rest/Fixtures.java   |  6 +++
 .../inra/oresing/rest/OreSiResourcesTest.java | 39 +++++++++++++++++++
 src/test/resources/data/duplication/data.csv  |  5 +++
 .../data/duplication/duplication.yaml         |  4 +-
 4 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 src/test/resources/data/duplication/data.csv

diff --git a/src/test/java/fr/inra/oresing/rest/Fixtures.java b/src/test/java/fr/inra/oresing/rest/Fixtures.java
index 6099f2d0f..f1cfa3296 100644
--- a/src/test/java/fr/inra/oresing/rest/Fixtures.java
+++ b/src/test/java/fr/inra/oresing/rest/Fixtures.java
@@ -482,6 +482,12 @@ public class Fixtures {
         return referentielFiles;
     }
 
+    public Map<String, String> getDuplicatedDataFiles() {
+        Map<String, String> referentielFiles = new LinkedHashMap<>();
+        referentielFiles.put("data_without_duplicateds", "/data/duplication/data.csv");
+        return referentielFiles;
+    }
+
     public String getPhysicoChimieSolsProDataResourceName() {
         return "/data/pros/physico_chimie_sols.csv";
     }
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index 6444a54ef..4f1fd4b68 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -1087,6 +1087,45 @@ on test le dépôt d'un fichier récursif
                 .andExpect(jsonPath("$.referenceValues.length()", IsEqual.equalTo(2)))
                 .andReturn().getResponse().getContentAsString();
 
+        //on teste un dépot de fichier de données
+        final String dataWithoutDuplicateds = fixtures.getDuplicatedDataFiles().get("data_without_duplicateds");
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "data_without_duplicateds.csv", "text/plain", refStream);
+            mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/data/dty")
+                            .file(refFile)
+                            .cookie(authCookie))
+                    .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
+        }
+        //on teste le nombre de ligne
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+            final String response = mockMvc.perform(get("/api/v1/applications/duplicated/data/dty")
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andExpect(jsonPath("$.totalRows", IsEqual.equalTo(4
+                    )))
+                .andReturn().getResponse().getContentAsString();
+            log.debug(response);
+        }
+
+        // on  redepose le fichier
+
+        try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+            MockMultipartFile refFile = new MockMultipartFile("file", "data_without_duplicateds.csv", "text/plain", refStream);
+            mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/duplicated/data/dty")
+                            .file(refFile)
+                            .cookie(authCookie))
+                    .andExpect(MockMvcResultMatchers.status().is2xxSuccessful());
+        }
+        // le nombre de ligne est inchangé
+        /*try (InputStream refStream = fixtures.getClass().getResourceAsStream(dataWithoutDuplicateds)) {
+            final String response = mockMvc.perform(get("/api/v1/applications/duplicated/data/dty")
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andExpect(jsonPath("$.totalRows", IsEqual.equalTo(4
+                    )))
+                .andReturn().getResponse().getContentAsString();
+            log.debug(response);
+        }*/
     }
 
     @Test
diff --git a/src/test/resources/data/duplication/data.csv b/src/test/resources/data/duplication/data.csv
new file mode 100644
index 000000000..250c8f311
--- /dev/null
+++ b/src/test/resources/data/duplication/data.csv
@@ -0,0 +1,5 @@
+site;date
+site1.site2;23/02/1980
+site1.site2;24/02/1980
+site1;23/02/1980
+site1;24/02/1980
\ No newline at end of file
diff --git a/src/test/resources/data/duplication/duplication.yaml b/src/test/resources/data/duplication/duplication.yaml
index b48a4d280..cae0e427e 100644
--- a/src/test/resources/data/duplication/duplication.yaml
+++ b/src/test/resources/data/duplication/duplication.yaml
@@ -59,8 +59,8 @@ dataTypes:
               params:
                 refType: zones_etudes
     format:
-      headerLine: 2
-      firstRowLine: 3
+      headerLine: 1
+      firstRowLine: 2
       columns:
         - header: "site"
           boundTo:
-- 
GitLab


From ca3c0e04c1eb6c323470eb4a406ef9d9a0c507e4 Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Tue, 22 Feb 2022 11:53:35 +0100
Subject: [PATCH 09/10] =?UTF-8?q?Corrige=20une=20expression=20bool=C3=A9en?=
 =?UTF-8?q?ne=20toujours=20fause=20(incompatibilit=C3=A9=20entre=20le=20ty?=
 =?UTF-8?q?pe=20de=20cl=C3=A9=20et=20l'argument=20pass=C3=A9)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/java/fr/inra/oresing/rest/OreSiService.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 91065571d..095b56d58 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -467,8 +467,8 @@ public class OreSiService {
                          * a noter que pour les references récursives on récupère l'id depuis  referenceLineChecker.getReferenceValues() ce qui revient au même
                          */
 
-                        if (storedReferences.containsKey(hierarchicalKey)) {
-                            e.setId(storedReferences.get(hierarchicalKey));
+                        if (storedReferences.containsKey(hierarchicalKey.getSql())) {
+                            e.setId(storedReferences.get(hierarchicalKey.getSql()));
                         }
                         e.setBinaryFile(fileId);
                         e.setReferenceType(refType);
-- 
GitLab


From e27d4534f9279fa451d60bb263ddcbffe5ca60db Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Tue, 22 Feb 2022 14:26:50 +0100
Subject: [PATCH 10/10] =?UTF-8?q?Utilise=20Multimap=20plut=C3=B4t=20que=20?=
 =?UTF-8?q?Map=20de=20collection?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/java/fr/inra/oresing/rest/OreSiService.java | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 095b56d58..4073563c9 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -502,7 +502,7 @@ public class OreSiService {
                 .map(rcd -> rcd.getParentRecursiveKey())
                 .map(rck -> columns.indexOf(rck))
                 .orElse(null);
-        Map<String, List<Long>> missingParentReferences = new HashMap<>();
+        ListMultimap<String, Long> missingParentReferences = LinkedListMultimap.create();
         if (parentRecursiveIndex == null || parentRecursiveIndex < 0) {
             return recordStream;
         }
@@ -532,12 +532,10 @@ public class OreSiService {
                         if (!referenceUUIDs.containsKey(s)) {
                             final UUID uuid = UUID.randomUUID();
                             referenceUUIDs.put(s, uuid);
-                            missingParentReferences
-                                    .computeIfAbsent(s, k -> new LinkedList<>())
-                                    .add(csvrecord.getRecordNumber());
+                            missingParentReferences.put(s, csvrecord.getRecordNumber());
                         }
                     }
-                    missingParentReferences.remove(naturalKey);
+                    missingParentReferences.removeAll(naturalKey);
                     return;
                 })
                 .collect(Collectors.toList());
@@ -545,7 +543,7 @@ public class OreSiService {
                 .ifPresent(slc -> slc.setReferenceValues(ImmutableMap.copyOf(referenceUUIDs)));
 
         if (!missingParentReferences.isEmpty()) {
-            missingParentReferences.entrySet().stream()
+            missingParentReferences.asMap().entrySet().stream()
                     .forEach(entry -> {
                         final String missingParentReference = entry.getKey();
                         entry.getValue().stream().forEach(
-- 
GitLab