From 6735dad2ca043b84344ad14beedbf4f55fb68cbe Mon Sep 17 00:00:00 2001 From: Johan Frostmark Date: Fri, 7 Aug 2020 20:37:00 +0200 Subject: [PATCH] clean up of code most important change. removed batch processing. each row is a transaction instead of one major bulk operation. otherwise, lots of small fixes and added some tests --- .../replica/service/SourceFileService.java | 5 + ...CsvAncestryRowToBovineEntityConverter.java | 21 +-- .../CsvRowToEntityConverterUtils.java | 27 ++++ .../service/processcsv/CsvValidator.java | 9 +- .../CsvWeightRowToBovineEntityConverter.java | 5 +- .../SourceFileProcessingService.java | 135 +++++++----------- .../replica/web/rest/SourceFileResource.java | 52 +++---- .../web/rest/errors/ExceptionTranslator.java | 13 ++ .../EntityChangeJaversAspectIT.java | 65 +++++++-- .../SourceFileProcessingServiceIT.java | 96 +++++++++++++ .../SourceFileResourceDoProcessIT.java | 90 ++++++++++-- .../fixtures/csv_se015112_broken_header.zip | Bin 0 -> 797 bytes .../fixtures/csv_se015112_empty_lines.zip | Bin 0 -> 1880 bytes .../csv_se015112_missing_id_first.zip | Bin 0 -> 3805 bytes .../csv_se015112_missing_id_second.zip | Bin 0 -> 3971 bytes .../fixtures/csv_se015112_missing_lines.zip | Bin 0 -> 1022 bytes .../fixtures/csv_se015112_upd_harst_name.zip | Bin 0 -> 4420 bytes 17 files changed, 368 insertions(+), 150 deletions(-) create mode 100644 src/main/java/com/bonlimousin/replica/service/processcsv/CsvRowToEntityConverterUtils.java create mode 100644 src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingServiceIT.java create mode 100644 src/test/resources/fixtures/csv_se015112_broken_header.zip create mode 100644 src/test/resources/fixtures/csv_se015112_empty_lines.zip create mode 100644 src/test/resources/fixtures/csv_se015112_missing_id_first.zip create mode 100644 src/test/resources/fixtures/csv_se015112_missing_id_second.zip create mode 100644 src/test/resources/fixtures/csv_se015112_missing_lines.zip create mode 100644 src/test/resources/fixtures/csv_se015112_upd_harst_name.zip diff --git a/src/main/java/com/bonlimousin/replica/service/SourceFileService.java b/src/main/java/com/bonlimousin/replica/service/SourceFileService.java index e5a3800..63bd18c 100644 --- a/src/main/java/com/bonlimousin/replica/service/SourceFileService.java +++ b/src/main/java/com/bonlimousin/replica/service/SourceFileService.java @@ -62,6 +62,11 @@ public Optional findOne(Long id) { log.debug("Request to get SourceFile : {}", id); return sourceFileRepository.findById(id); } + + @Transactional(readOnly = true) + public boolean exists(Long id) { + return sourceFileRepository.existsById(id); + } /** * Delete the sourceFile by id. diff --git a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvAncestryRowToBovineEntityConverter.java b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvAncestryRowToBovineEntityConverter.java index b0c4ab0..c90ad4d 100644 --- a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvAncestryRowToBovineEntityConverter.java +++ b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvAncestryRowToBovineEntityConverter.java @@ -4,7 +4,6 @@ import java.time.ZoneId; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import com.bonlimousin.replica.domain.BovineEntity; import com.bonlimousin.replica.domain.enumeration.BovineStatus; @@ -17,9 +16,8 @@ private CsvAncestryRowToBovineEntityConverter() { } - public static BovineEntity convert(String[] cells, BovineEntity be) { - String csvEarTagId = cells[CsvAncestryColumns.EAR_TAG_ID.columnIndex()]; - be.setEarTagId(NumberUtils.createInteger(StringUtils.replace(csvEarTagId, ".0", ""))); + public static BovineEntity convert(String[] cells, BovineEntity be) { + be.setEarTagId(CsvRowToEntityConverterUtils.createId(cells, CsvAncestryColumns.EAR_TAG_ID)); String csvMasterIdentifier = cells[CsvAncestryColumns.MASTER_IDENTIFIER.columnIndex()]; be.setMasterIdentifier(csvMasterIdentifier); @@ -29,9 +27,8 @@ public static BovineEntity convert(String[] cells, BovineEntity be) { String csvCountry = cells[CsvAncestryColumns.COUNTRY_ID.columnIndex()]; be.setCountry(StringUtils.lowerCase(csvCountry)); - - String csvHerdId = cells[CsvAncestryColumns.HERD_ID.columnIndex()]; - be.setHerdId(NumberUtils.createInteger(StringUtils.trimToNull(StringUtils.replace(csvHerdId, ".0", "")))); + + be.setHerdId(CsvRowToEntityConverterUtils.createId(cells, CsvAncestryColumns.HERD_ID)); String csvBirthDate = cells[CsvAncestryColumns.BIRTH_DATE.columnIndex()]; be.setBirthDate(LocalDate.parse(csvBirthDate).atStartOfDay(ZoneId.of("Europe/Stockholm")).toInstant()); @@ -39,15 +36,9 @@ public static BovineEntity convert(String[] cells, BovineEntity be) { String csvGender = cells[CsvAncestryColumns.GENDER.columnIndex()]; be.setGender("2".equals(csvGender) ? Gender.HEIFER : Gender.BULL); - String csvMatriId = cells[CsvAncestryColumns.MATRI_ID.columnIndex()]; - Integer matriId = NumberUtils.createInteger(StringUtils.trimToNull(StringUtils.replace(csvMatriId, ".0", ""))); - matriId = matriId != null ? matriId : 0; // unknown mothers seems to have zero anyway - be.setMatriId(matriId); + be.setMatriId(CsvRowToEntityConverterUtils.createId(cells, CsvAncestryColumns.MATRI_ID)); - String csvPatriId = cells[CsvAncestryColumns.PATRI_ID.columnIndex()]; - Integer patriId = NumberUtils.createInteger(StringUtils.trimToNull(StringUtils.replace(csvPatriId, ".0", ""))); - patriId = patriId != null ? patriId : 0; // unknown fathers seems to have zero anyway - be.setPatriId(patriId); + be.setPatriId(CsvRowToEntityConverterUtils.createId(cells, CsvAncestryColumns.PATRI_ID)); be.setHornStatus(HornStatus.UNKNOWN); // TODO if exists diff --git a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvRowToEntityConverterUtils.java b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvRowToEntityConverterUtils.java new file mode 100644 index 0000000..de92174 --- /dev/null +++ b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvRowToEntityConverterUtils.java @@ -0,0 +1,27 @@ +package com.bonlimousin.replica.service.processcsv; + +import java.text.MessageFormat; + +import javax.validation.ValidationException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +public class CsvRowToEntityConverterUtils { + + private CsvRowToEntityConverterUtils() { + + } + + public static Integer createId(String[] cells, CsvColumns col) { + String val = cells[col.columnIndex()]; + String stripped = StringUtils.trimToNull(StringUtils.replace(val, ".0", "")); + if(!col.nullableValue() && stripped == null) { + String msg = MessageFormat.format("The value {0} for column {1} is required but not and id", val, col.name()); + throw new ValidationException(msg); + } else if(stripped == null) { + return 0; // unknown parent seems to have zero anyway + } + return NumberUtils.createInteger(stripped); + } +} diff --git a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvValidator.java b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvValidator.java index 28c6c7b..54787a3 100644 --- a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvValidator.java +++ b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvValidator.java @@ -57,12 +57,17 @@ public static void validateCsvFile(String fileName, CsvColumns[] columns, InputS } String[] row = csvReader.readNext(); - if (row == null) { + if (row == null || row.length == 0) { throw new ValidationException("No records in csv " + fileName); } for (CsvColumns col : columns) { + if (row.length <= col.columnIndex()) { + String msg = MessageFormat.format("First row for {0} has fewer columns ({1}) then given index ({2}) for column {3}", fileName, row.length, col.columnIndex(), col.name()); + throw new ValidationException(msg); + } if (!col.nullableValue() && StringUtils.trimToNull(row[col.columnIndex()]) == null) { - throw new ValidationException(fileName + " is missing column " + col.name()); + String msg = MessageFormat.format("First row for {0} is missing column {1}", fileName, col.name()); + throw new ValidationException(msg); } } } diff --git a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvWeightRowToBovineEntityConverter.java b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvWeightRowToBovineEntityConverter.java index c3dab1d..8ac624c 100644 --- a/src/main/java/com/bonlimousin/replica/service/processcsv/CsvWeightRowToBovineEntityConverter.java +++ b/src/main/java/com/bonlimousin/replica/service/processcsv/CsvWeightRowToBovineEntityConverter.java @@ -11,9 +11,8 @@ private CsvWeightRowToBovineEntityConverter() { } - public static BovineEntity convert(String[] cells, BovineEntity be) { - String csvEarTagId = cells[CsvWeightColumns.EAR_TAG_ID.columnIndex()]; - be.setEarTagId(NumberUtils.createInteger(StringUtils.replace(csvEarTagId, ".0", ""))); + public static BovineEntity convert(String[] cells, BovineEntity be) { + be.setEarTagId(CsvRowToEntityConverterUtils.createId(cells, CsvWeightColumns.EAR_TAG_ID)); String csvWeight = cells[CsvWeightColumns.WEIGHT.columnIndex()]; Integer weight = NumberUtils.createInteger(StringUtils.replace(csvWeight, ".0", "")); diff --git a/src/main/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingService.java b/src/main/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingService.java index 02ee1a9..7267e6b 100644 --- a/src/main/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingService.java +++ b/src/main/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingService.java @@ -11,57 +11,46 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import javax.validation.ValidationException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StopWatch; import com.bonlimousin.replica.domain.BovineEntity; import com.bonlimousin.replica.domain.SourceFileEntity; import com.bonlimousin.replica.repository.BovineRepository; -import com.bonlimousin.replica.repository.SourceFileRepository; import com.bonlimousin.replica.service.BovineService; +import com.bonlimousin.replica.service.SourceFileService; import liquibase.util.csv.CSVReader; /** - * Service Implementation for managing {@link SourceFileEntity}. + * Service Implementation for managing import of CVS files with cow data */ @Service -@Transactional public class SourceFileProcessingService { private final Logger log = LoggerFactory.getLogger(SourceFileProcessingService.class); - private final SourceFileRepository sourceFileRepository; + private final SourceFileService sourceFileService; private final BovineRepository bovineRepository; private final BovineService bovineService; private final Executor executor; - public SourceFileProcessingService(SourceFileRepository sourceFileRepository, + public SourceFileProcessingService(SourceFileService sourceFileService, @Qualifier("taskExecutor") Executor executor, BovineRepository bovineRepository, BovineService bovineService) { - this.sourceFileRepository = sourceFileRepository; + this.sourceFileService = sourceFileService; this.executor = executor; this.bovineRepository = bovineRepository; this.bovineService = bovineService; } - /** - * Check existence of sourceFile by id. - * - * @param id the id of the entity. - * @return answer. - */ - @Transactional(readOnly = true) - public boolean exists(Long id) { - return sourceFileRepository.existsById(id); - } - /** * Process one sourceFile by id. * @@ -69,10 +58,13 @@ public boolean exists(Long id) { * @return the entity. * @throws IOException */ - @Transactional public void process(Long id, boolean isRunAsync, boolean isDryRun) throws IOException { - SourceFileEntity sfe = sourceFileRepository.getOne(id); - + Optional opt = sourceFileService.findOne(id); + if(opt.isEmpty()) { + throw new ValidationException("Entity not found"); + } + SourceFileEntity sfe = opt.get(); + CsvValidator.validateZipFile(CsvFile.ANCESTRY.fileName(), CsvAncestryColumns.values(), sfe.getZipFile()); CsvValidator.validateZipFile(CsvFile.WEIGHT.fileName(), CsvWeightColumns.values(), sfe.getZipFile()); CsvValidator.validateZipFile(CsvFile.JOURNAL.fileName(), CsvJournalColumns.values(), sfe.getZipFile()); @@ -94,110 +86,83 @@ public void processCsvFiles(SourceFileEntity sfe, boolean isDryRun) throws IOExc StopWatch watch = new StopWatch(); watch.start(); try { - processCsvFile(sfe, CsvFile.ANCESTRY, reader -> processAncestryCsvFile(sfe, reader, isDryRun)); - processCsvFile(sfe, CsvFile.WEIGHT, reader -> processWeightCsvFile(reader, isDryRun)); - processCsvFile(sfe, CsvFile.JOURNAL, reader -> processJournalCsvFile(reader, isDryRun)); - if(!isDryRun) { - sfe.setProcessed(Instant.now()); - sfe.setOutcome("success"); // XXX catch and save failures as well? - sourceFileRepository.save(sfe); - } + processCsvFile(sfe, CsvFile.ANCESTRY, cells -> processAncestryCsvLine(sfe, isDryRun, cells)); + processCsvFile(sfe, CsvFile.WEIGHT, cells -> processWeightCsvLine(isDryRun, cells)); + processCsvFile(sfe, CsvFile.JOURNAL, cells -> processJournalCsvLine(isDryRun, cells)); + String outcome = isDryRun ? "dryrun_success" : "success"; + storeOutcome(sfe, outcome); + } catch(Exception e) { + String outcome = isDryRun ? "dryrun_failure" : "failure"; + storeOutcome(sfe, outcome); + throw e; } finally { watch.stop(); log.info("Processing time of zip-file was {}", watch.getTotalTimeMillis()); } } + + private void storeOutcome(SourceFileEntity sfe, String outcome) { + sfe.setProcessed(Instant.now()); + sfe.setOutcome(outcome); + sourceFileService.save(sfe); + } - public void processCsvFile(SourceFileEntity sfe, CsvFile csvFile, Consumer processor) + private void processCsvFile(SourceFileEntity sfe, CsvFile csvFile, Consumer processor) throws IOException { try (ByteArrayInputStream bais = new ByteArrayInputStream(sfe.getZipFile()); ZipInputStream zis = new ZipInputStream(bais)) { for (ZipEntry ze; (ze = zis.getNextEntry()) != null;) { - if (csvFile.fileName().equals(ze.getName())) { - processor.accept(new CSVReader(new InputStreamReader(zis, StandardCharsets.UTF_8))); + if (csvFile.fileName().equals(ze.getName())) { + try (CSVReader csvReader = new CSVReader(new InputStreamReader(zis, StandardCharsets.UTF_8))) { + csvReader.readNext(); // ignore header + int rowCount = 0; + for(String[] cells; (cells = csvReader.readNext()) != null; rowCount++) { + log.debug("Process row {} of file {}", rowCount, csvFile); + processor.accept(cells); + } + return; + } catch (IOException e) { + log.error("Processing of csv-file failed", e); + } } } } } - public void processAncestryCsvFile(SourceFileEntity sfe, CSVReader csvReader, boolean isDryRun) { - try { - csvReader.readNext(); // ignore header - int rowCount = 0; - for(String[] cells; (cells = csvReader.readNext()) != null; rowCount++) { - if(cells.length <= CsvAncestryColumns.NAME.columnIndex()) { - log.warn("Row {} seems empty. Only {} cells", rowCount, cells.length); - continue; - } - processAncestryCsvLine(sfe, isDryRun, cells, rowCount); - } - } catch (IOException e) { - log.error("Processing of ancestry-csv failed", e); - } - } - - public void processAncestryCsvLine(SourceFileEntity sfe, boolean isDryRun, String[] cells, int rowCount) { - BovineEntity be = CsvAncestryRowToBovineEntityConverter.convert(cells, new BovineEntity()); - if(be.getEarTagId() == null) { - log.warn("(row: {}) Abort! Eartagid is missing for row", rowCount); - return; - } - if(be.getHerdId() == null) { - log.warn("(row: {}) Abort! HerdId is missing for row", rowCount); - return; - } + public void processAncestryCsvLine(SourceFileEntity sfe, boolean isDryRun, String[] cells) { + BovineEntity be = CsvAncestryRowToBovineEntityConverter.convert(cells, new BovineEntity()); Optional opt = bovineRepository.findOneByEarTagId(be.getEarTagId()); if(opt.isPresent()) { BovineEntity currbe = CsvAncestryRowToBovineEntityConverter.convert(cells, opt.get()); currbe.setSourceFile(sfe); - log.debug("(row: {})(dryrun: {}) Update of existing bovine with eartagid {} and herdid {}", rowCount, isDryRun, be.getEarTagId(), be.getHerdId()); + log.info("Update of existing bovine with eartagid {} and herdid {}", be.getEarTagId(), be.getHerdId()); if(!isDryRun) { bovineService.save(currbe); } } else { be.setSourceFile(sfe); - log.info("(row: {})(dryrun: {}) Create new bovine with eartagid {} and herdid {}", rowCount, isDryRun, be.getEarTagId(), be.getHerdId()); + log.info("Create new bovine with eartagid {} and herdid {}", be.getEarTagId(), be.getHerdId()); if(!isDryRun) { bovineService.save(be); } } } - public void processWeightCsvFile(CSVReader csvReader, boolean isDryRun) { - try { - csvReader.readNext(); // ignore header - int rowCount = 0; - for(String[] cells; (cells = csvReader.readNext()) != null; rowCount++) { - if(cells.length <= CsvWeightColumns.TYPE.columnIndex()) { - log.warn("Row {} seems empty. Only {} cells", rowCount, cells.length); - continue; - } - processWeightCsvLine(isDryRun, cells, rowCount); - } - } catch (IOException e) { - log.error("Processing of weight-csv failed", e); - } - } - - public void processWeightCsvLine(boolean isDryRun, String[] cells, int rowCount) { - BovineEntity be = CsvWeightRowToBovineEntityConverter.convert(cells, new BovineEntity()); - if(be.getEarTagId() == null) { - log.warn("(row: {}) Abort! Eartagid is missing for row", rowCount); - return; - } + public void processWeightCsvLine(boolean isDryRun, String[] cells) { + BovineEntity be = CsvWeightRowToBovineEntityConverter.convert(cells, new BovineEntity()); Optional opt = bovineRepository.findOneByEarTagId(be.getEarTagId()); if(opt.isPresent()) { BovineEntity currbe = CsvWeightRowToBovineEntityConverter.convert(cells, opt.get()); - log.debug("(row: {})(dryrun: {}) Update of existing bovine with eartagid {} and herdid {}", rowCount, isDryRun, be.getEarTagId(), be.getHerdId()); + log.info("Update of existing bovine with eartagid {} and herdid {}", be.getEarTagId(), be.getHerdId()); if(!isDryRun) { bovineService.save(currbe); } } else { - log.info("(row: {})(dryrun: {}) No bovine with eartagid {} and herdid {} exists. Ignore weight until present.", rowCount, isDryRun, be.getEarTagId(), be.getHerdId()); + log.info("No bovine with eartagid {} and herdid {} exists. Ignore weight until present.", be.getEarTagId(), be.getHerdId()); } } - public void processJournalCsvFile(CSVReader csvReader, boolean isDryRun) { + public void processJournalCsvLine(boolean isDryRun, String[] cells) { // TODO impl } } diff --git a/src/main/java/com/bonlimousin/replica/web/rest/SourceFileResource.java b/src/main/java/com/bonlimousin/replica/web/rest/SourceFileResource.java index 1f0b9ba..6a98d00 100644 --- a/src/main/java/com/bonlimousin/replica/web/rest/SourceFileResource.java +++ b/src/main/java/com/bonlimousin/replica/web/rest/SourceFileResource.java @@ -1,35 +1,42 @@ package com.bonlimousin.replica.web.rest; -import com.bonlimousin.replica.domain.SourceFileEntity; -import com.bonlimousin.replica.service.SourceFileService; -import com.bonlimousin.replica.web.rest.errors.BadRequestAlertException; -import com.bonlimousin.replica.service.dto.SourceFileCriteria; -import com.bonlimousin.replica.service.processcsv.SourceFileProcessingService; -import com.bonlimousin.replica.service.SourceFileQueryService; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Optional; + +import javax.validation.Valid; +import javax.validation.ValidationException; -import io.github.jhipster.web.util.HeaderUtil; -import io.github.jhipster.web.util.PaginationUtil; -import io.github.jhipster.web.util.ResponseUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; -import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; -import javax.validation.Valid; -import javax.validation.ValidationException; +import com.bonlimousin.replica.domain.SourceFileEntity; +import com.bonlimousin.replica.service.SourceFileQueryService; +import com.bonlimousin.replica.service.SourceFileService; +import com.bonlimousin.replica.service.dto.SourceFileCriteria; +import com.bonlimousin.replica.service.processcsv.SourceFileProcessingService; +import com.bonlimousin.replica.web.rest.errors.BadRequestAlertException; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Optional; +import io.github.jhipster.web.util.HeaderUtil; +import io.github.jhipster.web.util.PaginationUtil; +import io.github.jhipster.web.util.ResponseUtil; /** * REST controller for managing {@link com.bonlimousin.replica.domain.SourceFileEntity}. @@ -160,10 +167,7 @@ public ResponseEntity deleteSourceFile(@PathVariable Long id) { public ResponseEntity processSourceFile(@PathVariable Long id, @RequestParam(value = "isRunAsync", defaultValue = "true", required = true) boolean isRunAsync, @RequestParam(value = "isDryRun", defaultValue = "false", required = true) boolean isDryRun) { - log.debug("REST request to process SourceFile : {}", id); - if(!sourceFileProcessingService.exists(id)) { - throw new ResponseStatusException(HttpStatus.NOT_FOUND); - } + log.debug("REST request to process SourceFile : {}", id); try { sourceFileProcessingService.process(id, isRunAsync, isDryRun); } catch (IOException e) { diff --git a/src/main/java/com/bonlimousin/replica/web/rest/errors/ExceptionTranslator.java b/src/main/java/com/bonlimousin/replica/web/rest/errors/ExceptionTranslator.java index a3913db..3a6c295 100644 --- a/src/main/java/com/bonlimousin/replica/web/rest/errors/ExceptionTranslator.java +++ b/src/main/java/com/bonlimousin/replica/web/rest/errors/ExceptionTranslator.java @@ -21,6 +21,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; +import javax.validation.ValidationException; + import java.util.List; import java.util.stream.Collectors; @@ -104,4 +106,15 @@ public ResponseEntity handleConcurrencyFailure(ConcurrencyFailureExcept .build(); return create(ex, problem, request); } + + @ExceptionHandler + public ResponseEntity handleValidationException(ValidationException ex, NativeWebRequest request) { + Problem problem = Problem.builder() + .withType(ErrorConstants.CONSTRAINT_VIOLATION_TYPE) + .withTitle("Argument not valid") + .withStatus(defaultConstraintViolationStatus()) + .with(MESSAGE_KEY, ErrorConstants.ERR_VALIDATION) + .build(); + return create(ex, problem, request); + } } diff --git a/src/test/java/com/bonlimousin/replica/service/entitychange/EntityChangeJaversAspectIT.java b/src/test/java/com/bonlimousin/replica/service/entitychange/EntityChangeJaversAspectIT.java index 9d555e2..df5c559 100644 --- a/src/test/java/com/bonlimousin/replica/service/entitychange/EntityChangeJaversAspectIT.java +++ b/src/test/java/com/bonlimousin/replica/service/entitychange/EntityChangeJaversAspectIT.java @@ -5,20 +5,21 @@ import java.time.Duration; import java.time.Instant; -import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; -import java.util.Map; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.kafka.core.ConsumerFactory; @@ -35,6 +36,7 @@ @ActiveProfiles("emitentitychanges") @SpringBootTest(classes = { BonReplicaServiceApp.class, KafkaTestConfiguration.class }) +@TestMethodOrder(OrderAnnotation.class) class EntityChangeJaversAspectIT { @Autowired @@ -68,14 +70,11 @@ public static BovineEntity createEntity() { } @Test + @Order(1) @Transactional void testBonvineEntityChange() { - BovineEntity be = bovineService.save(createEntity()); - - Consumer consumer = entityChangeConsumerFactory.createConsumer(); - consumer.subscribe(Collections.singletonList("ENTITY_CHANGE_BOVINEENTITY")); - ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); - + BovineEntity be = bovineService.save(createEntity()); + ConsumerRecords records = consumeChanges(); assertThat(records.count()).isEqualTo(1); ConsumerRecord record = records.iterator().next(); assertEquals("CREATE", record.key()); @@ -101,7 +100,53 @@ void testBonvineEntityChange() { List fields = Arrays.asList("earTagId", "masterIdentifier", "country", "herdId", "birthDate", "gender", "name", "bovineStatus", "hornStatus", "matriId", "patriId", "weight0", "weight200", "weight365"); - assertThat(record.value().getChangedEntityFields()).containsAll(fields); + assertThat(record.value().getChangedEntityFields()).containsAll(fields); + } + + @Test + @Order(2) + @Transactional + void testBonvineEntityChangeUpdate() { + BovineEntity be = bovineService.save(createEntity()); + BovineEntity beUpd = bovineService.save(be.name("TEST")); + ConsumerRecords records = consumeChanges(); + assertThat(records.count()).isEqualTo(2); + Iterator> it = records.iterator(); + ConsumerRecord recordCreate = it.next(); + assertEquals("CREATE", recordCreate.key()); + ConsumerRecord record = it.next(); + assertEquals("UPDATE", record.key()); + assertThat(record.value().getEntityId()).isEqualTo(be.getId().toString()); + + assertThat(record.value().getEntityValue()) + .containsEntry("earTagId", be.getEarTagId()) + .containsEntry("masterIdentifier", be.getMasterIdentifier()) + .containsEntry("country", be.getCountry()) + .containsEntry("herdId", be.getHerdId()) + .containsEntry("birthDate", DateTimeFormatter.ISO_INSTANT.format(be.getBirthDate())) + .containsEntry("gender", be.getGender().name()) + .containsEntry("name", beUpd.getName()) + .containsEntry("bovineStatus", be.getBovineStatus().name()) + .containsEntry("hornStatus", be.getHornStatus().name()) + .containsEntry("matriId", be.getMatriId()) + .containsEntry("patriId", be.getPatriId()) + .containsEntry("weight0", be.getWeight0()) + .containsEntry("weight200", be.getWeight200()) + .containsEntry("weight365", be.getWeight365()) + ; + + List fields = Arrays.asList("name"); + assertThat(record.value().getChangedEntityFields()).containsAll(fields); + } + + private ConsumerRecords consumeChanges() { + Consumer consumer = entityChangeConsumerFactory.createConsumer(); + consumer.subscribe(Collections.singletonList("ENTITY_CHANGE_BOVINEENTITY")); + ConsumerRecords records = consumer.poll(Duration.ofSeconds(2)); + consumer.commitSync(); + consumer.unsubscribe(); + consumer.close(); + return records; } @AfterAll diff --git a/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingServiceIT.java b/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingServiceIT.java new file mode 100644 index 0000000..4a8947b --- /dev/null +++ b/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileProcessingServiceIT.java @@ -0,0 +1,96 @@ +package com.bonlimousin.replica.service.processcsv; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; + +import javax.validation.ValidationException; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.transaction.annotation.Transactional; + +import com.bonlimousin.replica.BonReplicaServiceApp; +import com.bonlimousin.replica.domain.SourceFileEntity; +import com.bonlimousin.replica.service.SourceFileService; + +@SpringBootTest(classes = BonReplicaServiceApp.class) +public class SourceFileProcessingServiceIT { + + private static final String TEST_ZIP_FILE_BROKEN_HEADER = "src/test/resources/fixtures/csv_se015112_broken_header.zip"; + private static final String TEST_ZIP_FILE_EMPTY_LINES = "src/test/resources/fixtures/csv_se015112_empty_lines.zip"; + private static final String TEST_ZIP_FILE_MISSING_LINES = "src/test/resources/fixtures/csv_se015112_missing_lines.zip"; + private static final String TEST_ZIP_FILE_MISSING_ID_FIRST = "src/test/resources/fixtures/csv_se015112_missing_id_first.zip"; + private static final String TEST_ZIP_FILE_MISSING_ID_SECOND = "src/test/resources/fixtures/csv_se015112_missing_id_second.zip"; + + @Autowired + private SourceFileService sourceFileService; + + @Autowired + private SourceFileProcessingService sourceFileProcessingService; + + public static SourceFileEntity createEntity(String zipFile) throws FileNotFoundException, IOException { + byte[] zipBytes = IOUtils.toByteArray(new FileInputStream(zipFile)); + SourceFileEntity sourceFileEntity = new SourceFileEntity() + .name("test.zip") + .zipFile(zipBytes) + .zipFileContentType("application/zip") + .processed(null) + .outcome(null); + return sourceFileEntity; + } + + @Test + @Transactional + void processBrokenHeader() throws Exception { + SourceFileEntity sfe = sourceFileService.save(createEntity(TEST_ZIP_FILE_BROKEN_HEADER)); + Long id = sfe.getId(); + assertThrows(ValidationException.class, () -> + sourceFileProcessingService.process(id, false, true) + ); + } + + @Test + @Transactional + void processEmptyLines() throws Exception { + SourceFileEntity sfe = sourceFileService.save(createEntity(TEST_ZIP_FILE_EMPTY_LINES)); + Long id = sfe.getId(); + assertThrows(ValidationException.class, () -> + sourceFileProcessingService.process(id, false, true) + ); + } + + @Test + @Transactional + void processMissingLines() throws Exception { + SourceFileEntity sfe = sourceFileService.save(createEntity(TEST_ZIP_FILE_MISSING_LINES)); + Long id = sfe.getId(); + assertThrows(ValidationException.class, () -> + sourceFileProcessingService.process(id, false, true) + ); + } + + @Test + @Transactional + void processMissingIdFirst() throws Exception { + SourceFileEntity sfe = sourceFileService.save(createEntity(TEST_ZIP_FILE_MISSING_ID_FIRST)); + Long id = sfe.getId(); + assertThrows(ValidationException.class, () -> + sourceFileProcessingService.process(id, false, true) + ); + } + + @Test + @Transactional + void processMissingIdSecond() throws Exception { + SourceFileEntity sfe = sourceFileService.save(createEntity(TEST_ZIP_FILE_MISSING_ID_SECOND)); + Long id = sfe.getId(); + assertThrows(ValidationException.class, () -> + sourceFileProcessingService.process(id, false, true) + ); + } +} diff --git a/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileResourceDoProcessIT.java b/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileResourceDoProcessIT.java index da2df69..312856c 100644 --- a/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileResourceDoProcessIT.java +++ b/src/test/java/com/bonlimousin/replica/service/processcsv/SourceFileResourceDoProcessIT.java @@ -1,5 +1,7 @@ package com.bonlimousin.replica.service.processcsv; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.not; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -7,10 +9,10 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Optional; +import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; import org.junit.Assert; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; @@ -46,7 +48,9 @@ class SourceFileResourceDoProcessIT { * content is reduced to 2916 and her mother 2713 and father 2688 */ private static final String TEST_ZIP_FILE = "src/test/resources/fixtures/csv_se015112_truncated.zip"; - + + private static final String TEST_ZIP_FILE_UPDATED_NAME = "src/test/resources/fixtures/csv_se015112_upd_harst_name.zip"; + @Autowired private SourceFileService sourceFileService; @@ -58,8 +62,8 @@ class SourceFileResourceDoProcessIT { private SourceFileEntity sourceFileEntity; - public static SourceFileEntity createEntity() throws FileNotFoundException, IOException { - byte[] zipBytes = IOUtils.toByteArray(new FileInputStream(TEST_ZIP_FILE)); + public static SourceFileEntity createEntity(String fileName) throws FileNotFoundException, IOException { + byte[] zipBytes = IOUtils.toByteArray(new FileInputStream(fileName)); SourceFileEntity sourceFileEntity = new SourceFileEntity() .name("test.zip") .zipFile(zipBytes) @@ -68,15 +72,21 @@ public static SourceFileEntity createEntity() throws FileNotFoundException, IOEx .outcome(null); return sourceFileEntity; } - - @BeforeEach - public void initTest() throws FileNotFoundException, IOException { - sourceFileEntity = createEntity(); + + @Test + void missingSourceFile() throws Exception { + restSourceFileMockMvc.perform( + post("/api/source-files/{id}/process", Integer.MAX_VALUE) + .param("isRunAsync", "false") + .param("isDryRun", "true") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().is4xxClientError()); } @Test @Transactional void dryRunSyncSourceFileProcess() throws Exception { + sourceFileEntity = createEntity(TEST_ZIP_FILE); // Initialize the database sourceFileEntity = sourceFileService.save(sourceFileEntity); @@ -90,8 +100,8 @@ void dryRunSyncSourceFileProcess() throws Exception { Optional sfeOpt = sourceFileService.findOne(sourceFileEntity.getId()); Assert.assertTrue(sfeOpt.isPresent()); - Assert.assertNull(sfeOpt.get().getOutcome()); - Assert.assertNull(sfeOpt.get().getProcessed()); + Assert.assertNotNull(sfeOpt.get().getOutcome()); + Assert.assertNotNull(sfeOpt.get().getProcessed()); Optional opt = bovineRepository.findOneByEarTagId(2916); Assert.assertFalse(opt.isPresent()); @@ -100,7 +110,7 @@ void dryRunSyncSourceFileProcess() throws Exception { @Test @Transactional void syncSourceFileProcess() throws Exception { - // Initialize the database + sourceFileEntity = createEntity(TEST_ZIP_FILE); sourceFileEntity = sourceFileService.save(sourceFileEntity); Assert.assertNull(sourceFileEntity.getOutcome()); Assert.assertNull(sourceFileEntity.getProcessed()); @@ -138,4 +148,62 @@ void syncSourceFileProcess() throws Exception { // not impl, yet Assert.assertEquals(0, bovine2916.getJournalEntries().size()); } + + @Test + void sourceFileProcess() throws Exception { + SourceFileEntity sfe = createEntity(TEST_ZIP_FILE); + sfe = sourceFileService.save(sfe); + Assert.assertNull(sfe.getOutcome()); + Assert.assertNull(sfe.getProcessed()); + + // Process the sourceFile + restSourceFileMockMvc.perform( + post("/api/source-files/{id}/process", sfe.getId()) + .param("isRunAsync", "false") + .param("isDryRun", "false") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isNoContent()); + Optional sfeOpt = sourceFileService.findOne(sfe.getId()); + Assert.assertTrue(sfeOpt.isPresent()); + Assert.assertNotNull(sfeOpt.get().getOutcome()); + Assert.assertNotNull(sfeOpt.get().getProcessed()); + + sourceFileEntity = createEntity(TEST_ZIP_FILE_UPDATED_NAME); + sourceFileEntity = sourceFileService.save(sourceFileEntity); + Assert.assertNull(sourceFileEntity.getOutcome()); + Assert.assertNull(sourceFileEntity.getProcessed()); + + Optional optOrg = bovineRepository.findOneByEarTagId(2916); + Assert.assertTrue(optOrg.isPresent()); + Assert.assertThat(optOrg.get().getName(), not(containsString("TEST"))); + + restSourceFileMockMvc.perform( + post("/api/source-files/{id}/process", sourceFileEntity.getId()) + .param("isRunAsync", "true") + .param("isDryRun", "false") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isNoContent()); + + Optional sfeUpdOpt = null; + for(int i=1; i<10; i++) { + TimeUnit.SECONDS.sleep(i); + sfeUpdOpt = sourceFileService.findOne(sourceFileEntity.getId()); + if(sfeUpdOpt.isPresent() && sfeUpdOpt.get().getProcessed() != null) { + break; + } + } + if(sfeUpdOpt.isEmpty() || sfeUpdOpt.get().getProcessed() == null) { + Assert.fail("Could not find update of bovine name"); + } + + Optional opt = bovineRepository.findOneByEarTagId(2916); + Assert.assertTrue(opt.isPresent()); + + BovineEntity be = opt.get(); + Assert.assertThat(be.getName(), containsString("TEST")); + + bovineRepository.deleteAll(); + sourceFileService.delete(sfe.getId()); + sourceFileService.delete(sourceFileEntity.getId()); + } } diff --git a/src/test/resources/fixtures/csv_se015112_broken_header.zip b/src/test/resources/fixtures/csv_se015112_broken_header.zip new file mode 100644 index 0000000000000000000000000000000000000000..f13d3238c03d9ce2b2c311f4499f6be9e3ee3901 GIT binary patch literal 797 zcmWIWW@Zs#-~d960`@=#C{SW#U=UzXU`R>I*DKD7&qypPF40RaE(;CeWnk}h(v4pU z#2~t~f}4Sne6HtaUSg&zrC*S+}C0GHDBwTf!CN@Z8j|QJqoQOr2Y-k?T?#xggSh5h& z3Ld}*>yifm2w(^V2Sr=NMv%iPwAeU{L$QnlL{&f`AHAJCD{*P0A6EusKq46c07;J) zc|MD1MXH>(Tr_Yi5NeF~a?7(Fk0+X$lHw`))@vTleHoy*Rs0#(vf5VP%E9ouD7UsnF(f`x}X z1`zwFzhlxaeOJQGWNRka7A#;gQ&lc&;);KCU|H(d7P#gU1$8`nh1~SK69H3j#l%ft zx8(M6E4x(ZCM1Zs8zYa&a>A&BRLn>D{!WzmTwPWb53I+k@=bJ!*eurVmg!&%i>H#k z`f3fhE@Ns!Zj^g7l7MxBBa(p7t?0sC!-Zy}HP<*pj!gdUHe)Jjzw1*$b0aNYaG#c5 zKZ^%n0f(e<(Sw}Q$fg!8ZAfl)aY}S$`IDJ%50*};i(|9!nHTX64^eM&INHaBud$q* zN2betgK2#ZGx+N5@G>1Zugy9!3_fJ!BM_QxrBz~H+1tp1JQhwQ89uOcVD_J||H0N& zsq%o#bFw_U+NYc=OJ5ofpf941GQ#CXrwwV9cF3t2kvKc{C7s&PF}?E77InS#Leacz z#axJ}JTRpJAm9|h?gt_IRT2QDm=iv+0lR?>KB@o!{mh1lXgZBbiTuQe0SOOYOI#Yi z;KS8yGA$1YX&Z5Pf}0!aP51tLrXD^Y1A#9wku4Vkr$mPU!6!ie5{p{*7B4nGt-)j=9{0jnpM(@$Ot#QZJJ`vy}EYFb{$cyH5|5^ zYd0`9vi|P0HS=Ur_oGfOdEwYxE7d?&X)K*xw9Rn*cDqt3FSiy_Tyg_uBdnL|^7N7P5P2hKB4zV%AB zZqTUwYTR#b@hWzCDCZ_-Oq@dF za#(%Ony2ZC?0ArSll_3-SyDoj6%@f52fKmyH(>{eHdIY@oA;?#?AMM*ZCY^kDAx_) zEwu8Bd<2cDzg(~zSe{82Bu>VJF9l4FK2KK$e9!S+Bp>FMcejcr2{z(iwk{Pw`(!cr zx-0-lY5rLNs0{+hlE`C&02q`o>S1cwCjs1*5Pw4A()fh{qAvZU#x#VutmB74BSkXv z3CiK9Zf$pF4N-Ydk4OhnRGEleF37#}4a}v?HO|S?Kd;@VB7YWBtoI774B1xG@3@cY zCHtBJ39PSYLL2H>#=0+)jyK&7H`BS(gg+4)JW`9m_ku|Kvb)dqPGid-xjo(zlhN&% z0U)_^7f{Y?@9in%g2CCfAM0*(T%p$`!vJ@hdl|}kvMaL9d1*flFJSw-LgVp6l{M!N zEzxbvcE44Y7NzuO2DDnz(WOr(;ACXA#Jr7=d7$=GE}Gq&uCN=b-Gku7QNgtF&a z%PvuvqLLdwNlEp4-;QM(?mgYV?(?4ayff!L=lh(`=lOh|=Xov7VDtzGGjJsYU9*6! z#uO6-f`A|)?ruJc0bZ!%X#apfMU2nMAa7iNt<81_Q)Mah(SAb(R5x%3g|I@P^ovjk z1ULx-iM8!I+@!qiq4+%4{s0F#S7**ceG1_FR>2sBW8Lh5GZ5n*AkvX0&p47{LdBtGXCB{#3eP$(s4mj)J@2c+?to zvR70NA8jfUsdn@WeHZ!qw?~n_`2!}-44QC0u2dyKv)8HJ^)XW3*C^^? zyL2weFGfwjk=MCkN7venOi^kk1~q1#cf9gmfPk{hY0KdZsbq>w)2JnTh!`w|U&wjs z$JxvB_EO_;v>Ft{?ZMJhDMr|7fh5=?4CzecAQs58;Zf~^i;o+>zsSIRKEZ_fe0Mot zKr=}A$(PA?yr0Z6oZ_o3<{a1i$FM*Aljuva7@3+DF(X{Art3RM<4%*M#JVT{`c7wtc2hg=q3%Hvejw z>>@o`XR;oD%$WH6q4J7BcOh4N+I(lYGWjaG%4})WJOaUHkDWZ~cPzF6?=J62wBR1< z^zFQ_h*NwqTC!_O6Bc{N7;4y{G(-%w>l9ahb@TW1xtjrm^EUYe4c6vPxSxsuPn!R? zQ>hV04oL}<9ZIMC1K+DEcKTV%6q(9tL~%AaZoNDwdfJQx1ADLiVF zI7DKCykzC&KfT0mSCxcSwy6_U7Cm$J2CE6znd%L6zY^QgN$OA&KYq9?QH{8_0Tt_y*&PQ_ zFfrdKJ6mfGlnN}!)}?v_)Jza46$)j3P|xCsgOVcZXGhai&1rMhFs?ipPgghBIrBqy zbL@8UmUy~z{4gmFLFjxil5rlT!oeV{uUyr<)oqIde!SY;2 zab|{tv$a!pR(iTada)44HrsG${7wOWea^QmX52|!3#KXy)&lJV7by+JIG2}nR|{MoGBfrQ7dMHlZzErc5PMWmbhW^zQb%-WXpb>BP5Ea0=NNx+iC4)0k( z$SPo9szMai4vl^Nhi*j1!z5JFZJzXdch+H>rcsokr!8#qJCPR%2eP!&cpkcp5hrB7 zFg?zk{K^38>vTVPmKo3&$mj;w$yBX?0KDbeg6maT!2vnh05|xgX$I!0E^Lj+ZTJ`F zwe;otRR1j8j}L=dK%hcjpmgMd5K$#b1nJ%N4>Lnm3^HXd7V1WxM6!K0-En(gm0${y z?-jRyb(b*(sX8Gl=<=4wuprvr`jOj9@$Gu_k{;ZR61?N%w?6U?s;HXc$RUvCsIcz?j5VdO*oWtE%M7Q; zsN12u%#oooGsQ;>OAkEni(vgzlH{J5x%msmpZN>}Ir6&OmDmhu5oBU(jzFaf0~~Gk z;JP$v==#rGI80QZJeauS=E5A|weD#cj8z=lrzTEu56}z1*7c4O`}5tfj6ZCc8*PtF z{zn_m1B6i7Fc|HLJAw23WgA)^WQ$VfsJ;3rAx8_}6S7;EIl*7&Kt-W~P2SFmbDzR9 z6Imq3(Y;T*yZ>^(5@%hpy~$`mJNAi;$0g$rHpLa!^X*Q2EE|+7B&(^PMrCLu(~12a zg+UKoeQlm|hDE7W^$rYe;ft#5d9XK`OM4FERs$WpwlK=9)iVZd3WQgT-<&-&^+(VV z^WOcG4?-iwlXwc_(oJz#*YsjG`|_TKj{)MhGw~XN$mta}b59S!eFF!JGp7niuOWz-9pN!0-qfkbR5BZJC4!@yfA_pDR2DhJ+G_kd3t_h=6a%0Dk!+3e@74czaQ`#)cbj0FqYhf$Axa1UDx5O|nYrk*ZfqgV?Pp@x1P#(InV_f&y% zin=?4PFo8SL5gv64w^Q+#nht40G?0X2EiV`79@fhus&}lYaI?vs$Zasx}E$?moD=< zlyy)2d%D2JL~ZAv={g4AoMRQkKeQ=oy1?_Pee*M2CM;XgwGM|S)s}QI02PHm;J|ej K5ULI<`0xMngfT<_ literal 0 HcmV?d00001 diff --git a/src/test/resources/fixtures/csv_se015112_missing_id_second.zip b/src/test/resources/fixtures/csv_se015112_missing_id_second.zip new file mode 100644 index 0000000000000000000000000000000000000000..ab91f82ed31c122a2bc981ea75cf66c4bdd13502 GIT binary patch literal 3971 zcmc&$c{G&!AAV;h%%B>gOic{Rz9ovtzNE+yk<<($YxYP;2FcD%mWr`PBiy->WXXC7 z+4sbyDJqJRz4V*66DGO8U-$g}xS#jD=Y8kQIp5FuJn!>7p9gykwuu9P)2>jDWPRX! zjWPiM2Y>;rES)9X?QymiZtfQ(2+rp{9G%>ajWz>J_w(RqyR^h{EwtOiiv@sent=cS z?IZxWYTSIhMvC<@{|9yjcT-WkIh$_ho<55S2E(MHQ_nK(3v|j3A3uqGE-_SPR?=FL z^3KXaLv+VhmbuP_6s=p0-6PnNi^jqShqZstS)3eSc1lYvJiPqHqtE%`F2$EZO&ExH z=0R^)-h*LMC@RX5@f%uS>9__%q8&L7Ws@k%@G~$Zk(Je zUa1*>euAMjwjof(JTa^`#a5`s!FFoXuvAvKoI`t(PN1N1q4vP*sFungLC0jOtoJ^Z zc(Iv)u{UBWH%>z9YcY|MwV58(w|+Zya}q_wVMz@nHn00ke$jGe#nwEUb3QjnFJ0L> znBT@%_XWy82-~cg+wb!c_v&MaCLCJOqrrea=9*7HaTyr5*F;32nz*^=_o~y z-EB)R8vR{`7ZFq!Wj^!Zj)#42hgZ1G-zlLBi?m%4!$7x!2-|e2f-d=f}cXG8J5sKCCC8mDmNP3 zm<(39QNsWL$e+03?)i-u-rC|gAKDE**YV(Bp=iVG8-&XjJb_2i+Pe>=CuSLRWzo ziTEuk%N(~B#rvcQDXMJbkJ}zT&Mtr7jXWz~y;QnTU&n}zMEZ$|S2uMxGN8rUp?tBk ziXR8#(tSl@G7f#k&PPwlH9RX0w0{$Q-<){SiDczVK5Ij4oxo+~br<7SC3G~E=pefW#VCmkRDV7s<+?h_vM z`Hca+wup{gUblNPQg3D+Ec~|L2~$=(bMdD6ZOVXa=huP*sw9+r^Vth#BxjHfv67ss z6dD`AZ@w`b-q2{Vi^c{VPES=`|AeWe1nx)ESVxSspNT=ha}sRumiQ?>-IG&WPljOq zp)p)ALF6vT2Tu&+2b?sL0j;hUkg6LK1yjil%?d(@NK{s0;*wJCC75DkVsjE=P0bC> zl5SnUZhAcjjbt_UfrRWuacQuRA|vB=t)479 ztvapIQR^PRJCAh8nt%4N(q?3WNBEKk!)a52lda86?~Z0L0NB2!0DET-Hz$kpzwD0& zO-?l!WSPlZeU}zq&se1>V}fRU2fE{PrJ27Tr3z|KzVcU|I+|kTZCWK=Bc*_3*v&8c z!f&zS&bb`V=Hd}2LHiN016p0pc|N|F0Fh4r^pJ_L=#SH1{7Y?eH=p!8XQ~P*{e7CO z_`aUXc`Eu9gQA6MYWEin;pj0-TvFMQ?P09d0vbYI2^V*a_c8Vg1Wj}0B}UfbC`EH@ z52Sx9jr@J!ov2c1E1n!z5HNk|NT=`hqb>H!jsg;{f`Rk->K$mYvTaWVd=PA31w(mrtB3=SLJJ8Qf zFBJ)Hv%XO_YdfZTl{vK~-kUho&lS-_etoW$Don)M?4YI!48MM~nLmOvzZbA8Yu$ak zzbs^Wg6ol(;#U0K6NwyNb>2!*_~p73Dm1_--F7B`Yw+=iRe9EcA!CGdqGiW{VuvT@ zzLztvokyfCOeX(v?`fd&*RQiWAIW)+`8ppOgZb2YgOjoY`}In056ZSydkFbFbx(O! zDzB)KHgCt#^KL3DzeNASH{%`>V09d)Q5FPjV<c1w|zD(H{s$tyO z(^JIPYL{U!7Ji3L8GfpjySlqWMaT2Zu5?QW#&2p(2{ge({8zPppz(`Nt)3P(PIgW< zzpPfQ>Xra0WZCWcP|{)lcCQ0!@K84uB}$gKQR-ew%)C!R7_-2zMMr;2%X_Pv!G@I0 zHQL?ESKEZG@)W_c6w;GC{TzO{eGpP}0YMqges^W`p~nfm4h8Bb^s}57{#3@fY<^ht z*i6#a#e-F|?)>)>{N;9G#y)S+v$5WFs9QBBVKhU8Sso!*GQ15GStaV2U_VA2f zS8Zm$-+l!JU4jFXCw`}M^N|W!gNuFrbusPG+)zi^%^BfAQu5b^5j^;qB89R_AE5 zekBoXEL>3jV3=?0{%7{|Wyj{62eNuY;QShzb0(>@`N1hSp8xNa@pn-CYszTRai|UC zxTFj>+SE?B#~Q2~$m{l)#I*DKD7&qypPF40RaE(;CeWnedV(v7zR zVh~+g!Og(P@|BT+fd!=1b&}(zBMKZY?X`C#0Nzt1ZcB645zeadEp$6^BN;-OeRD!zWFTT6sWas~i6uw)g(m_i(IY z2q?X;7kK}AOL4fBoKpqIq38>(jV-?G{w+Ti&vd`^N8h|lIeV`)N^-sPioNRA%yhJ+ ztiDIm_`E`e+QC^2*(_@$KS<3eO?s5N=fi`{q~)D+)xvz&ctu~`?49>8vD!{^p~|h> z;hAA?3-4|@_DCvC@?%j{*zZHhGAq7&hjnSxAIyC9nTOHhg>d!DmtB@q{+(j35AbH^ zNS|@t_$koCpnwX12VntH5Pk;A3L%0pE5EcTFEIxmh^o*)1kq@LXnWRC=&%9@>xJDr z-YwfJ$ahPC*CNTbp4Tnh=HD51_Y+?k+`bpJcyX|;oepw7!eeDf9&14LSXp9vUS?i8++&7NkAY}(kDUq> zYEa-{xv=}g-KEk#P7M=+Ojw-%Atk4@ z$y5C9#GBXs+-ZDv7uxJ(j5egc_X+AMVFtPOn$$~8KcIU+ZVK>bWD;RUq-EqN0;Odb z*wzSQk(#oRO$S9YayWsa83wjBIy1seCn3_2%?E`xvd>V>mjNa@sOw1x`v7lNHjo>c OflvFZ)eW!&iCHuZ*OCk|9*IwC)tE|J= z6GnC-GB-uPnNDev`}^H{et+EWGtZeh^PKbjoX`7tecsReL2801Spd|;Bg`vP7w~OO zLjV94032Xt;Uel`kG3;+_wW?8baC`@a`rGZ*av`=6jT4^p(~2EB%WTrv;ZLG2Ot1I z+z9}~>|*fA&C&ziSRo!xSt7-Om&)z~rY=GhkB1&Od4g8!lL5iHYzq%q$YA zS4nnU1a~F01uK}QhBx1}JKE%6w?a97;z5LzLr;cAuz+Ei+Q@8lXZ=k9r%b%W)sqUz z!XJWW=7bdzjVM~1;Zet%bG;hV;*9US;|N3}v)Zy4eM=zzF;dl4Hry(NYl&G?Zj4x( z`Q*mIU~!-&vkgs82~Xe=U3lOH7FCLKApl+Y?Q20VIX`qYetDB%`SCKu@?+^{4u_1_ z!R}9!F9X~JH=%e}c^=acY~`Rk^v}anJU0cOzu+;1JHSvc2l@04yYz|+xwg9S1DzVy zHL7FjkBeWon#-9nXhmP0(rarI4t(h365=VaYHr5r4@7E0sExUl^q&!LC5Zs!4jV!m z)Oz39=m!G;uwU5l@czn0Hj#&W#FO;Bi;Fz{5qlyRe%y=P=duPE1?HCAf@|rhf;S+e zL(b=0z12q>xBY7$b?6(NM*a~e%&pP-(!S9kMb=tksAp^|sZ375RBw)g4j(%aV~;h{ zFzqZ@81e0|jEeYhfOgq}jRSLk%J+Q+B)S}wy0Ez@xV{Z*pg&^3j=(6QmJb$L2V|h6 zw{Rxcr>j+2j~!#MEYv|9Mi?mZff4ZveETTrgt5)0 z3g!W~LRC=0jTVf_;TS{*6E{dY0KB$YCgK*pxxrknJ~|;?9wo?5MFV|EnFp)q;hV+h zfAUZnWm|lBx(oh$>|7YIXI9|EJI3197*g$56igoQRmP)u?4;-g=jpNUa zm*QxxxV&edW98LeoM;=w1iLT$U#X>hhvjNp!8#gWou5>j&)m3|782i<9*?p-GxS(; z#PxIzmg1>FBSquhtYZjYjB~OX<#d0;;e`A^BpwX7@Zb_6 z52U-kHxF|}FgS@kpwZfj%DU%K$3@XU+r}<5W`9M)(u~#8*380eMO({gg~2Ek89))s z4ibQI0$07^RI6w)7??|0DJWYjHX5W*6!zdIR7kYG9-fqO_fZN88Jkd;l7KSRH_1qg zk4ME9a>3{fuK`1kaIhm7=V>&VBbe6CiLL2#yu>Bp8w#B-O;PCG+N7dlQ|*6nS+zxV zdfKMnKkunVfy#@1I%1Cy>d5O^uOo> z6C753VP4m@^RRbo1U`SVRv@$7 zif1*Bc?*^?SIow3Ap~P)EYKO%>TKb3jr@qCz9~NY7Y3;Y_;0SU7pF!wqcIf(#&WT^ z=TT2amX67Vb(s|;l?JU{QSZGTf6g@p=1;R^Sk&8D?nuqya5R#t#g=pcg&@PYjexlaFDoz4SVFLg*8uD`j#E zOw!t~uWE^!$QMb`A6j2{M7iTPn)?6%raiR&hLul+AR0?VYL6#TW-{76{RFJ#&Dmbf zm0@hp99*yFUQ>9SRLD2V3&6I9~=m1M!suImpnQnX`5HuJz@XEAj zA(hiv;~`Bw5Y!=a_nFNcp_TSwfuxv~>Zv-pak%k|I3>n;?#gWaKm3x|bLyP>`6XfL z*AnJi*ItR>SlXB$QbG;D397E1w_?6iAE46YS6UlUj(gAZ^z9jl zOAgC~c1F2|oN*lXBF||z?&L^JE)7cO)0B#mOW4#{{KM)n><5mpG9A7S(K@5z;<1uA z9&kiFH7&33Xv^??F=x>wexX}q2)KCFtN{O(Cj|`p+L{;VkD05bjMlsz2ku%o$sITj97U!C=C@CA6}e|6SX zq8E@ktGBtW^JQn-zwN9@Mf#uCJy&xwSvsW}tLB)?*5@f_Hnxi(5I+w&UO1rtb^UoPkO`8)qhw zQO}_%P)C%AB*YIgT*(YEmUxBC5O*Z{hC|+88=_bACHh7~#oz_Wl)0 z2Rqh__by-VI*v5yvr`(p)pHVIO%(}a0u_fB5O&S=vF9)XFoDM?-Xx!Rg zeSm2r1xk3|?l*&OW3{doey+m9Xclj2Qp3R`m%#BQMcD36Jn^rp+$9@ZDI2t8DGz_$ zS8DRM)0s+%l8_qTTav#XmcLTxUmKQA4Tt7nmMikq29!NyF{I6=tzcISK@6w9`Y|j( z3Kr0h7d7&jPI^)MzA6GqzkbeNfA6&osrjPzORbFVCY_A?X-}mmyJjC`DbmMfF?*BbxZ)yR7w?N2^8Kg)%o+e@yUc6OE8Q!X&^jsgHs;&GdpSo4XG HZvfyw7>b7D literal 0 HcmV?d00001