diff --git a/pom.xml b/pom.xml index b28c19e..1f634b7 100644 --- a/pom.xml +++ b/pom.xml @@ -1,30 +1,32 @@ - 4.0.0 org.springframework.boot spring-boot-starter-parent 3.4.0 - + de.avatic taricdb 0.0.1-SNAPSHOT taricdb custom duties api - + - + - + - - - - + + + + 23 @@ -39,17 +41,18 @@ spring-boot-starter-web - org.springframework.boot + org.springframework.boot spring-boot-starter-data-jdbc - - org.mariadb.jdbc - mariadb-java-client - + + com.mysql + mysql-connector-j + runtime + org.springframework.boot spring-boot-starter-test - test + test org.springframework.boot @@ -57,28 +60,53 @@ - org.thymeleaf - thymeleaf-spring6 + org.thymeleaf + thymeleaf-spring6 - org.apache.poi - poi - 5.0.0 + org.apache.poi + poi + 5.0.0 - org.apache.poi - poi-ooxml - 5.0.0 + org.apache.poi + poi-ooxml + 5.0.0 + + + + org.codehaus.mojo + license-maven-plugin + 2.5.0 + + + + org.springframework.boot spring-boot-maven-plugin + + org.codehaus.mojo + license-maven-plugin + + + + + org.codehaus.mojo + license-maven-plugin + 2.5.0 + + + + + diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/ExcelRow.java b/src/main/java/de/avatic/taricdb/helper/workbook/ExcelRow.java index 28eb4d4..529b801 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/ExcelRow.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/ExcelRow.java @@ -26,6 +26,15 @@ public class ExcelRow { private List date; + /** + * Constructor for the ExcelRow class. + * It processes a given Excel row and extracts cell values, converting them to appropriate formats, + * and associates them with an Excel header for mapping column indices. + * + * @param row the Excel row object to be processed, represented as a {@link Row}. + * @param header the ExcelHeader object containing header information for column mappings. + * @throws IllegalArgumentException if a cell type in the row is unsupported. + */ public ExcelRow(Row row, ExcelHeader header) { this.date = new ArrayList<>(); this.row = row; @@ -71,6 +80,14 @@ public class ExcelRow { + /** + * Retrieves the value of a specific column from the Excel row and converts it to the specified type. + * + * @param string the name of the column to retrieve the value from + * @param clazz the class type to which the value will be converted; supported types are String, Integer, Boolean, and LocalDate + * @return the value of the column converted to the specified type, or null if the column value is empty or invalid for the requested type + * @throws IllegalArgumentException if the column with the specified name is not found in the header + */ @SuppressWarnings("unchecked") public T getAs(String string, Class clazz) { int columnIdx = getColumnIdx(string.trim()); @@ -121,6 +138,14 @@ public class ExcelRow { return null; } + /** + * Retrieves the column index associated with the specified column name + * from the header information of an Excel sheet. + * + * @param string the name of the column whose index is to be retrieved + * @return the index of the specified column, or -1 if the column name + * is not found in the header + */ private int getColumnIdx(String string) { int idx = header.getIndex(string); @@ -131,6 +156,11 @@ public class ExcelRow { + /** + * Retrieves the index of the current Excel row. + * + * @return the index of the row as an integer + */ public int index() { return index; } diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/AdditionalCodeConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/AdditionalCodeConverter.java index 6ed6fac..9a91e5d 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/AdditionalCodeConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/AdditionalCodeConverter.java @@ -27,6 +27,10 @@ public class AdditionalCodeConverter implements ExcelTableConverter c.getAdditionalCode().equals(code)).findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConditionConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConditionConverter.java index 6c2194d..84f7569 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConditionConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConditionConverter.java @@ -1,7 +1,9 @@ package de.avatic.taricdb.helper.workbook.converters; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; import org.springframework.data.jdbc.core.mapping.AggregateReference; @@ -38,6 +40,11 @@ public class AppliedMeasureConditionConverter implements ExcelTableConverter c.getHsCode().equals(hsCode) + .filter(c -> c.getHsCode().equals(hsCode) && c.getOriginCode().equals(originCode) - && c.getAdditionalCode().equals(additionalCode) - && c.getOrderNumber().equals(orderNumber) + && Objects.equals(c.getAdditionalCode(), additionalCode) + && Objects.equals(c.getOrderNumber(), orderNumber) && c.getAppliedMeasureCondition().getSequenceNo().equals(sequenceNumber) - && c.getAppliedMeasureCondition().getConditionType().getId().equals(conditionType.getId()) - && c.getMeasureCode().equals(measureCode)) + && Objects.equals(c.getAppliedMeasureCondition().getConditionType().getId(), conditionType != null ? conditionType.getId() : null) + && c.getMeasureCode().equals(measureCode) + && Objects.equals(c.getStartDate(), startDate) + && Objects.equals(c.getEndDate(), endDate)) .findAny().orElse(null); if (condition == null) { condition = new AppliedMeasureConditionWrapper( - new AppliedMeasureCondition(AggregateReference.to(action.getId()), + new AppliedMeasureCondition(action != null ? AggregateReference.to(action.getId() ): null, monetaryUnit == null ? null : AggregateReference.to(monetaryUnit.getId()), unit == null ? null : AggregateReference.to(unit.getId()), certificate == null ? null : AggregateReference.to(certificate.getId()), - AggregateReference.to(conditionType.getId()), amount, sequenceNumber), - hsCode, originCode, additionalCode, orderNumber, measureCode); + conditionType != null ? AggregateReference.to( conditionType.getId() ): null, amount, sequenceNumber), + hsCode, originCode, additionalCode, orderNumber, measureCode, startDate, endDate); conditions.add(condition); } else { throw new IllegalArgumentException("Duplicate applied measure condtion found: " + hsCode + " " diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConverter.java index dab1f08..e831cab 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/AppliedMeasureConverter.java @@ -1,137 +1,147 @@ package de.avatic.taricdb.helper.workbook.converters; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Set; -import java.util.stream.Collectors; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.jdbc.core.mapping.AggregateReference; - import de.avatic.taricdb.helper.stats.HeapStat; import de.avatic.taricdb.helper.workbook.ExcelTable; import de.avatic.taricdb.helper.workbook.wrapper.AppliedMeasureConditionWrapper; import de.avatic.taricdb.helper.workbook.wrapper.AppliedMeasureWrapper; import de.avatic.taricdb.helper.workbook.wrapper.MeasureExclusionWrapper; import de.avatic.taricdb.helper.workbook.wrapper.MeasureFootnoteWrapper; -import de.avatic.taricdb.model.AppliedMeasure; -import de.avatic.taricdb.model.AppliedMeasureCondition; -import de.avatic.taricdb.model.LegalBase; -import de.avatic.taricdb.model.Measure; -import de.avatic.taricdb.model.MeasureExclusion; -import de.avatic.taricdb.model.MeasureFootnote; +import de.avatic.taricdb.model.*; import de.avatic.taricdb.service.ReferenceStorageService; import de.avatic.taricdb.service.SetupStatusService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.jdbc.core.mapping.AggregateReference; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; public class AppliedMeasureConverter implements ExcelTableConverter { - Logger log = LoggerFactory.getLogger(AppliedMeasureConverter.class); - - private ReferenceStorageService referenceStorage; - private SetupStatusService statusService; - private int recordOffset; - private int count = 0; + Logger log = LoggerFactory.getLogger(AppliedMeasureConverter.class); - public AppliedMeasureConverter(ReferenceStorageService referenceStorage, SetupStatusService statusService, int recordCurrentCount) { - this.referenceStorage = referenceStorage; - this.statusService = statusService; - this.recordOffset = recordCurrentCount; - } + private ReferenceStorageService referenceStorage; + private SetupStatusService statusService; + private int recordOffset; + private int count = 0; - @Override - public Collection convert(ExcelTable table) { - Collection appliedMeasures = new ArrayList<>(); + public AppliedMeasureConverter(ReferenceStorageService referenceStorage, SetupStatusService statusService, int recordCurrentCount) { + this.referenceStorage = referenceStorage; + this.statusService = statusService; + this.recordOffset = recordCurrentCount; + } - Collection measureFootnoteWrappers = referenceStorage - .getEntites(MeasureFootnoteWrapper.class); - Collection measures = referenceStorage.getEntites(Measure.class); - Collection measureExclusions = referenceStorage - .getEntites(MeasureExclusionWrapper.class); - Collection appliedMeasureConditions = referenceStorage - .getEntites(AppliedMeasureConditionWrapper.class); + @Override + public Collection convert(ExcelTable table) { + Collection appliedMeasures = new ArrayList<>(); - Collection legalBases = referenceStorage.getEntites(LegalBase.class); + Collection measureFootnoteWrappers = referenceStorage + .getEntites(MeasureFootnoteWrapper.class); + Collection measures = referenceStorage.getEntites(Measure.class); + Collection measureExclusions = referenceStorage + .getEntites(MeasureExclusionWrapper.class); + Collection appliedMeasureConditions = referenceStorage + .getEntites(AppliedMeasureConditionWrapper.class); - table.getRows().forEach((row) -> { - count++; - if (row.index() == 0) { - return; - } - - - if (count % 10000 == 0) { - int percentage = Math.round((count*100)/table.getRows().size()); - log.info("applied measures processed: "+percentage+"% - " + count + " of " + table.getRows().size() + " " + HeapStat.getHeapUtilization(), true, null); - statusService.setStatus(recordOffset, "Convert applied measures ... " + percentage + "% ", true , null); - } + Collection legalBases = referenceStorage.getEntites(LegalBase.class); - String hsCode = row.getAs("Goods code", String.class); - String origin = row.getAs("Origin code", String.class); - String addCode = row.getAs("Add code", String.class); - Integer measureCode = row.getAs("Meas. type code", Integer.class); - String measureCodeStr = row.getAs("Meas. type code", String.class); - String legalBase = row.getAs("Legal base", String.class); - String orderNumber = row.getAs("Order No.", String.class); - Integer orderNumberInt = row.getAs("Order No.", Integer.class); - - LocalDate startDate = row.getAs("Start date", LocalDate.class); - LocalDate endDate = row.getAs("End date", LocalDate.class); - String amount = row.getAs("Duty", String.class); - - Measure measure = measures.stream().filter(c -> c.getMeasureCode().equals(measureCodeStr)).findAny() - .orElse(null); - - if (null == measure) { - throw new RuntimeException("Measure not found: " + row.getAs("Meas. type code", String.class)); - } - - Set foundMeasureFootnotes = measureFootnoteWrappers.stream() - .filter(s -> s.getMeasureCode().equals(measureCode) && s.getHsCode().equals(hsCode) - && s.getOrigin().equals(origin) && s.getAddCode().equals(addCode) && s.getOrderNumber().equals(orderNumber)) - .map(MeasureFootnoteWrapper::getMeasureFootnote).collect(Collectors.toSet()); - - Set foundMeasureExclusions = null; - - if(isNumeric(origin)) { - Integer originCode = Integer.parseInt(origin); - foundMeasureExclusions = measureExclusions.stream() - .filter(s -> s.getMeasureCode().equals(measureCode) && s.getHsCode().equals(hsCode) - && s.getOriginCode().equals(originCode) && s.getAddCode().equals(addCode) && s.getOrderNumber().equals(orderNumber)) - .map(MeasureExclusionWrapper::getMeasureExclusion).collect(Collectors.toSet()); - } + table.getRows().forEach((row) -> { + count++; + if (row.index() == 0) { + return; + } - Set foundMeasureConditions = appliedMeasureConditions.stream() - .filter(s -> s.getHsCode().equals(hsCode) && s.getOriginCode().equals(origin) - && s.getAdditionalCode().equals(addCode) && s.getOrderNumber().equals(orderNumber) && s.getMeasureCode().equals(measureCode)) - .map(AppliedMeasureConditionWrapper::getAppliedMeasureCondition).collect(Collectors.toSet()); + if (count % 10000 == 0) { + int percentage = Math.round((float) (count * 100) / table.getRows().size()); + log.info("applied measures processed: " + percentage + "% - " + count + " of " + table.getRows().size() + " " + HeapStat.getHeapUtilization(), true, null); + statusService.setStatus(recordOffset, "Convert applied measures ... " + percentage + "% ", true, null); + } - LegalBase foundLegalBase = legalBases.stream().filter(s -> s.getLegalBase().equals(legalBase)).findAny() - .orElse(null); + String hsCode = row.getAs("Goods code", String.class); - if (legalBase == null) { - throw new RuntimeException("Legal base not found: " + legalBase); - } + if (hsCode == null || hsCode.isEmpty()) { + return; + } - AppliedMeasureWrapper appliedMeasure = new AppliedMeasureWrapper( - new AppliedMeasure(AggregateReference.to(measure.getId()), foundMeasureFootnotes, - foundLegalBase == null ? null : AggregateReference.to(foundLegalBase.getId()), foundLegalBase == null ? legalBase : null, foundMeasureExclusions, - foundMeasureConditions, startDate, endDate, amount, orderNumberInt), - hsCode, addCode, origin); + String origin = row.getAs("Origin code", String.class); + String addCode = row.getAs("Add code", String.class); + Integer measureCode = row.getAs("Meas. type code", Integer.class); + String measureCodeStr = row.getAs("Meas. type code", String.class); + String legalBase = row.getAs("Legal base", String.class); + String orderNumber = row.getAs("Order No.", String.class); + Integer orderNumberInt = row.getAs("Order No.", Integer.class); - appliedMeasures.add(appliedMeasure); + LocalDate startDate = row.getAs("Start date", LocalDate.class); + LocalDate endDate = row.getAs("End date", LocalDate.class); + String amount = row.getAs("Duty", String.class); - }); + Measure measure = measures.stream().filter(c -> c.getMeasureCode().equals(measureCodeStr)).findAny() + .orElse(null); - return appliedMeasures; - } - + if (null == measure) { + throw new RuntimeException("Measure not found: " + row.getAs("Meas. type code", String.class)); + } - private boolean isNumeric(String str) { - return str.matches("-?\\d+(\\.\\d+)?"); // match a number with optional '-' and decimal. - } + Set foundMeasureFootnotes = measureFootnoteWrappers.stream() + .filter(s -> Objects.equals(s.getMeasureCode(), measureCode) + && Objects.equals(s.getHsCode(), hsCode) + && Objects.equals(s.getOrigin(), origin) + && Objects.equals(s.getAddCode(), addCode) + && Objects.equals(s.getOrderNumber(), orderNumber)) + .map(MeasureFootnoteWrapper::getMeasureFootnote).collect(Collectors.toSet()); + + Set foundMeasureExclusions = null; + + if (isNumeric(origin)) { + Integer originCode = Integer.parseInt(origin); + foundMeasureExclusions = measureExclusions.stream() + .filter(s -> Objects.equals(s.getMeasureCode(), measureCode) + && Objects.equals(s.getHsCode(), hsCode) + && Objects.equals(s.getOriginCode(), originCode) + && Objects.equals(s.getAddCode(), addCode) + && Objects.equals(s.getOrderNumber(), orderNumber)) + .map(MeasureExclusionWrapper::getMeasureExclusion).collect(Collectors.toSet()); + } + + + Set foundMeasureConditions = appliedMeasureConditions.stream() + .filter(s -> Objects.equals(s.getHsCode(), hsCode) + && Objects.equals(s.getOriginCode(), origin) + && Objects.equals(s.getAdditionalCode(), addCode) + && Objects.equals(s.getOrderNumber(), orderNumber) + && Objects.equals(s.getMeasureCode(), measureCode)) + .map(AppliedMeasureConditionWrapper::getAppliedMeasureCondition) + .collect(Collectors.toSet()); + + LegalBase foundLegalBase = legalBases.stream().filter(s -> s.getLegalBase().equals(legalBase)).findAny() + .orElse(null); + + if (foundLegalBase == null) { + throw new RuntimeException("Legal base not found: " + legalBase); + } + + AppliedMeasureWrapper appliedMeasure = new AppliedMeasureWrapper( + new AppliedMeasure(AggregateReference.to(measure.getId()), foundMeasureFootnotes, + AggregateReference.to(foundLegalBase.getId()), legalBase, foundMeasureExclusions, + foundMeasureConditions, startDate, endDate, amount, orderNumberInt), + hsCode, addCode, origin); + + appliedMeasures.add(appliedMeasure); + + }); + + return appliedMeasures; + } + + + private boolean isNumeric(String str) { + return str.matches("-?\\d+(\\.\\d+)?"); // match a number with optional '-' and decimal. + } } diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateConverter.java index 013d5b1..dc8eb00 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateConverter.java @@ -34,6 +34,10 @@ public class CertificateConverter implements ExcelTableConverter { if (row.index() == 0) { return; } + + if (code == null || code.isEmpty()) { + return; + } Certificate cert = certs.stream().filter(s -> s.getCertificateCode().equals(code)).findAny().orElse(null); @@ -42,7 +46,7 @@ public class CertificateConverter implements ExcelTableConverter { String certTypeCode = String.valueOf(code.charAt(0)); CertificateType foundCertType = referenceStorage.getEntites(CertificateType.class).stream().filter(s -> s.getCertificateTypeCode().equals(certTypeCode)).findAny().orElse(null); - cert = new Certificate(code, row.getAs("Start date", LocalDate.class), row.getAs("End date", LocalDate.class), AggregateReference.to(foundCertType.getId())); + cert = new Certificate(code, row.getAs("Start date", LocalDate.class), row.getAs("End date", LocalDate.class), foundCertType == null ? null : AggregateReference.to(foundCertType.getId())); certs.add(cert); } diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateTypeConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateTypeConverter.java index 770c1e5..471bf3b 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateTypeConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/CertificateTypeConverter.java @@ -23,6 +23,10 @@ public class CertificateTypeConverter implements ExcelTableConverter s.getCertificateTypeCode().equals(code)).findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/ConditionTypeConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/ConditionTypeConverter.java index 9d096a0..05c7a52 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/ConditionTypeConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/ConditionTypeConverter.java @@ -26,6 +26,10 @@ public class ConditionTypeConverter implements ExcelTableConverter s.getConditionTypeCode().equals(code)).findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/DeclarableCodesConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/DeclarableCodesConverter.java index 93a7fcc..2370b53 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/DeclarableCodesConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/DeclarableCodesConverter.java @@ -20,6 +20,15 @@ public class DeclarableCodesConverter implements ExcelTableConverter { String goodsCode = row.getAs("Goods code", String.class); + + if (row.index() == 0) { + return; + } + + if (goodsCode == null || goodsCode.isEmpty()) { + return; + } + String hsCode = goodsCode.substring(0, 10); String suffix = goodsCode.substring(11, 13); Boolean isLeaf = row.getAs("IS_LEAF", Boolean.class); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/FootnoteConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/FootnoteConverter.java index c464d3d..211c11c 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/FootnoteConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/FootnoteConverter.java @@ -25,6 +25,10 @@ public class FootnoteConverter implements ExcelTableConverter { if (row.index() == 0) { return; } + + if (txt == null || txt.isEmpty()) { + return; + } Footnote footnote = footnotes.stream().filter(f -> f.getFootnote().equals(txt)).findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoConverter.java index 28e8b26..209975c 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoConverter.java @@ -27,6 +27,10 @@ public class GeoConverter implements ExcelTableConverter { if (row.index() == 0) { return; } + + if (origin == null || origin.isEmpty()) { + return; + } if(!isNumeric(origin)) { Geo geo = geos.stream().filter(s -> s.getIso3166Code().equals(origin)).findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupConverter.java index e263aee..2161620 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupConverter.java @@ -32,6 +32,10 @@ public class GeoGroupConverter implements ExcelTableConverter { return; } + if (origin == null || origin.isEmpty()) { + return; + } + if (isNumeric(origin)) { Integer groupId = Integer.parseInt(origin); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupMembershipConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupMembershipConverter.java index 63b50dc..fae560d 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupMembershipConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/GeoGroupMembershipConverter.java @@ -3,6 +3,7 @@ package de.avatic.taricdb.helper.workbook.converters; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; import org.springframework.data.jdbc.core.mapping.AggregateReference; @@ -34,11 +35,16 @@ public class GeoGroupMembershipConverter implements ExcelTableConverter s.getIso3166Code().equals(member)).findAny().orElse(null); + Geo geo = geos.stream().filter(s -> s.getIso3166Code().equals(member)).findAny().orElseThrow(() -> new RuntimeException("Geo not found: " + member)); Integer memberId = geo.getId(); - GeoGroupMembershipWrapper membership = memberships.stream().filter(s -> s.getGeoGroupId().equals(group) && s.getGeoGroupMembership().getGeo().getId().equals(memberId)).findAny().orElse(null); + GeoGroupMembershipWrapper membership = memberships.stream().filter(s -> s.getGeoGroupId().equals(group) && Objects.equals(s.getGeoGroupMembership().getGeo().getId(), memberId)).findAny().orElse(null); if (membership == null) { membership = new GeoGroupMembershipWrapper(group, new GeoGroupMembership(row.getAs("Mbship start date", LocalDate.class), row.getAs("Mbship end date", LocalDate.class), AggregateReference.to(memberId))); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBase2Converter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBase2Converter.java new file mode 100644 index 0000000..7d22f09 --- /dev/null +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBase2Converter.java @@ -0,0 +1,48 @@ +package de.avatic.taricdb.helper.workbook.converters; + +import de.avatic.taricdb.helper.workbook.ExcelTable; +import de.avatic.taricdb.model.LegalBase; +import de.avatic.taricdb.service.ReferenceStorageService; + +import java.util.ArrayList; +import java.util.Collection; + +public class LegalBase2Converter implements ExcelTableConverter { + + private ReferenceStorageService referenceStorage; + + public LegalBase2Converter(ReferenceStorageService referenceStorage) { + this.referenceStorage = referenceStorage; + } + + @Override + public Collection convert(ExcelTable table) { + Collection legalBases = new ArrayList<>(); + + Collection existingLegalBases = referenceStorage.getEntites(LegalBase.class); + + table.getRows().forEach((row) -> { + + String legalBaseStr = row.getAs("Legal base", String.class); + + if (row.index() == 0) { + return; + } + + if (legalBaseStr == null || legalBaseStr.isEmpty()) { + return; + } + + LegalBase foundLegalBase = existingLegalBases.stream().filter(s -> s.getLegalBase().equals(legalBaseStr)).findAny() + .orElse(null); + + if (foundLegalBase == null) { + LegalBase legalBase = new LegalBase(legalBaseStr, null, null, null); + legalBases.add(legalBase); + } + }); + + return legalBases; + + } +} diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBaseConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBaseConverter.java index dc9d341..b68f207 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBaseConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/LegalBaseConverter.java @@ -19,13 +19,19 @@ public class LegalBaseConverter implements ExcelTableConverter { table.getRows().forEach((row) -> { + String legalBaseStr = row.getAs("LEGAL_BASE", String.class); + if (row.index() == 0) { return; } - LegalBase legalBase = new LegalBase(row.getAs("Legal base", String.class), - row.getAs("Off. Journal", String.class), row.getAs("Page", Integer.class), - row.getAs("Publ. date", LocalDate.class)); + if (legalBaseStr == null || legalBaseStr.isEmpty()) { + return; + } + + LegalBase legalBase = new LegalBase(legalBaseStr, + row.getAs("OFF_JOURNAL", String.class), row.getAs("PAGE", Integer.class), + row.getAs("PUBL_DATE", LocalDate.class)); legalBases.add(legalBase); }); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureActionConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureActionConverter.java index 02fd950..dbad611 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureActionConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureActionConverter.java @@ -25,6 +25,10 @@ public class MeasureActionConverter implements ExcelTableConverter s.getMeasureActionCode().equals(code)).findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureConverter.java index 9e0349d..2f71d29 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureConverter.java @@ -14,7 +14,7 @@ import de.avatic.taricdb.service.ReferenceStorageService; public class MeasureConverter implements ExcelTableConverter { - private ReferenceStorageService referenceStorage; + private final ReferenceStorageService referenceStorage; public MeasureConverter(ReferenceStorageService referenceStorage) { this.referenceStorage = referenceStorage; @@ -33,6 +33,10 @@ public class MeasureConverter implements ExcelTableConverter { return; } + if (code == null || code.isEmpty()) { + return; + } + Measure measure = measures.stream().filter(s -> s.getMeasureCode().equals(code)).findAny().orElse(null); @@ -41,7 +45,7 @@ public class MeasureConverter implements ExcelTableConverter { MeasureSeries foundSeries = series.stream().filter(s -> s.getMeasureSeriesCode().equals(seriesCode)).findAny().orElse(null); measure = new Measure(code, row.getAs("Short descr.", String.class), - row.getAs("TM code", Integer.class), row.getAs("Start date", LocalDate.class), AggregateReference.to(foundSeries.getId())); + row.getAs("TM code", Integer.class), row.getAs("Start date", LocalDate.class), foundSeries == null ? null : AggregateReference.to(foundSeries.getId())); measures.add(measure); } diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureExclusionConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureExclusionConverter.java index f1b76ae..0cc3e3e 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureExclusionConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureExclusionConverter.java @@ -1,7 +1,9 @@ package de.avatic.taricdb.helper.workbook.converters; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; import org.springframework.data.jdbc.core.mapping.AggregateReference; @@ -31,22 +33,35 @@ public class MeasureExclusionConverter implements ExcelTableConverter c.getIso3166Code().equals(iso3166Code)).findAny().orElse(null); + LocalDate startDate = row.getAs("Start date", LocalDate.class); + LocalDate endDate = row.getAs("End date", LocalDate.class); + + Geo geo = geos.stream().filter(c -> c.getIso3166Code().equals(iso3166Code)).findAny().orElseThrow(() -> new RuntimeException("Geo not found: " + iso3166Code)); MeasureExclusionWrapper exclusion = exclusions.stream() - .filter(c -> c.getHsCode().equals(hsCode) && c.getMeasureCode().equals(measureCode) - && c.getOriginCode().equals(originCode) && c.getIso3166Code().equals(iso3166Code) && c.getAddCode().equals(addCode) && c.getOrderNumber().equals(orderNumber)) + .filter(c -> c.getHsCode().equals(hsCode) + && c.getMeasureCode().equals(measureCode) + && c.getOriginCode().equals(originCode) + && c.getIso3166Code().equals(iso3166Code) + && c.getAddCode().equals(addCode) + && c.getOrderNumber().equals(orderNumber) + && Objects.equals(c.getStartDate(), startDate) + && Objects.equals(c.getEndDate(), endDate)) .findAny().orElse(null); if (exclusion == null) { exclusion = new MeasureExclusionWrapper(new MeasureExclusion(AggregateReference.to(geo.getId())), - hsCode, originCode, measureCode, iso3166Code, addCode, orderNumber); + hsCode, originCode, measureCode, iso3166Code, addCode, orderNumber, startDate, endDate); exclusions.add(exclusion); } else { throw new IllegalArgumentException("Duplicate measure exclusion found: measure: " + measureCode + " - origin: " + originCode + " - iso: " + iso3166Code + " - add code: " + addCode + " goods code: " + hsCode + " order number: " + orderNumber); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureFootnoteConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureFootnoteConverter.java index 5283c06..70fa6e6 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureFootnoteConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureFootnoteConverter.java @@ -1,7 +1,9 @@ package de.avatic.taricdb.helper.workbook.converters; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; import org.springframework.data.jdbc.core.mapping.AggregateReference; @@ -13,7 +15,7 @@ import de.avatic.taricdb.service.ReferenceStorageService; public class MeasureFootnoteConverter implements ExcelTableConverter { - private ReferenceStorageService referenceStorage; + private final ReferenceStorageService referenceStorage; public MeasureFootnoteConverter(ReferenceStorageService referenceStorage) { this.referenceStorage = referenceStorage; @@ -31,25 +33,40 @@ public class MeasureFootnoteConverter implements ExcelTableConverter s.getFootnote().equals(footnote)).findAny() .orElse(null); MeasureFootnoteWrapper wrapper = measureFootnoteWrappers.stream() - .filter(s -> s.getMeasureFootnote().getFootnote().getId().equals(foundFootnote.getId()) - && s.getMeasureCode().equals(measureCode) && s.getHsCode().equals(hsCode) - && s.getOrigin().equals(origin) && s.getAddCode().equals(addCode) - && s.getOrderNumber().equals(orderNumber)) + .filter(s -> + Objects.equals(s.getMeasureFootnote().getFootnote().getId(), foundFootnote != null ? foundFootnote.getId() : null) + && s.getMeasureCode().equals(measureCode) + && s.getHsCode().equals(hsCode) + && s.getOrigin().equals(origin) + && Objects.equals(s.getAddCode(), addCode) + && Objects.equals(s.getOrderNumber(),orderNumber) + && Objects.equals(s.getStartDate(),startDate) + && Objects.equals(s.getEndDate(),endDate)) .findAny().orElse(null); if (wrapper == null) { - wrapper = new MeasureFootnoteWrapper(new MeasureFootnote(AggregateReference.to(foundFootnote.getId())), - measureCode, hsCode, addCode, origin, orderNumber); + wrapper = new MeasureFootnoteWrapper(new MeasureFootnote(foundFootnote != null ? AggregateReference.to(foundFootnote.getId() ): null), + measureCode, hsCode, addCode, origin, orderNumber, startDate, endDate); + measureFootnoteWrappers.add(wrapper); } else { throw new RuntimeException("MeasureFootnoteWrapper already exists: " + measureCode + " " + hsCode + " " + origin + " " + addCode + " " + orderNumber); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureSeriesConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureSeriesConverter.java index ee26027..4bc996f 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureSeriesConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MeasureSeriesConverter.java @@ -26,6 +26,10 @@ public class MeasureSeriesConverter implements ExcelTableConverter s.getMeasureSeriesCode().equals(code)).findAny().orElse(null); if (measureSeries == null) { diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MonetaryUnitConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MonetaryUnitConverter.java index c1cb001..e641c18 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/MonetaryUnitConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/MonetaryUnitConverter.java @@ -27,6 +27,10 @@ public class MonetaryUnitConverter implements ExcelTableConverter return; } + if (code == null || code.isEmpty()) { + return; + } + MonetaryUnit monetaryUnit = units.stream().filter( s -> s.getMonetaryUnitCode().equals(code)) .findAny().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureDeConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureDeConverter.java index 885235c..07df561 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureDeConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureDeConverter.java @@ -35,13 +35,18 @@ public class NomenclatureDeConverter implements ExcelTableConverter n.getHscode().equals(hsCode)).findAny().orElse(null); NomenclatureFootnoteWrapper footnoteWrapper = nomenclatureFootnoteWrappers.stream().filter(n -> n.getHsCode().equals(hsCode) && n.getSuffix().equals(suffix)).findFirst().orElse(null); - NomenclatureDesc descEn = nomenclatureDescEn.stream().filter(n -> n.getHscode().equals(hsCode) && n.getSuffix().equals(suffix)).map(n -> n.getDesc()).findFirst().orElse(null); - NomenclatureDesc descFr = nomenclatureDescFr.stream().filter(n -> n.getHscode().equals(hsCode) && n.getSuffix().equals(suffix)).map(n -> n.getDesc()).findFirst().orElse(null); + NomenclatureDesc descEn = nomenclatureDescEn.stream().filter(n -> n.getHscode().equals(hsCode) && n.getSuffix().equals(suffix)).map(NomenclatureDescEnWrapper::getDesc).findFirst().orElse(null); + NomenclatureDesc descFr = nomenclatureDescFr.stream().filter(n -> n.getHscode().equals(hsCode) && n.getSuffix().equals(suffix)).map(NomenclatureDescFrWrapper::getDesc).findFirst().orElse(null); diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureEnConverter.java b/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureEnConverter.java index b5a761c..64189cf 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureEnConverter.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/converters/NomenclatureEnConverter.java @@ -30,6 +30,11 @@ public class NomenclatureEnConverter implements ExcelTableConverter { String goodsCode = row.getAs("Goods code", String.class); + + if (row.index() == 0) { + return; + } + + if(goodsCode == null || goodsCode.isEmpty()) + return; + String hsCode = goodsCode.substring(0, 10); String suffix = goodsCode.substring(11, 13); - Footnote footnote = footnotes.stream().filter(f -> f.getFootnote().equals(row.getAs("Footnote code", String.class))).findFirst().orElse(null); + Footnote footnote = footnotes.stream().filter(f -> f.getFootnote().equals(row.getAs("Footnote code", String.class))).findFirst().orElseThrow( () -> new RuntimeException("Footnote not found: " + row.getAs("Footnote code", String.class))); NomenclatureFootnoteWrapper foundWrapper = nomenclatureFootnoteWrappers.stream().filter(w -> w.getHsCode().equals(hsCode)).findAny().orElse(null); @@ -39,7 +47,7 @@ public class NomenclatureFootnoteConverter implements ExcelTableConverter { return; } + if (code == null || code.isEmpty()) { + return; + } + Unit unit = units.stream().filter( s -> s.getUnitCode().equals(code) && (((qualifier == null) && (s.getUnitQualifier() == null)) || ((s.getUnitQualifier() != null && qualifier != null) diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/AppliedMeasureConditionWrapper.java b/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/AppliedMeasureConditionWrapper.java index bb0ac36..a778799 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/AppliedMeasureConditionWrapper.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/AppliedMeasureConditionWrapper.java @@ -2,6 +2,8 @@ package de.avatic.taricdb.helper.workbook.wrapper; import de.avatic.taricdb.model.AppliedMeasureCondition; +import java.time.LocalDate; + public class AppliedMeasureConditionWrapper { private AppliedMeasureCondition appliedMeasureCondition; @@ -11,15 +13,19 @@ public class AppliedMeasureConditionWrapper { private String orderNumber; private Integer measureCode; + private LocalDate startDate; + private LocalDate endDate; - public AppliedMeasureConditionWrapper(AppliedMeasureCondition appliedMeasureCondition, String hsCode, String originCode, String additionalCode, String orderNumber, Integer measureCode) { + public AppliedMeasureConditionWrapper(AppliedMeasureCondition appliedMeasureCondition, String hsCode, String originCode, String additionalCode, String orderNumber, Integer measureCode, LocalDate startDate, LocalDate endDate) { this.appliedMeasureCondition = appliedMeasureCondition; this.hsCode = hsCode; this.originCode = originCode; this.additionalCode = additionalCode; this.orderNumber = orderNumber; this.measureCode = measureCode; + this.startDate = startDate; + this.endDate = endDate; } public AppliedMeasureCondition getAppliedMeasureCondition() { @@ -46,4 +52,11 @@ public class AppliedMeasureConditionWrapper { return measureCode; } + public LocalDate getStartDate() { + return startDate; + } + + public LocalDate getEndDate() { + return endDate; + } } diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureExclusionWrapper.java b/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureExclusionWrapper.java index 30140b6..544679a 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureExclusionWrapper.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureExclusionWrapper.java @@ -2,6 +2,8 @@ package de.avatic.taricdb.helper.workbook.wrapper; import de.avatic.taricdb.model.MeasureExclusion; +import java.time.LocalDate; + public class MeasureExclusionWrapper { private MeasureExclusion exclusion; @@ -17,8 +19,11 @@ public class MeasureExclusionWrapper { private String addCode; private String orderNumber; + + private LocalDate startDate; + private LocalDate endDate; - public MeasureExclusionWrapper(MeasureExclusion exclusion, String hsCode, Integer originCode, Integer measureCode, String iso3166Code, String addCode, String orderNumber) { + public MeasureExclusionWrapper(MeasureExclusion exclusion, String hsCode, Integer originCode, Integer measureCode, String iso3166Code, String addCode, String orderNumber, LocalDate startDate, LocalDate endDate) { this.exclusion = exclusion; this.hsCode = hsCode; this.originCode = originCode; @@ -26,6 +31,8 @@ public class MeasureExclusionWrapper { this.iso3166Code = iso3166Code; this.addCode = addCode; this.orderNumber = orderNumber; + this.startDate = startDate; + this.endDate = endDate; } public MeasureExclusion getExclusion() { @@ -59,6 +66,13 @@ public class MeasureExclusionWrapper { public Object getAddCode() { return addCode; } - + + public LocalDate getStartDate() { + return startDate; + } + + public LocalDate getEndDate() { + return endDate; + } } diff --git a/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureFootnoteWrapper.java b/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureFootnoteWrapper.java index d72cc5b..cebfb55 100644 --- a/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureFootnoteWrapper.java +++ b/src/main/java/de/avatic/taricdb/helper/workbook/wrapper/MeasureFootnoteWrapper.java @@ -2,6 +2,8 @@ package de.avatic.taricdb.helper.workbook.wrapper; import de.avatic.taricdb.model.MeasureFootnote; +import java.time.LocalDate; + public class MeasureFootnoteWrapper { private MeasureFootnote measureFootnote; @@ -15,14 +17,19 @@ public class MeasureFootnoteWrapper { private String addCode; private Integer orderNumber; + + private LocalDate startDate; + private LocalDate endDate; - public MeasureFootnoteWrapper(MeasureFootnote measureFootnote, Integer measureCode, String hsCode, String addCode, String origin, Integer orderNumber) { + public MeasureFootnoteWrapper(MeasureFootnote measureFootnote, Integer measureCode, String hsCode, String addCode, String origin, Integer orderNumber, LocalDate startDate, LocalDate endDate) { this.measureFootnote = measureFootnote; this.measureCode = measureCode; this.hsCode = hsCode; this.origin = origin; this.addCode = addCode; this.orderNumber = orderNumber; + this.startDate = startDate; + this.endDate = endDate; } public Integer getMeasureCode() { @@ -57,4 +64,11 @@ public class MeasureFootnoteWrapper { return orderNumber; } + public LocalDate getStartDate() { + return startDate; + } + + public LocalDate getEndDate() { + return endDate; + } } diff --git a/src/main/java/de/avatic/taricdb/helper/zip/Unzipper.java b/src/main/java/de/avatic/taricdb/helper/zip/Unzipper.java index b53c2c4..ab9b672 100644 --- a/src/main/java/de/avatic/taricdb/helper/zip/Unzipper.java +++ b/src/main/java/de/avatic/taricdb/helper/zip/Unzipper.java @@ -26,6 +26,15 @@ public class Unzipper { Logger log = LoggerFactory.getLogger(Unzipper.class); + /** + * Extracts a ZIP file from a byte array and returns a list of in-memory files. + * The method iterates through the entries in the ZIP archive, and for each non-directory entry, + * it creates an {@code InMemFile} object containing the file name and content. + * + * @param zipFile A byte array containing the ZIP file data. + * @return A list of {@code InMemFile} objects representing the files extracted from the ZIP archive. + * @throws IOException If an I/O error occurs during the extraction process. + */ public List unzip(byte[] zipFile) throws IOException { List files = new ArrayList<>(); ZipInputStream zipStream = new ZipInputStream(new ByteArrayInputStream(zipFile)); @@ -33,7 +42,7 @@ public class Unzipper { while ((entry = zipStream.getNextEntry()) != null) { if (!entry.isDirectory()) { - files.add(new InMemFile(entry.getName(), readFileContent(zipStream, entry))); + files.add(new InMemFile(entry.getName(), readFileContent(zipStream))); } zipStream.closeEntry(); log.info("Unzip done: " + entry.getName()); @@ -42,25 +51,56 @@ public class Unzipper { return files; } - - private byte[] readFileContent(InputStream zipStream, ZipEntry entry) throws IOException { + + private static final int CHUNK_SIZE = 10240; // Konstant für die Blockgröße + + /** + * Reads the content of a specified {@code ZipEntry} from the provided {@code InputStream}. + * The method processes the input stream in chunks, stores the read data, and combines it to form the complete file content. + * + * @param zipStream The input stream of the ZIP archive. + * @return A byte array containing the content of the specified ZIP entry. + * @throws IOException If an I/O error occurs while reading the entry content. + */ + private byte[] readFileContent(InputStream zipStream) throws IOException { + List chunks = new ArrayList<>(); // Zwischenspeicherung der gelesenen Blöcke. int totalRead = 0; - List content = new ArrayList<>(); - - while (zipStream.available() > 0) { - byte[] bytes = new byte[10240]; - int read = zipStream.read(bytes, 0, bytes.length); - - if (read == -1) { - break; + + int bytesRead; + do { + byte[] chunk = readChunk(zipStream); // Lese einen einzelnen Datenblock. + bytesRead = chunk.length; + if (bytesRead > 0) { + chunks.add(chunk); + totalRead += bytesRead; } - - content.add(new ByteArrayWrapper(Arrays.copyOf(bytes, read))); - totalRead += read; - } - - return content.stream().reduce(new ByteArrayWrapper(new byte[0]), ByteArrayWrapper::combine).getBytes(); - + } while (bytesRead > 0); + + // Kombiniere alle gelesenen Blöcke. + byte[] result = new byte[totalRead]; + int destPos = 0; + for (byte[] chunk : chunks) { + System.arraycopy(chunk, 0, result, destPos, chunk.length); + destPos += chunk.length; + } + + return result; + } + + /** + * Reads a single chunk of data from the {@code InputStream}. + * + * @param zipStream The input stream of the ZIP archive. + * @return A byte array containing the read chunk (might be smaller than CHUNK_SIZE at the end of the stream). + * @throws IOException If an I/O error occurs during the read operation. + */ + private byte[] readChunk(InputStream zipStream) throws IOException { + byte[] buffer = new byte[CHUNK_SIZE]; + int bytesRead = zipStream.read(buffer); + if (bytesRead == -1) { + return new byte[0]; // Kein weiterer Inhalt vorhanden. + } + return Arrays.copyOf(buffer, bytesRead); // Rückgabe mit tatsächlicher Größe. } diff --git a/src/main/java/de/avatic/taricdb/service/AsyncSetupDutiesService.java b/src/main/java/de/avatic/taricdb/service/AsyncSetupDutiesService.java index 71d5282..def52cc 100644 --- a/src/main/java/de/avatic/taricdb/service/AsyncSetupDutiesService.java +++ b/src/main/java/de/avatic/taricdb/service/AsyncSetupDutiesService.java @@ -49,8 +49,15 @@ public class AsyncSetupDutiesService { Geo geo = geos.stream().filter(g -> g.getIso3166Code().equals(origin)).findFirst().orElse(null); GeoGroup geoGroup = geoGroups.stream().filter(g -> g.getGeoGroupCode().equals(origin)).findFirst().orElse(null); - im.completeImport(new Import(AggregateReference.to(nomenclature.getId()), additionalCode == null ? null : AggregateReference.to(additionalCode.getId()), foundAppliedMeasures, geo == null ? null : AggregateReference.to(geo.getId()), geoGroup == null ? null : AggregateReference.to(geoGroup.getId()))); - + if( nomenclature == null ) { + logger.error("No nomenclature found for duty: {}", hsCode); + im.completeImport(null); + } + else { + im.completeImport(new Import(AggregateReference.to(nomenclature.getId()), additionalCode == null ? null : AggregateReference.to(additionalCode.getId()), foundAppliedMeasures, geo == null ? null : AggregateReference.to(geo.getId()), geoGroup == null ? null : AggregateReference.to(geoGroup.getId()))); + } + + } } diff --git a/src/main/java/de/avatic/taricdb/service/SetupService.java b/src/main/java/de/avatic/taricdb/service/SetupService.java index 0827516..21c3649 100644 --- a/src/main/java/de/avatic/taricdb/service/SetupService.java +++ b/src/main/java/de/avatic/taricdb/service/SetupService.java @@ -1,13 +1,10 @@ package de.avatic.taricdb.service; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; +import de.avatic.taricdb.helper.workbook.converters.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -20,29 +17,6 @@ import de.avatic.taricdb.helper.workbook.ExcelFileType; import de.avatic.taricdb.helper.workbook.ExcelRow; import de.avatic.taricdb.helper.workbook.ExcelTable; import de.avatic.taricdb.helper.workbook.ExcelTableType; -import de.avatic.taricdb.helper.workbook.converters.AdditionalCodeConverter; -import de.avatic.taricdb.helper.workbook.converters.AppliedMeasureConditionConverter; -import de.avatic.taricdb.helper.workbook.converters.AppliedMeasureConverter; -import de.avatic.taricdb.helper.workbook.converters.CertificateConverter; -import de.avatic.taricdb.helper.workbook.converters.CertificateTypeConverter; -import de.avatic.taricdb.helper.workbook.converters.ConditionTypeConverter; -import de.avatic.taricdb.helper.workbook.converters.DeclarableCodesConverter; -import de.avatic.taricdb.helper.workbook.converters.FootnoteConverter; -import de.avatic.taricdb.helper.workbook.converters.GeoConverter; -import de.avatic.taricdb.helper.workbook.converters.GeoGroupConverter; -import de.avatic.taricdb.helper.workbook.converters.GeoGroupMembershipConverter; -import de.avatic.taricdb.helper.workbook.converters.LegalBaseConverter; -import de.avatic.taricdb.helper.workbook.converters.MeasureActionConverter; -import de.avatic.taricdb.helper.workbook.converters.MeasureConverter; -import de.avatic.taricdb.helper.workbook.converters.MeasureExclusionConverter; -import de.avatic.taricdb.helper.workbook.converters.MeasureFootnoteConverter; -import de.avatic.taricdb.helper.workbook.converters.MeasureSeriesConverter; -import de.avatic.taricdb.helper.workbook.converters.MonetaryUnitConverter; -import de.avatic.taricdb.helper.workbook.converters.NomenclatureDeConverter; -import de.avatic.taricdb.helper.workbook.converters.NomenclatureEnConverter; -import de.avatic.taricdb.helper.workbook.converters.NomenclatureFootnoteConverter; -import de.avatic.taricdb.helper.workbook.converters.NomenclatureFrConverter; -import de.avatic.taricdb.helper.workbook.converters.UnitConverter; import de.avatic.taricdb.helper.workbook.wrapper.AppliedMeasureConditionWrapper; import de.avatic.taricdb.helper.workbook.wrapper.AppliedMeasureWrapper; import de.avatic.taricdb.helper.workbook.wrapper.DeclarableCodesWrapper; @@ -201,6 +175,8 @@ public class SetupService { measureExclusions(); measureConditions(); + legalBase2(); + appliedMeasure(); importCodes(); @@ -259,7 +235,7 @@ public class SetupService { for (Collection chunk : toChunks(m, 1000)) { referenceStorage.addAll(toList(importRepository.saveAll(chunk)), Import.class, Integer.class); totalProcessed += chunk.size(); - statusService.setStatus(statusService.getRecordTotalCount(), "Save import duties ... " + Math.round((totalProcessed * 100) / m.size()) + "%", true, null); + statusService.setStatus(statusService.getRecordTotalCount(), "Save import duties ... " + Math.round((float) (totalProcessed * 100) / m.size()) + "%", true, null); } } @@ -276,7 +252,7 @@ public class SetupService { } if (count % 10000 == 0) { - int percentage = Math.round((count * 100) / table.getRows().size()); + int percentage = Math.round((float) (count * 100) / table.getRows().size()); log.info("import duties processed: " + percentage + "% - " + count + " of " + table.getRows().size() + " " + HeapStat.getHeapUtilization(), true, null); statusService.setStatus(recordCurrentCount, "Convert import duties ... " + percentage + "%", @@ -301,9 +277,22 @@ public class SetupService { } } - return imports.stream().map(ImportWrapper::getImport).collect(Collectors.toSet()); + return imports.stream().map(ImportWrapper::getImport).filter(Objects::nonNull).collect(Collectors.toSet()); } + + private void legalBase2() { + log.info("Convert legal base from applied measure"); + statusService.setStatus(recordCurrentCount, "Convert legal base (2)", true, null); + Collection m = new LegalBase2Converter(referenceStorage) + .convert(tables.get(ExcelTableType.APPLIED_MEASURES)); + + log.info("Convert legal base (found {} additional legal bases)", m.size()); + + referenceStorage.addAll(toList(legalBaseRepository.saveAll(m)), LegalBase.class, Integer.class); + + } + private void appliedMeasure() { log.info("Convert applied measures"); statusService.setStatus(recordCurrentCount, "Convert applied measures", true, null); diff --git a/src/main/java/de/avatic/taricdb/web/ImportController.java b/src/main/java/de/avatic/taricdb/web/ImportController.java index dbdac5f..4527675 100644 --- a/src/main/java/de/avatic/taricdb/web/ImportController.java +++ b/src/main/java/de/avatic/taricdb/web/ImportController.java @@ -6,4 +6,4 @@ import org.springframework.web.bind.annotation.RestController; public class ImportController { -} +} \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5bed244..a7a312c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,10 +1,16 @@ spring.application.name=taricdb -spring.datasource.url=jdbc:mariadb://localhost:3306/taric +spring.datasource.url=jdbc:mysql://localhost:3306/taric_import spring.datasource.username=spring -spring.datasource.password=z8gc!w6L$Z5J32$n2Tbw#!Sjj5usxRTxNg67 -spring.datasource.driver-class-name=org.mariadb.jdbc.Driver +spring.datasource.password=wrHpGVYwX5YHpmEsXX7y +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver spring.sql.init.mode=always spring.profiles.active=setup spring.servlet.multipart.enabled=true spring.servlet.multipart.max-file-size=30MB -spring.servlet.multipart.max-request-size=30MB \ No newline at end of file +spring.servlet.multipart.max-request-size=30MB +spring.datasource.hikari.maximum-pool-size=10 +spring.datasource.hikari.minimum-idle=5 +spring.datasource.hikari.connection-timeout=30000 +spring.datasource.hikari.idle-timeout=600000 +spring.datasource.hikari.max-lifetime=1800000 +spring.datasource.hikari.leak-detection-threshold=60000 \ No newline at end of file diff --git a/src/main/resources/schema.sql b/src/main/resources/schema.sql index 983f424..3c68538 100644 --- a/src/main/resources/schema.sql +++ b/src/main/resources/schema.sql @@ -82,7 +82,8 @@ create table if not exists `geo` id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, iso_3166_code CHAR(2), start_date DATE, - end_date DATE + end_date DATE, + UNIQUE (iso_3166_code) ); create table if not exists `geo_group` @@ -172,6 +173,8 @@ create table if not exists `measure_footnote` id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, footnote_id INT NOT NULL, applied_measure_id INT NOT NULL, + start_date DATE, + end_date DATE, FOREIGN KEY (footnote_id) REFERENCES footnote(id), FOREIGN KEY (applied_measure_id) REFERENCES applied_measure(id) ); @@ -187,6 +190,8 @@ create table if not exists `applied_measure_condition` certificate_id INT DEFAULT NULL, condition_type_id INT NOT NULL, amount TEXT, + start_date DATE, + end_date DATE, FOREIGN KEY (applied_measure_id) REFERENCES applied_measure(id), FOREIGN KEY (measure_action_id) REFERENCES measure_action(id), FOREIGN KEY (monetary_unit_id) REFERENCES monetary_unit(id), @@ -200,6 +205,8 @@ create table if not exists `measure_exclusion` id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, geo_id INT NOT NULL, applied_measure_id INT NOT NULL, + start_date DATE, + end_date DATE, FOREIGN KEY (geo_id) REFERENCES geo(id), FOREIGN KEY (applied_measure_id) REFERENCES applied_measure(id) ); diff --git a/uploads/taric_data.zip b/uploads/taric_data.zip index f85890c..de95f7d 100644 Binary files a/uploads/taric_data.zip and b/uploads/taric_data.zip differ diff --git a/uploads/taric_data_10_2025.zip b/uploads/taric_data_10_2025.zip new file mode 100644 index 0000000..793044a Binary files /dev/null and b/uploads/taric_data_10_2025.zip differ