diff --git a/pom.xml b/pom.xml index 8f633db..94a45e6 100644 --- a/pom.xml +++ b/pom.xml @@ -114,6 +114,16 @@ poi 5.4.1 + + org.dhatim + fastexcel + 0.17.0 + + + org.dhatim + fastexcel-reader + 0.17.0 + org.apache.poi poi-ooxml diff --git a/src/frontend/src/components/UI/AppListItem.vue b/src/frontend/src/components/UI/AppListItem.vue index a48c8f7..fc022a2 100644 --- a/src/frontend/src/components/UI/AppListItem.vue +++ b/src/frontend/src/components/UI/AppListItem.vue @@ -43,7 +43,7 @@ export default { display: grid; grid-template-columns: 1fr 2fr 0.5fr; grid-gap: 2.4rem; - padding: 1.6rem 0; + padding: 1.2rem 0; font-weight: 400; font-size: 1.4rem; border-bottom: 0.1rem solid #E3EDFF; diff --git a/src/frontend/src/components/layout/error/ErrorModalOverview.vue b/src/frontend/src/components/layout/error/ErrorModalOverview.vue index a296852..df076e3 100644 --- a/src/frontend/src/components/layout/error/ErrorModalOverview.vue +++ b/src/frontend/src/components/layout/error/ErrorModalOverview.vue @@ -109,9 +109,9 @@ export default { await navigator.clipboard.writeText(JSON.stringify(this.error)); this.$refs.toast.addToast({ - icon: 'Bug', + icon: 'Clipboard', message: "The error has been copied to clipboard", - title: "Copied to clipboard", + title: "Successful copied to clipboard", variant: 'success', duration: 4000 }) diff --git a/src/main/java/de/avatic/lcc/service/bulk/BulkExportService.java b/src/main/java/de/avatic/lcc/service/bulk/BulkExportService.java index b437cce..6baf9f7 100644 --- a/src/main/java/de/avatic/lcc/service/bulk/BulkExportService.java +++ b/src/main/java/de/avatic/lcc/service/bulk/BulkExportService.java @@ -30,8 +30,9 @@ public class BulkExportService { private final HiddenNodeExcelMapper hiddenNodeExcelMapper; private final HiddenCountryExcelMapper hiddenCountryExcelMapper; private final String sheetPassword; + private final MaterialFastExcelMapper materialFastExcelMapper; - public BulkExportService(@Value("${lcc.bulk.sheet_password}") String sheetPassword, HeaderCellStyleProvider headerCellStyleProvider, ContainerRateExcelMapper containerRateExcelMapper, MatrixRateExcelMapper matrixRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, HiddenNodeExcelMapper hiddenNodeExcelMapper, HiddenCountryExcelMapper hiddenCountryExcelMapper) { + public BulkExportService(@Value("${lcc.bulk.sheet_password}") String sheetPassword, HeaderCellStyleProvider headerCellStyleProvider, ContainerRateExcelMapper containerRateExcelMapper, MatrixRateExcelMapper matrixRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, HiddenNodeExcelMapper hiddenNodeExcelMapper, HiddenCountryExcelMapper hiddenCountryExcelMapper, MaterialFastExcelMapper materialFastExcelMapper) { this.headerCellStyleProvider = headerCellStyleProvider; this.containerRateExcelMapper = containerRateExcelMapper; this.matrixRateExcelMapper = matrixRateExcelMapper; @@ -41,10 +42,24 @@ public class BulkExportService { this.hiddenNodeExcelMapper = hiddenNodeExcelMapper; this.hiddenCountryExcelMapper = hiddenCountryExcelMapper; this.sheetPassword = sheetPassword; + this.materialFastExcelMapper = materialFastExcelMapper; + } + + public void processOperation(BulkOperation op) throws IOException { + var bulkFileType = op.getFileType(); + + // Use FastExcel for MATERIAL, Apache POI for others + if (bulkFileType.equals(BulkFileType.MATERIAL)) { + byte[] excelData = materialFastExcelMapper.exportToExcel(); + op.setFile(excelData); + } else { + processOperationWithApachePOI(op); + } } - public void processOperation(BulkOperation op) throws IOException { + + public void processOperationWithApachePOI(BulkOperation op) throws IOException { var bulkFileType = op.getFileType(); var periodId = op.getValidityPeriodId(); @@ -78,10 +93,10 @@ public class BulkExportService { matrixRateExcelMapper.fillSheet(worksheet, style, periodId); matrixRateExcelMapper.createConstraints(workbook, worksheet); break; - case MATERIAL: - materialExcelMapper.fillSheet(worksheet, style); - materialExcelMapper.createConstraints(worksheet); - break; +// case MATERIAL: +// materialExcelMapper.fillSheet(worksheet, style); +// materialExcelMapper.createConstraints(worksheet); +// break; case PACKAGING: packagingExcelMapper.fillSheet(worksheet, style); packagingExcelMapper.createConstraints(workbook, worksheet); diff --git a/src/main/java/de/avatic/lcc/service/bulk/BulkImportService.java b/src/main/java/de/avatic/lcc/service/bulk/BulkImportService.java index 9acb10b..aa77891 100644 --- a/src/main/java/de/avatic/lcc/service/bulk/BulkImportService.java +++ b/src/main/java/de/avatic/lcc/service/bulk/BulkImportService.java @@ -1,5 +1,6 @@ package de.avatic.lcc.service.bulk; +import de.avatic.lcc.dto.bulk.BulkFileType; import de.avatic.lcc.model.bulk.BulkFileTypes; import de.avatic.lcc.model.bulk.BulkOperation; import de.avatic.lcc.service.api.BatchGeoApiService; @@ -31,8 +32,9 @@ public class BulkImportService { private final MatrixRateImportService matrixRateImportService; private final ContainerRateImportService containerRateImportService; private final BatchGeoApiService batchGeoApiService; + private final MaterialFastExcelMapper materialFastExcelMapper; - public BulkImportService(MatrixRateExcelMapper matrixRateExcelMapper, ContainerRateExcelMapper containerRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, NodeBulkImportService nodeBulkImportService, PackagingBulkImportService packagingBulkImportService, MaterialBulkImportService materialBulkImportService, MatrixRateImportService matrixRateImportService, ContainerRateImportService containerRateImportService, BatchGeoApiService batchGeoApiService) { + public BulkImportService(MatrixRateExcelMapper matrixRateExcelMapper, ContainerRateExcelMapper containerRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, NodeBulkImportService nodeBulkImportService, PackagingBulkImportService packagingBulkImportService, MaterialBulkImportService materialBulkImportService, MatrixRateImportService matrixRateImportService, ContainerRateImportService containerRateImportService, BatchGeoApiService batchGeoApiService, MaterialFastExcelMapper materialFastExcelMapper) { this.matrixRateExcelMapper = matrixRateExcelMapper; this.containerRateExcelMapper = containerRateExcelMapper; this.materialExcelMapper = materialExcelMapper; @@ -44,9 +46,27 @@ public class BulkImportService { this.matrixRateImportService = matrixRateImportService; this.containerRateImportService = containerRateImportService; this.batchGeoApiService = batchGeoApiService; + this.materialFastExcelMapper = materialFastExcelMapper; } public void processOperation(BulkOperation op) throws IOException { + var file = op.getFile(); + var type = op.getFileType(); + + // Use FastExcel for MATERIAL, Apache POI for others + if (type.equals(BulkFileType.MATERIAL)) { + processOperationWithFastExcel(op); + } else { + processOperationWithApachePOI(op); + } + } + + private void processOperationWithFastExcel(BulkOperation op) throws IOException { + var materials = materialFastExcelMapper.importFromExcel(op.getFile()); + materials.forEach(materialBulkImportService::processMaterialInstructions); + } + + private void processOperationWithApachePOI(BulkOperation op) throws IOException { var file = op.getFile(); var type = op.getFileType(); @@ -76,10 +96,10 @@ public class BulkImportService { var matrixRates = matrixRateExcelMapper.extractSheet(sheet); matrixRateImportService.processMatrixRates(matrixRates); break; - case MATERIAL: - var materials = materialExcelMapper.extractSheet(sheet); - materials.forEach(materialBulkImportService::processMaterialInstructions); - break; +// case MATERIAL: +// var materials = materialExcelMapper.extractSheet(sheet); +// materials.forEach(materialBulkImportService::processMaterialInstructions); +// break; case PACKAGING: var packaging = packagingExcelMapper.extractSheet(sheet); packaging.forEach(packagingBulkImportService::processPackagingInstructions); diff --git a/src/main/java/de/avatic/lcc/service/bulk/BulkOperationExecutionService.java b/src/main/java/de/avatic/lcc/service/bulk/BulkOperationExecutionService.java index 81d62f1..8c1ecb0 100644 --- a/src/main/java/de/avatic/lcc/service/bulk/BulkOperationExecutionService.java +++ b/src/main/java/de/avatic/lcc/service/bulk/BulkOperationExecutionService.java @@ -37,27 +37,25 @@ public class BulkOperationExecutionService { @Async("bulkProcessingExecutor") public CompletableFuture launchExecution(Integer id) { - return CompletableFuture.runAsync(() -> { - execution(id); - }, bulkProcessingExecutor) - .orTimeout(30, TimeUnit.MINUTES) - .exceptionally(e -> { - bulkOperationRepository.updateState(id, BulkOperationState.EXCEPTION); + try { + execution(id); + return CompletableFuture.completedFuture(null); + } catch (Exception e) { + bulkOperationRepository.updateState(id, BulkOperationState.EXCEPTION); + var error = new SysError(); + error.setType(SysErrorType.BULK); + error.setCode(e.getClass().getSimpleName()); + error.setTitle("Bulk operation execution id" + id + " failed"); + error.setMessage(e.getMessage() == null ? "" : e.getMessage()); + error.setUserId(null); + error.setBulkOperationId(id); + error.setTrace(Arrays.stream(e.getStackTrace()).map(sysErrorTransformer::toSysErrorTraceItem).toList()); - var error = new SysError(); - error.setType(SysErrorType.BULK); - error.setCode(e.getClass().getSimpleName()); - error.setTitle("Bulk operation execution id" + id + " failed with timeout"); - error.setMessage(e.getMessage() == null ? "" : e.getMessage()); - error.setUserId(null); - error.setBulkOperationId(id); - error.setTrace(Arrays.stream(e.getStackTrace()).map(sysErrorTransformer::toSysErrorTraceItem).toList()); + sysErrorRepository.insert(error); - sysErrorRepository.insert(error); - - return null; - }); + return CompletableFuture.failedFuture(e); + } } public void execution(Integer id) { diff --git a/src/main/java/de/avatic/lcc/service/excelMapper/MaterialFastExcelMapper.java b/src/main/java/de/avatic/lcc/service/excelMapper/MaterialFastExcelMapper.java new file mode 100644 index 0000000..fbf1440 --- /dev/null +++ b/src/main/java/de/avatic/lcc/service/excelMapper/MaterialFastExcelMapper.java @@ -0,0 +1,254 @@ +package de.avatic.lcc.service.excelMapper; + +import de.avatic.lcc.model.bulk.BulkInstruction; +import de.avatic.lcc.model.bulk.BulkInstructionType; +import de.avatic.lcc.model.bulk.header.MaterialHeader; +import de.avatic.lcc.model.db.materials.Material; +import de.avatic.lcc.repositories.MaterialRepository; +import de.avatic.lcc.util.exception.internalerror.ExcelValidationError; +import org.dhatim.fastexcel.Workbook; +import org.dhatim.fastexcel.Worksheet; +import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.Row; +import org.dhatim.fastexcel.reader.Sheet; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class MaterialFastExcelMapper { + + private final MaterialRepository materialRepository; + + public MaterialFastExcelMapper(MaterialRepository materialRepository) { + this.materialRepository = materialRepository; + } + + /** + * Exports materials to Excel using FastExcel + */ + public byte[] exportToExcel() throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + try (Workbook workbook = new Workbook(outputStream, "LCC", "1.0")) { + Worksheet worksheet = workbook.newWorksheet("Materials"); + + // Create header row + createHeaderRow(worksheet); + + // Fill data rows + List materials = materialRepository.listAllMaterials(); + int rowIndex = 1; + for (Material material : materials) { + mapToRow(material, worksheet, rowIndex); + rowIndex++; + } + + // Auto-size columns + for (int i = 0; i < MaterialHeader.values().length; i++) { + worksheet.width(i, 20); + } + + workbook.finish(); + } + + return outputStream.toByteArray(); + } + + /** + * Creates the header row with proper formatting matching Apache POI style + */ + private void createHeaderRow(Worksheet worksheet) { + int colIndex = 0; + for (MaterialHeader header : MaterialHeader.values()) { + worksheet.value(0, colIndex, header.name()); + + // Apply styling to match HeaderCellStyleProvider + worksheet.style(0, colIndex) + .bold() + .fontName("Arial") + .fontSize(10) + .fontColor("002F54") // RGB(0, 47, 84) - Blue text + .fillColor("5AF0B4") // RGB(90, 240, 180) - Green background + .horizontalAlignment("center") + .borderStyle("thin") // All borders thin + .set(); + + colIndex++; + } + } + + /** + * Maps a Material entity to a row in the worksheet + */ + private void mapToRow(Material material, Worksheet worksheet, int rowIndex) { + worksheet.value(rowIndex, MaterialHeader.OPERATION.ordinal(), BulkInstructionType.UPDATE.name()); + worksheet.value(rowIndex, MaterialHeader.PART_NUMBER.ordinal(), material.getPartNumber()); + worksheet.value(rowIndex, MaterialHeader.DESCRIPTION.ordinal(), material.getName()); + worksheet.value(rowIndex, MaterialHeader.HS_CODE.ordinal(), material.getHsCode()); + } + + /** + * Imports materials from Excel using FastExcel + */ + public List> importFromExcel(byte[] fileData) throws IOException { + List> materials = new ArrayList<>(); + + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(fileData); + ReadableWorkbook workbook = new ReadableWorkbook(inputStream)) { + + Sheet sheet = workbook.getFirstSheet(); + + // Validate header + validateHeader(sheet); + + // Process data rows (skip header row at index 0) + List rows = sheet.read(); + for (int i = 1; i < rows.size(); i++) { + Row row = rows.get(i); + if (!isEmpty(row)) { + materials.add(mapToEntity(row, i + 1)); + } + } + } + + return materials; + } + + /** + * Validates that the Excel file has the correct header structure + */ + private void validateHeader(Sheet sheet) throws IOException { + List rows = sheet.read(); + if (rows.isEmpty()) { + throw new ExcelValidationError("Excel file is empty"); + } + + Row headerRow = rows.get(0); + List expectedHeaders = Arrays.stream(MaterialHeader.values()) + .map(Enum::name) + .collect(Collectors.toList()); + + for (int i = 0; i < expectedHeaders.size(); i++) { + String cellValue = headerRow.getCellAsString(i).orElse(""); + if (!expectedHeaders.get(i).equals(cellValue)) { + throw new ExcelValidationError( + String.format("Invalid header at column %d. Expected '%s' but found '%s'", + i, expectedHeaders.get(i), cellValue) + ); + } + } + } + + /** + * Checks if a row is empty + */ + private boolean isEmpty(Row row) { + for (int i = 0; i < MaterialHeader.values().length; i++) { + if (row.getCellAsString(i).isPresent() && !row.getCellAsString(i).get().trim().isEmpty()) { + return false; + } + } + return true; + } + + /** + * Maps a row from Excel to a BulkInstruction + */ + private BulkInstruction mapToEntity(Row row, int rowNumber) { + Material entity = new Material(); + + try { + // Extract and validate data + String partNumber = getCellValue(row, MaterialHeader.PART_NUMBER.ordinal(), rowNumber); + String description = getCellValue(row, MaterialHeader.DESCRIPTION.ordinal(), rowNumber); + String hsCode = getCellValue(row, MaterialHeader.HS_CODE.ordinal(), rowNumber); + String operation = getCellValue(row, MaterialHeader.OPERATION.ordinal(), rowNumber); + + // Validate lengths + validateLength(partNumber, 0, 12, "Part Number", rowNumber); + validateLength(hsCode, 0, 11, "HS Code", rowNumber); + validateLength(description, 1, 500, "Description", rowNumber); + + // Validate operation enum + BulkInstructionType instructionType; + try { + instructionType = BulkInstructionType.valueOf(operation); + } catch (IllegalArgumentException e) { + throw new ExcelValidationError( + String.format("Invalid operation '%s' at row %d. Must be one of: %s", + operation, rowNumber, Arrays.toString(BulkInstructionType.values())) + ); + } + + // Set entity properties + entity.setPartNumber(partNumber); + entity.setName(description); + entity.setHsCode(hsCode); + entity.setNormalizedPartNumber(normalizePartNumber(partNumber)); + entity.setDeprecated(false); + + // Validate HS Code + if (!validateHsCode(entity.getHsCode())) { + throw new ExcelValidationError( + String.format("Invalid HS Code '%s' at row %d", hsCode, rowNumber) + ); + } + + return new BulkInstruction<>(entity, instructionType); + + } catch (ExcelValidationError e) { + throw e; + } catch (Exception e) { + throw new ExcelValidationError( + String.format("Error processing row %d: %s", rowNumber, e.getMessage()) + ); + } + } + + /** + * Gets a cell value as string with proper error handling + */ + private String getCellValue(Row row, int columnIndex, int rowNumber) { + return row.getCellAsString(columnIndex) + .orElseThrow(() -> new ExcelValidationError( + String.format("Missing value at row %d, column %d", rowNumber, columnIndex) + )); + } + + /** + * Validates the length of a field + */ + private void validateLength(String value, int minLength, int maxLength, String fieldName, int rowNumber) { + if (value.length() < minLength || value.length() > maxLength) { + throw new ExcelValidationError( + String.format("%s at row %d must be between %d and %d characters (found %d)", + fieldName, rowNumber, minLength, maxLength, value.length()) + ); + } + } + + /** + * Normalizes part number by padding with zeros + */ + private String normalizePartNumber(String partNumber) { + if (partNumber.length() > 12) { + throw new IllegalArgumentException("Part number must be less than 12 characters"); + } + return "000000000000".concat(partNumber).substring(partNumber.length()); + } + + /** + * Validates HS Code (placeholder for API validation) + */ + private boolean validateHsCode(String hsCode) { + //TODO check via api?! + return true; + } +} \ No newline at end of file diff --git a/src/main/java/de/avatic/lcc/service/transformer/nodes/NodeUpdateDTOTransformer.java b/src/main/java/de/avatic/lcc/service/transformer/nodes/NodeUpdateDTOTransformer.java index b2ac5c7..cdb748f 100644 --- a/src/main/java/de/avatic/lcc/service/transformer/nodes/NodeUpdateDTOTransformer.java +++ b/src/main/java/de/avatic/lcc/service/transformer/nodes/NodeUpdateDTOTransformer.java @@ -2,7 +2,6 @@ package de.avatic.lcc.service.transformer.nodes; import de.avatic.lcc.dto.configuration.nodes.update.NodeUpdateDTO; import de.avatic.lcc.model.db.nodes.Node; -import org.apache.commons.lang3.NotImplementedException; import org.springframework.stereotype.Service; @Service @@ -12,7 +11,7 @@ public class NodeUpdateDTOTransformer { public Node fromNodeUpdateDTO(NodeUpdateDTO dto) { - throw new NotImplementedException("Not yet implemented fromNodeUpdateDTO"); + throw new IllegalStateException("Not yet implemented fromNodeUpdateDTO"); }