Replaced MaterialExcelMapper with MaterialFastExcelMapper using fastexcel instead of Apache POI.
This commit is contained in:
parent
a009c35b52
commit
3e9e9e1d34
8 changed files with 330 additions and 34 deletions
10
pom.xml
10
pom.xml
|
|
@ -114,6 +114,16 @@
|
|||
<artifactId>poi</artifactId>
|
||||
<version>5.4.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dhatim</groupId>
|
||||
<artifactId>fastexcel</artifactId>
|
||||
<version>0.17.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.dhatim</groupId>
|
||||
<artifactId>fastexcel-reader</artifactId>
|
||||
<version>0.17.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.poi</groupId>
|
||||
<artifactId>poi-ooxml</artifactId>
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ export default {
|
|||
display: grid;
|
||||
grid-template-columns: 1fr 2fr 0.5fr;
|
||||
grid-gap: 2.4rem;
|
||||
padding: 1.6rem 0;
|
||||
padding: 1.2rem 0;
|
||||
font-weight: 400;
|
||||
font-size: 1.4rem;
|
||||
border-bottom: 0.1rem solid #E3EDFF;
|
||||
|
|
|
|||
|
|
@ -109,9 +109,9 @@ export default {
|
|||
await navigator.clipboard.writeText(JSON.stringify(this.error));
|
||||
|
||||
this.$refs.toast.addToast({
|
||||
icon: 'Bug',
|
||||
icon: 'Clipboard',
|
||||
message: "The error has been copied to clipboard",
|
||||
title: "Copied to clipboard",
|
||||
title: "Successful copied to clipboard",
|
||||
variant: 'success',
|
||||
duration: 4000
|
||||
})
|
||||
|
|
|
|||
|
|
@ -30,8 +30,9 @@ public class BulkExportService {
|
|||
private final HiddenNodeExcelMapper hiddenNodeExcelMapper;
|
||||
private final HiddenCountryExcelMapper hiddenCountryExcelMapper;
|
||||
private final String sheetPassword;
|
||||
private final MaterialFastExcelMapper materialFastExcelMapper;
|
||||
|
||||
public BulkExportService(@Value("${lcc.bulk.sheet_password}") String sheetPassword, HeaderCellStyleProvider headerCellStyleProvider, ContainerRateExcelMapper containerRateExcelMapper, MatrixRateExcelMapper matrixRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, HiddenNodeExcelMapper hiddenNodeExcelMapper, HiddenCountryExcelMapper hiddenCountryExcelMapper) {
|
||||
public BulkExportService(@Value("${lcc.bulk.sheet_password}") String sheetPassword, HeaderCellStyleProvider headerCellStyleProvider, ContainerRateExcelMapper containerRateExcelMapper, MatrixRateExcelMapper matrixRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, HiddenNodeExcelMapper hiddenNodeExcelMapper, HiddenCountryExcelMapper hiddenCountryExcelMapper, MaterialFastExcelMapper materialFastExcelMapper) {
|
||||
this.headerCellStyleProvider = headerCellStyleProvider;
|
||||
this.containerRateExcelMapper = containerRateExcelMapper;
|
||||
this.matrixRateExcelMapper = matrixRateExcelMapper;
|
||||
|
|
@ -41,10 +42,24 @@ public class BulkExportService {
|
|||
this.hiddenNodeExcelMapper = hiddenNodeExcelMapper;
|
||||
this.hiddenCountryExcelMapper = hiddenCountryExcelMapper;
|
||||
this.sheetPassword = sheetPassword;
|
||||
this.materialFastExcelMapper = materialFastExcelMapper;
|
||||
}
|
||||
|
||||
public void processOperation(BulkOperation op) throws IOException {
|
||||
var bulkFileType = op.getFileType();
|
||||
|
||||
// Use FastExcel for MATERIAL, Apache POI for others
|
||||
if (bulkFileType.equals(BulkFileType.MATERIAL)) {
|
||||
byte[] excelData = materialFastExcelMapper.exportToExcel();
|
||||
op.setFile(excelData);
|
||||
} else {
|
||||
processOperationWithApachePOI(op);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void processOperation(BulkOperation op) throws IOException {
|
||||
|
||||
public void processOperationWithApachePOI(BulkOperation op) throws IOException {
|
||||
|
||||
var bulkFileType = op.getFileType();
|
||||
var periodId = op.getValidityPeriodId();
|
||||
|
|
@ -78,10 +93,10 @@ public class BulkExportService {
|
|||
matrixRateExcelMapper.fillSheet(worksheet, style, periodId);
|
||||
matrixRateExcelMapper.createConstraints(workbook, worksheet);
|
||||
break;
|
||||
case MATERIAL:
|
||||
materialExcelMapper.fillSheet(worksheet, style);
|
||||
materialExcelMapper.createConstraints(worksheet);
|
||||
break;
|
||||
// case MATERIAL:
|
||||
// materialExcelMapper.fillSheet(worksheet, style);
|
||||
// materialExcelMapper.createConstraints(worksheet);
|
||||
// break;
|
||||
case PACKAGING:
|
||||
packagingExcelMapper.fillSheet(worksheet, style);
|
||||
packagingExcelMapper.createConstraints(workbook, worksheet);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
package de.avatic.lcc.service.bulk;
|
||||
|
||||
import de.avatic.lcc.dto.bulk.BulkFileType;
|
||||
import de.avatic.lcc.model.bulk.BulkFileTypes;
|
||||
import de.avatic.lcc.model.bulk.BulkOperation;
|
||||
import de.avatic.lcc.service.api.BatchGeoApiService;
|
||||
|
|
@ -31,8 +32,9 @@ public class BulkImportService {
|
|||
private final MatrixRateImportService matrixRateImportService;
|
||||
private final ContainerRateImportService containerRateImportService;
|
||||
private final BatchGeoApiService batchGeoApiService;
|
||||
private final MaterialFastExcelMapper materialFastExcelMapper;
|
||||
|
||||
public BulkImportService(MatrixRateExcelMapper matrixRateExcelMapper, ContainerRateExcelMapper containerRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, NodeBulkImportService nodeBulkImportService, PackagingBulkImportService packagingBulkImportService, MaterialBulkImportService materialBulkImportService, MatrixRateImportService matrixRateImportService, ContainerRateImportService containerRateImportService, BatchGeoApiService batchGeoApiService) {
|
||||
public BulkImportService(MatrixRateExcelMapper matrixRateExcelMapper, ContainerRateExcelMapper containerRateExcelMapper, MaterialExcelMapper materialExcelMapper, PackagingExcelMapper packagingExcelMapper, NodeExcelMapper nodeExcelMapper, NodeBulkImportService nodeBulkImportService, PackagingBulkImportService packagingBulkImportService, MaterialBulkImportService materialBulkImportService, MatrixRateImportService matrixRateImportService, ContainerRateImportService containerRateImportService, BatchGeoApiService batchGeoApiService, MaterialFastExcelMapper materialFastExcelMapper) {
|
||||
this.matrixRateExcelMapper = matrixRateExcelMapper;
|
||||
this.containerRateExcelMapper = containerRateExcelMapper;
|
||||
this.materialExcelMapper = materialExcelMapper;
|
||||
|
|
@ -44,9 +46,27 @@ public class BulkImportService {
|
|||
this.matrixRateImportService = matrixRateImportService;
|
||||
this.containerRateImportService = containerRateImportService;
|
||||
this.batchGeoApiService = batchGeoApiService;
|
||||
this.materialFastExcelMapper = materialFastExcelMapper;
|
||||
}
|
||||
|
||||
public void processOperation(BulkOperation op) throws IOException {
|
||||
var file = op.getFile();
|
||||
var type = op.getFileType();
|
||||
|
||||
// Use FastExcel for MATERIAL, Apache POI for others
|
||||
if (type.equals(BulkFileType.MATERIAL)) {
|
||||
processOperationWithFastExcel(op);
|
||||
} else {
|
||||
processOperationWithApachePOI(op);
|
||||
}
|
||||
}
|
||||
|
||||
private void processOperationWithFastExcel(BulkOperation op) throws IOException {
|
||||
var materials = materialFastExcelMapper.importFromExcel(op.getFile());
|
||||
materials.forEach(materialBulkImportService::processMaterialInstructions);
|
||||
}
|
||||
|
||||
private void processOperationWithApachePOI(BulkOperation op) throws IOException {
|
||||
|
||||
var file = op.getFile();
|
||||
var type = op.getFileType();
|
||||
|
|
@ -76,10 +96,10 @@ public class BulkImportService {
|
|||
var matrixRates = matrixRateExcelMapper.extractSheet(sheet);
|
||||
matrixRateImportService.processMatrixRates(matrixRates);
|
||||
break;
|
||||
case MATERIAL:
|
||||
var materials = materialExcelMapper.extractSheet(sheet);
|
||||
materials.forEach(materialBulkImportService::processMaterialInstructions);
|
||||
break;
|
||||
// case MATERIAL:
|
||||
// var materials = materialExcelMapper.extractSheet(sheet);
|
||||
// materials.forEach(materialBulkImportService::processMaterialInstructions);
|
||||
// break;
|
||||
case PACKAGING:
|
||||
var packaging = packagingExcelMapper.extractSheet(sheet);
|
||||
packaging.forEach(packagingBulkImportService::processPackagingInstructions);
|
||||
|
|
|
|||
|
|
@ -37,18 +37,16 @@ public class BulkOperationExecutionService {
|
|||
|
||||
@Async("bulkProcessingExecutor")
|
||||
public CompletableFuture<Void> launchExecution(Integer id) {
|
||||
return CompletableFuture.runAsync(() -> {
|
||||
try {
|
||||
execution(id);
|
||||
}, bulkProcessingExecutor)
|
||||
.orTimeout(30, TimeUnit.MINUTES)
|
||||
.exceptionally(e -> {
|
||||
return CompletableFuture.completedFuture(null);
|
||||
} catch (Exception e) {
|
||||
bulkOperationRepository.updateState(id, BulkOperationState.EXCEPTION);
|
||||
|
||||
|
||||
var error = new SysError();
|
||||
error.setType(SysErrorType.BULK);
|
||||
error.setCode(e.getClass().getSimpleName());
|
||||
error.setTitle("Bulk operation execution id" + id + " failed with timeout");
|
||||
error.setTitle("Bulk operation execution id" + id + " failed");
|
||||
error.setMessage(e.getMessage() == null ? "" : e.getMessage());
|
||||
error.setUserId(null);
|
||||
error.setBulkOperationId(id);
|
||||
|
|
@ -56,8 +54,8 @@ public class BulkOperationExecutionService {
|
|||
|
||||
sysErrorRepository.insert(error);
|
||||
|
||||
return null;
|
||||
});
|
||||
return CompletableFuture.failedFuture(e);
|
||||
}
|
||||
}
|
||||
|
||||
public void execution(Integer id) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,254 @@
|
|||
package de.avatic.lcc.service.excelMapper;
|
||||
|
||||
import de.avatic.lcc.model.bulk.BulkInstruction;
|
||||
import de.avatic.lcc.model.bulk.BulkInstructionType;
|
||||
import de.avatic.lcc.model.bulk.header.MaterialHeader;
|
||||
import de.avatic.lcc.model.db.materials.Material;
|
||||
import de.avatic.lcc.repositories.MaterialRepository;
|
||||
import de.avatic.lcc.util.exception.internalerror.ExcelValidationError;
|
||||
import org.dhatim.fastexcel.Workbook;
|
||||
import org.dhatim.fastexcel.Worksheet;
|
||||
import org.dhatim.fastexcel.reader.ReadableWorkbook;
|
||||
import org.dhatim.fastexcel.reader.Row;
|
||||
import org.dhatim.fastexcel.reader.Sheet;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class MaterialFastExcelMapper {
|
||||
|
||||
private final MaterialRepository materialRepository;
|
||||
|
||||
public MaterialFastExcelMapper(MaterialRepository materialRepository) {
|
||||
this.materialRepository = materialRepository;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exports materials to Excel using FastExcel
|
||||
*/
|
||||
public byte[] exportToExcel() throws IOException {
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
|
||||
try (Workbook workbook = new Workbook(outputStream, "LCC", "1.0")) {
|
||||
Worksheet worksheet = workbook.newWorksheet("Materials");
|
||||
|
||||
// Create header row
|
||||
createHeaderRow(worksheet);
|
||||
|
||||
// Fill data rows
|
||||
List<Material> materials = materialRepository.listAllMaterials();
|
||||
int rowIndex = 1;
|
||||
for (Material material : materials) {
|
||||
mapToRow(material, worksheet, rowIndex);
|
||||
rowIndex++;
|
||||
}
|
||||
|
||||
// Auto-size columns
|
||||
for (int i = 0; i < MaterialHeader.values().length; i++) {
|
||||
worksheet.width(i, 20);
|
||||
}
|
||||
|
||||
workbook.finish();
|
||||
}
|
||||
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the header row with proper formatting matching Apache POI style
|
||||
*/
|
||||
private void createHeaderRow(Worksheet worksheet) {
|
||||
int colIndex = 0;
|
||||
for (MaterialHeader header : MaterialHeader.values()) {
|
||||
worksheet.value(0, colIndex, header.name());
|
||||
|
||||
// Apply styling to match HeaderCellStyleProvider
|
||||
worksheet.style(0, colIndex)
|
||||
.bold()
|
||||
.fontName("Arial")
|
||||
.fontSize(10)
|
||||
.fontColor("002F54") // RGB(0, 47, 84) - Blue text
|
||||
.fillColor("5AF0B4") // RGB(90, 240, 180) - Green background
|
||||
.horizontalAlignment("center")
|
||||
.borderStyle("thin") // All borders thin
|
||||
.set();
|
||||
|
||||
colIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a Material entity to a row in the worksheet
|
||||
*/
|
||||
private void mapToRow(Material material, Worksheet worksheet, int rowIndex) {
|
||||
worksheet.value(rowIndex, MaterialHeader.OPERATION.ordinal(), BulkInstructionType.UPDATE.name());
|
||||
worksheet.value(rowIndex, MaterialHeader.PART_NUMBER.ordinal(), material.getPartNumber());
|
||||
worksheet.value(rowIndex, MaterialHeader.DESCRIPTION.ordinal(), material.getName());
|
||||
worksheet.value(rowIndex, MaterialHeader.HS_CODE.ordinal(), material.getHsCode());
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports materials from Excel using FastExcel
|
||||
*/
|
||||
public List<BulkInstruction<Material>> importFromExcel(byte[] fileData) throws IOException {
|
||||
List<BulkInstruction<Material>> materials = new ArrayList<>();
|
||||
|
||||
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(fileData);
|
||||
ReadableWorkbook workbook = new ReadableWorkbook(inputStream)) {
|
||||
|
||||
Sheet sheet = workbook.getFirstSheet();
|
||||
|
||||
// Validate header
|
||||
validateHeader(sheet);
|
||||
|
||||
// Process data rows (skip header row at index 0)
|
||||
List<Row> rows = sheet.read();
|
||||
for (int i = 1; i < rows.size(); i++) {
|
||||
Row row = rows.get(i);
|
||||
if (!isEmpty(row)) {
|
||||
materials.add(mapToEntity(row, i + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return materials;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that the Excel file has the correct header structure
|
||||
*/
|
||||
private void validateHeader(Sheet sheet) throws IOException {
|
||||
List<Row> rows = sheet.read();
|
||||
if (rows.isEmpty()) {
|
||||
throw new ExcelValidationError("Excel file is empty");
|
||||
}
|
||||
|
||||
Row headerRow = rows.get(0);
|
||||
List<String> expectedHeaders = Arrays.stream(MaterialHeader.values())
|
||||
.map(Enum::name)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
for (int i = 0; i < expectedHeaders.size(); i++) {
|
||||
String cellValue = headerRow.getCellAsString(i).orElse("");
|
||||
if (!expectedHeaders.get(i).equals(cellValue)) {
|
||||
throw new ExcelValidationError(
|
||||
String.format("Invalid header at column %d. Expected '%s' but found '%s'",
|
||||
i, expectedHeaders.get(i), cellValue)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a row is empty
|
||||
*/
|
||||
private boolean isEmpty(Row row) {
|
||||
for (int i = 0; i < MaterialHeader.values().length; i++) {
|
||||
if (row.getCellAsString(i).isPresent() && !row.getCellAsString(i).get().trim().isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a row from Excel to a BulkInstruction<Material>
|
||||
*/
|
||||
private BulkInstruction<Material> mapToEntity(Row row, int rowNumber) {
|
||||
Material entity = new Material();
|
||||
|
||||
try {
|
||||
// Extract and validate data
|
||||
String partNumber = getCellValue(row, MaterialHeader.PART_NUMBER.ordinal(), rowNumber);
|
||||
String description = getCellValue(row, MaterialHeader.DESCRIPTION.ordinal(), rowNumber);
|
||||
String hsCode = getCellValue(row, MaterialHeader.HS_CODE.ordinal(), rowNumber);
|
||||
String operation = getCellValue(row, MaterialHeader.OPERATION.ordinal(), rowNumber);
|
||||
|
||||
// Validate lengths
|
||||
validateLength(partNumber, 0, 12, "Part Number", rowNumber);
|
||||
validateLength(hsCode, 0, 11, "HS Code", rowNumber);
|
||||
validateLength(description, 1, 500, "Description", rowNumber);
|
||||
|
||||
// Validate operation enum
|
||||
BulkInstructionType instructionType;
|
||||
try {
|
||||
instructionType = BulkInstructionType.valueOf(operation);
|
||||
} catch (IllegalArgumentException e) {
|
||||
throw new ExcelValidationError(
|
||||
String.format("Invalid operation '%s' at row %d. Must be one of: %s",
|
||||
operation, rowNumber, Arrays.toString(BulkInstructionType.values()))
|
||||
);
|
||||
}
|
||||
|
||||
// Set entity properties
|
||||
entity.setPartNumber(partNumber);
|
||||
entity.setName(description);
|
||||
entity.setHsCode(hsCode);
|
||||
entity.setNormalizedPartNumber(normalizePartNumber(partNumber));
|
||||
entity.setDeprecated(false);
|
||||
|
||||
// Validate HS Code
|
||||
if (!validateHsCode(entity.getHsCode())) {
|
||||
throw new ExcelValidationError(
|
||||
String.format("Invalid HS Code '%s' at row %d", hsCode, rowNumber)
|
||||
);
|
||||
}
|
||||
|
||||
return new BulkInstruction<>(entity, instructionType);
|
||||
|
||||
} catch (ExcelValidationError e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
throw new ExcelValidationError(
|
||||
String.format("Error processing row %d: %s", rowNumber, e.getMessage())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a cell value as string with proper error handling
|
||||
*/
|
||||
private String getCellValue(Row row, int columnIndex, int rowNumber) {
|
||||
return row.getCellAsString(columnIndex)
|
||||
.orElseThrow(() -> new ExcelValidationError(
|
||||
String.format("Missing value at row %d, column %d", rowNumber, columnIndex)
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the length of a field
|
||||
*/
|
||||
private void validateLength(String value, int minLength, int maxLength, String fieldName, int rowNumber) {
|
||||
if (value.length() < minLength || value.length() > maxLength) {
|
||||
throw new ExcelValidationError(
|
||||
String.format("%s at row %d must be between %d and %d characters (found %d)",
|
||||
fieldName, rowNumber, minLength, maxLength, value.length())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes part number by padding with zeros
|
||||
*/
|
||||
private String normalizePartNumber(String partNumber) {
|
||||
if (partNumber.length() > 12) {
|
||||
throw new IllegalArgumentException("Part number must be less than 12 characters");
|
||||
}
|
||||
return "000000000000".concat(partNumber).substring(partNumber.length());
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates HS Code (placeholder for API validation)
|
||||
*/
|
||||
private boolean validateHsCode(String hsCode) {
|
||||
//TODO check via api?!
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,6 @@ package de.avatic.lcc.service.transformer.nodes;
|
|||
|
||||
import de.avatic.lcc.dto.configuration.nodes.update.NodeUpdateDTO;
|
||||
import de.avatic.lcc.model.db.nodes.Node;
|
||||
import org.apache.commons.lang3.NotImplementedException;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
@Service
|
||||
|
|
@ -12,7 +11,7 @@ public class NodeUpdateDTOTransformer {
|
|||
|
||||
|
||||
public Node fromNodeUpdateDTO(NodeUpdateDTO dto) {
|
||||
throw new NotImplementedException("Not yet implemented fromNodeUpdateDTO");
|
||||
throw new IllegalStateException("Not yet implemented fromNodeUpdateDTO");
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue