diff --git a/financer-common/src/main/java/de/financer/dto/TransactionUploadFormat.java b/financer-common/src/main/java/de/financer/dto/TransactionUploadFormat.java index 8a52516..5bc2567 100644 --- a/financer-common/src/main/java/de/financer/dto/TransactionUploadFormat.java +++ b/financer-common/src/main/java/de/financer/dto/TransactionUploadFormat.java @@ -1,5 +1,6 @@ package de.financer.dto; public enum TransactionUploadFormat { - MT940_CSV; + MT940_CSV, + VB_CSV } diff --git a/financer-web-client/src/main/java/de/financer/config/FinancerConfig.java b/financer-web-client/src/main/java/de/financer/config/FinancerConfig.java index 24757a3..39876d7 100644 --- a/financer-web-client/src/main/java/de/financer/config/FinancerConfig.java +++ b/financer-web-client/src/main/java/de/financer/config/FinancerConfig.java @@ -98,6 +98,9 @@ public class FinancerConfig { public static class Format { private String dateFormat; private Locale locale; + private Integer skipLinesHead; + private Integer skipLinesTail; + private boolean removeEmptyLines; public String getDateFormat() { return dateFormat; @@ -114,6 +117,30 @@ public class FinancerConfig { public void setLocale(Locale locale) { this.locale = locale; } + + public Integer getSkipLinesHead() { + return skipLinesHead; + } + + public void setSkipLinesHead(Integer skipLinesHead) { + this.skipLinesHead = skipLinesHead; + } + + public Integer getSkipLinesTail() { + return skipLinesTail; + } + + public void setSkipLinesTail(Integer skipLinesTail) { + this.skipLinesTail = skipLinesTail; + } + + public boolean isRemoveEmptyLines() { + return removeEmptyLines; + } + + public void setRemoveEmptyLines(boolean removeEmptyLines) { + this.removeEmptyLines = removeEmptyLines; + } } } } diff --git a/financer-web-client/src/main/java/de/financer/config/FormatConverter.java b/financer-web-client/src/main/java/de/financer/config/FormatConverter.java index a0cf0df..3bcb2fe 100644 --- a/financer-web-client/src/main/java/de/financer/config/FormatConverter.java +++ b/financer-web-client/src/main/java/de/financer/config/FormatConverter.java @@ -13,7 +13,7 @@ public class FormatConverter implements Converter { - final Pair accountAndDescription = - this.getAccountAndDescription(buildDescription(r)); + final List records = parser.getRecords(); + final long limit = records.size() - (this.getFormatConfig().getSkipLinesHead() + + this.getFormatConfig().getSkipLinesTail()); - return new UploadedTransaction( - // Amount - Math.abs(formatAmount(removeQuotes(r.get(AMOUNT_INDEX)))), - // Description - accountAndDescription.getRight(), - // Date - formatDate(LocalDate.parse(removeQuotes(r.get(DATE_INDEX)), - DateTimeFormatter.ofPattern(this.getFormatConfig() - .getDateFormat()))), - // To account key - Optional.ofNullable(accountAndDescription.getLeft()) - .map(Account::getKey).orElse(null)); - } - ) - .collect(Collectors.toList())); + retVal.addAll(records.stream() + .skip(this.getFormatConfig().getSkipLinesHead()) + .limit(limit) + .map(r -> { + final Pair accountAndDescription = + this.getAccountAndDescription(buildDescription(r)); + + return new UploadedTransaction( + // Amount + formatAmount(r.get(AMOUNT_INDEX)), + // Description + accountAndDescription.getRight(), + // Date + formatDate(r.get(DATE_INDEX)), + // To account key + Optional.ofNullable(accountAndDescription.getLeft()) + .map(Account::getKey).orElse(null)); + } + ) + .collect(Collectors.toList())); } catch (IOException | RuntimeException e) { if (e instanceof RuntimeException) { if (e.getCause() instanceof ParseException) { @@ -80,11 +85,6 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo return retVal; } - private String formatDate(LocalDate date) { - // Format to common format - return date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); - } - private String buildDescription(CSVRecord r) { final String description1 = r.get(DESCRIPTION1_INDEX); final String description2 = r.get(DESCRIPTION2_INDEX); @@ -95,15 +95,4 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo return description2; } - - private Long formatAmount(String amountString) { - try { - return NumberFormat.getNumberInstance(this.getFormatConfig().getLocale()) - .parse(amountString) - .longValue(); - } catch (ParseException e) { - // Make it unchecked because of the usage in the lambda pipeline - throw new RuntimeException(e); - } - } } diff --git a/financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java b/financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java index ad5ea11..a8b0b29 100644 --- a/financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java +++ b/financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java @@ -18,6 +18,10 @@ public class TransactionUploadWorkerFactory { case MT940_CSV: worker = new MT940CSVTransactionUploadWorker(this.financerConfig, accounts); + break; + case VB_CSV: + worker = new VBCSVTransactionUploadWorker(this.financerConfig, accounts); + break; default: worker = null; diff --git a/financer-web-client/src/main/java/de/financer/transactionUpload/VBCSVTransactionUploadWorker.java b/financer-web-client/src/main/java/de/financer/transactionUpload/VBCSVTransactionUploadWorker.java new file mode 100644 index 0000000..d4d6f86 --- /dev/null +++ b/financer-web-client/src/main/java/de/financer/transactionUpload/VBCSVTransactionUploadWorker.java @@ -0,0 +1,91 @@ +package de.financer.transactionUpload; + +import de.financer.config.FinancerConfig; +import de.financer.dto.TransactionUploadFormat; +import de.financer.model.Account; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.web.multipart.MultipartFile; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +public class VBCSVTransactionUploadWorker extends AbstractTransactionUploadWorker { + private static final int DATE_INDEX = 1; + private static final int DESCRIPTION1_INDEX = 8; + private static final int AMOUNT_INDEX = 11; + + protected VBCSVTransactionUploadWorker(FinancerConfig financerConfig, Iterable accounts) { + super(TransactionUploadFormat.VB_CSV, financerConfig, accounts); + } + + @Override + public Collection process(MultipartFile file) throws Exception { + final Collection retVal = new ArrayList<>(); + + try (InputStream is = file.getInputStream()) { + final CSVParser parser = CSVFormat.DEFAULT.builder() + .setDelimiter(";") + .setIgnoreEmptyLines(this.getFormatConfig().isRemoveEmptyLines()) + .build() + .parse(new InputStreamReader(is)); + + final List records = parser.getRecords(); + final long limit = records.size() - (this.getFormatConfig().getSkipLinesHead() + + this.getFormatConfig().getSkipLinesTail()); + + retVal.addAll(records.stream() + .skip(this.getFormatConfig().getSkipLinesHead()) + .limit(limit) + .map(r -> { + final Pair accountAndDescription = + this.getAccountAndDescription(buildDescription(r)); + + return new UploadedTransaction( + // Amount + formatAmount(r.get(AMOUNT_INDEX)), + // Description + accountAndDescription.getRight(), + // Date + formatDate(r.get(DATE_INDEX)), + // To account key + Optional.ofNullable(accountAndDescription.getLeft()) + .map(Account::getKey).orElse(null)); + } + ) + .collect(Collectors.toList())); + } catch (IOException | RuntimeException e) { + if (e instanceof RuntimeException) { + if (e.getCause() instanceof ParseException) { + // Make it checked again + throw (ParseException) e.getCause(); + } + } + + throw e; + } + + return retVal; + } + + private String buildDescription(CSVRecord r) { + final String description1 = r.get(DESCRIPTION1_INDEX); + + if (StringUtils.isEmpty(description1)) { + return ""; + } + + // \R is shorthand for line break chars + return description1.replaceAll("\\R", " "); + } +} diff --git a/financer-web-client/src/main/resources/config/application.properties b/financer-web-client/src/main/resources/config/application.properties index 84873e7..b984bbb 100644 --- a/financer-web-client/src/main/resources/config/application.properties +++ b/financer-web-client/src/main/resources/config/application.properties @@ -62,4 +62,8 @@ financer.darkMode=true # The list-style property has to be compliant to the format # 1 -> date format # 2 -> locale -financer.transactionUpload.formats.MT940_CSV=dd.MM.yy,de_DE \ No newline at end of file +# 3 -> skip number of lines HEAD +# 4 -> skip number of lines TAIL +# 5 -> remove empty lines +financer.transactionUpload.formats.MT940_CSV=dd.MM.yy,de_DE,1,0,false +financer.transactionUpload.formats.VB_CSV=dd.MM.yyyy,de_DE,9,2,true \ No newline at end of file diff --git a/financer-web-client/src/main/resources/i18n/message.properties b/financer-web-client/src/main/resources/i18n/message.properties index 4802346..76c4b71 100644 --- a/financer-web-client/src/main/resources/i18n/message.properties +++ b/financer-web-client/src/main/resources/i18n/message.properties @@ -175,13 +175,15 @@ financer.search-transactions.show-query-options.between=Filtering of values (str financer.upload-transactions.title=financer\: upload transactions financer.upload-transactions.label.format=Format\: -financer.upload-transactions.format.MT940_CSV=MT940 CSV financer.upload-transactions.label.file=File\: financer.upload-transactions.submit=Upload transactions financer.upload-transactions.label.new-period-on-recurring-transaction-summary=New period on encountering a certain recurring transaction financer.upload-transactions.label.new-period-on-recurring-transaction=Recurring transaction\: financer.upload-transactions.label.new-period-on-recurring-transaction-enable=Enable\: +financer.upload-transactions.format.MT940_CSV=MT940 CSV +financer.upload-transactions.format.VB_CSV=Volksbank CSV + financer.create-upload-transactions.title=financer\: create uploaded transactions financer.create-upload-transactions.table-header.create=Create financer.create-upload-transactions.table-header.fromAccount=From account diff --git a/financer-web-client/src/main/resources/i18n/message_de_DE.properties b/financer-web-client/src/main/resources/i18n/message_de_DE.properties index 5a2c230..847a42e 100644 --- a/financer-web-client/src/main/resources/i18n/message_de_DE.properties +++ b/financer-web-client/src/main/resources/i18n/message_de_DE.properties @@ -175,13 +175,15 @@ financer.search-transactions.show-query-options.between=Filtern von Werten (Zeic financer.upload-transactions.title=financer\: Buchungen hochladen financer.upload-transactions.label.format=Format\: -financer.upload-transactions.format.MT940_CSV=MT940 CSV financer.upload-transactions.label.file=Datei\: financer.upload-transactions.submit=Buchungen hochladen financer.upload-transactions.label.new-period-on-recurring-transaction-summary=Neue Periode bei wiederkehrender Buchung financer.upload-transactions.label.new-period-on-recurring-transaction=Wiederkehrende Buchung\: financer.upload-transactions.label.new-period-on-recurring-transaction-enable=Aktiv\: +financer.upload-transactions.format.MT940_CSV=MT940 CSV +financer.upload-transactions.format.VB_CSV=Volksbank CSV + financer.create-upload-transactions.title=financer\: Erstelle hochgeladene Buchungen financer.create-upload-transactions.table-header.create=Erstellen financer.create-upload-transactions.table-header.fromAccount=Von Konto diff --git a/financer-web-client/src/main/resources/static/changelog.txt b/financer-web-client/src/main/resources/static/changelog.txt index 57b7a54..c6809d6 100644 --- a/financer-web-client/src/main/resources/static/changelog.txt +++ b/financer-web-client/src/main/resources/static/changelog.txt @@ -8,6 +8,7 @@ v48 -> v49: - Added a new column to the transaction upload creation form that is a consecutive number - #24 Added the possibility to add regular expressions to accounts so that transactions uploaded can be automatically matched to accounts +- #25 Added Volksbank CSV as transaction upload format v47 -> v48: - #20 Added new property 'transaction type' to a transaction, denoting the type of the transaction, e.g. asset swap,