MK13/financer
1
0
Fork 0
Code Issues 10 Actions Activity

#28 Transaction import: Volksbank CSV format

Browse Source
This commit is contained in:
MK13
2021-09-03 12:23:08 +02:00
parent d88e2583b5
commit fa1754d512
11 changed files with 189 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
financer-common/src/main/java/de/financer/dto/TransactionUploadFormat.java
View File

@@ -1,5 +1,6 @@
package de.financer.dto;
public enum TransactionUploadFormat {
MT940_CSV;
MT940_CSV,
VB_CSV
}

27
financer-web-client/src/main/java/de/financer/config/FinancerConfig.java
View File

@@ -98,6 +98,9 @@ public class FinancerConfig {
public static class Format {
private String dateFormat;
private Locale locale;
private Integer skipLinesHead;
private Integer skipLinesTail;
private boolean removeEmptyLines;
public String getDateFormat() {
return dateFormat;
@@ -114,6 +117,30 @@ public class FinancerConfig {
public void setLocale(Locale locale) {
this.locale = locale;
}
public Integer getSkipLinesHead() {
return skipLinesHead;
}
public void setSkipLinesHead(Integer skipLinesHead) {
this.skipLinesHead = skipLinesHead;
}
public Integer getSkipLinesTail() {
return skipLinesTail;
}
public void setSkipLinesTail(Integer skipLinesTail) {
this.skipLinesTail = skipLinesTail;
}
public boolean isRemoveEmptyLines() {
return removeEmptyLines;
}
public void setRemoveEmptyLines(boolean removeEmptyLines) {
this.removeEmptyLines = removeEmptyLines;
}
}
}
}

5
financer-web-client/src/main/java/de/financer/config/FormatConverter.java
View File

@@ -13,7 +13,7 @@ public class FormatConverter implements Converter<String, FinancerConfig.Transac
public FinancerConfig.TransactionUpload.Format convert(String source) {
final String[] data = source.split(",");
if (data.length != 2) {
if (data.length != 5) {
throw new IllegalArgumentException("Wrong format of Format definition!");
}
@@ -21,6 +21,9 @@ public class FormatConverter implements Converter<String, FinancerConfig.Transac
format.setDateFormat(data[0]);
format.setLocale(new Locale(data[1]));
format.setSkipLinesHead(Integer.valueOf(data[2]));
format.setSkipLinesTail(Integer.valueOf(data[3]));
format.setRemoveEmptyLines(Boolean.parseBoolean(data[4]));
return format;
}

26
financer-web-client/src/main/java/de/financer/transactionUpload/AbstractTransactionUploadWorker.java
View File

@@ -5,8 +5,13 @@ import de.financer.config.FinancerConfig;
import de.financer.dto.TransactionUploadFormat;
import de.financer.model.Account;
import org.apache.commons.collections4.IterableUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import java.text.NumberFormat;
import java.text.ParseException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
@@ -27,10 +32,6 @@ public abstract class AbstractTransactionUploadWorker implements TransactionUplo
return this.financerConfig.getTransactionUpload().getFormats().get(this.format);
}
protected String removeQuotes(String value) {
return CharMatcher.is('\"').trimFrom(value);
}
public FinancerConfig getFinancerConfig() {
return financerConfig;
}
@@ -52,4 +53,21 @@ public abstract class AbstractTransactionUploadWorker implements TransactionUplo
return Pair.of(null, rawDescription);
}
protected Long formatAmount(String amountString) {
try {
return Math.abs(NumberFormat.getNumberInstance(this.getFormatConfig().getLocale())
.parse(StringUtils.replace(amountString, ".", ""))
.longValue());
} catch (ParseException e) {
// Make it unchecked because of the usage in the lambda pipeline
throw new RuntimeException(e);
}
}
protected String formatDate(String rawDate) {
// Format to common format
return LocalDate.parse(rawDate, DateTimeFormatter.ofPattern(this.getFormatConfig().getDateFormat()))
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
}
}

65
financer-web-client/src/main/java/de/financer/transactionUpload/MT940CSVTransactionUploadWorker.java
View File

@@ -13,14 +13,14 @@ import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.NumberFormat;
import java.text.ParseException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWorker {
/**
@@ -43,29 +43,34 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
final CSVParser parser = CSVFormat.DEFAULT.builder()
.setDelimiter(";")
.setNullString(null)
.setIgnoreEmptyLines(this.getFormatConfig().isRemoveEmptyLines())
.build()
.parse(new InputStreamReader(is));
retVal.addAll(parser.stream().skip(1)
.map(r -> {
final Pair<Account, String> accountAndDescription =
this.getAccountAndDescription(buildDescription(r));
final List<CSVRecord> records = parser.getRecords();
final long limit = records.size() - (this.getFormatConfig().getSkipLinesHead() +
this.getFormatConfig().getSkipLinesTail());
return new UploadedTransaction(
// Amount
Math.abs(formatAmount(removeQuotes(r.get(AMOUNT_INDEX)))),
// Description
accountAndDescription.getRight(),
// Date
formatDate(LocalDate.parse(removeQuotes(r.get(DATE_INDEX)),
DateTimeFormatter.ofPattern(this.getFormatConfig()
.getDateFormat()))),
// To account key
Optional.ofNullable(accountAndDescription.getLeft())
.map(Account::getKey).orElse(null));
}
)
.collect(Collectors.toList()));
retVal.addAll(records.stream()
.skip(this.getFormatConfig().getSkipLinesHead())
.limit(limit)
.map(r -> {
final Pair<Account, String> accountAndDescription =
this.getAccountAndDescription(buildDescription(r));
return new UploadedTransaction(
// Amount
formatAmount(r.get(AMOUNT_INDEX)),
// Description
accountAndDescription.getRight(),
// Date
formatDate(r.get(DATE_INDEX)),
// To account key
Optional.ofNullable(accountAndDescription.getLeft())
.map(Account::getKey).orElse(null));
}
)
.collect(Collectors.toList()));
} catch (IOException | RuntimeException e) {
if (e instanceof RuntimeException) {
if (e.getCause() instanceof ParseException) {
@@ -80,11 +85,6 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
return retVal;
}
private String formatDate(LocalDate date) {
// Format to common format
return date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
}
private String buildDescription(CSVRecord r) {
final String description1 = r.get(DESCRIPTION1_INDEX);
final String description2 = r.get(DESCRIPTION2_INDEX);
@@ -95,15 +95,4 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
return description2;
}
private Long formatAmount(String amountString) {
try {
return NumberFormat.getNumberInstance(this.getFormatConfig().getLocale())
.parse(amountString)
.longValue();
} catch (ParseException e) {
// Make it unchecked because of the usage in the lambda pipeline
throw new RuntimeException(e);
}
}
}

4
financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java
View File

@@ -18,6 +18,10 @@ public class TransactionUploadWorkerFactory {
case MT940_CSV:
worker = new MT940CSVTransactionUploadWorker(this.financerConfig, accounts);
break;
case VB_CSV:
worker = new VBCSVTransactionUploadWorker(this.financerConfig, accounts);
break;
default:
worker = null;

91
financer-web-client/src/main/java/de/financer/transactionUpload/VBCSVTransactionUploadWorker.java Normal file
View File

@@ -0,0 +1,91 @@
package de.financer.transactionUpload;
import de.financer.config.FinancerConfig;
import de.financer.dto.TransactionUploadFormat;
import de.financer.model.Account;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
public class VBCSVTransactionUploadWorker extends AbstractTransactionUploadWorker {
private static final int DATE_INDEX = 1;
private static final int DESCRIPTION1_INDEX = 8;
private static final int AMOUNT_INDEX = 11;
protected VBCSVTransactionUploadWorker(FinancerConfig financerConfig, Iterable<Account> accounts) {
super(TransactionUploadFormat.VB_CSV, financerConfig, accounts);
}
@Override
public Collection<UploadedTransaction> process(MultipartFile file) throws Exception {
final Collection<UploadedTransaction> retVal = new ArrayList<>();
try (InputStream is = file.getInputStream()) {
final CSVParser parser = CSVFormat.DEFAULT.builder()
.setDelimiter(";")
.setIgnoreEmptyLines(this.getFormatConfig().isRemoveEmptyLines())
.build()
.parse(new InputStreamReader(is));
final List<CSVRecord> records = parser.getRecords();
final long limit = records.size() - (this.getFormatConfig().getSkipLinesHead() +
this.getFormatConfig().getSkipLinesTail());
retVal.addAll(records.stream()
.skip(this.getFormatConfig().getSkipLinesHead())
.limit(limit)
.map(r -> {
final Pair<Account, String> accountAndDescription =
this.getAccountAndDescription(buildDescription(r));
return new UploadedTransaction(
// Amount
formatAmount(r.get(AMOUNT_INDEX)),
// Description
accountAndDescription.getRight(),
// Date
formatDate(r.get(DATE_INDEX)),
// To account key
Optional.ofNullable(accountAndDescription.getLeft())
.map(Account::getKey).orElse(null));
}
)
.collect(Collectors.toList()));
} catch (IOException | RuntimeException e) {
if (e instanceof RuntimeException) {
if (e.getCause() instanceof ParseException) {
// Make it checked again
throw (ParseException) e.getCause();
}
}
throw e;
}
return retVal;
}
private String buildDescription(CSVRecord r) {
final String description1 = r.get(DESCRIPTION1_INDEX);
if (StringUtils.isEmpty(description1)) {
return "";
}
// \R is shorthand for line break chars
return description1.replaceAll("\\R", " ");
}
}

6
financer-web-client/src/main/resources/config/application.properties
View File

@@ -62,4 +62,8 @@ financer.darkMode=true
# The list-style property has to be compliant to the format
# 1 -> date format
# 2 -> locale
financer.transactionUpload.formats.MT940_CSV=dd.MM.yy,de_DE
# 3 -> skip number of lines HEAD
# 4 -> skip number of lines TAIL
# 5 -> remove empty lines
financer.transactionUpload.formats.MT940_CSV=dd.MM.yy,de_DE,1,0,false
financer.transactionUpload.formats.VB_CSV=dd.MM.yyyy,de_DE,9,2,true

4
financer-web-client/src/main/resources/i18n/message.properties
View File

@@ -175,13 +175,15 @@ financer.search-transactions.show-query-options.between=Filtering of values (str
financer.upload-transactions.title=financer\: upload transactions
financer.upload-transactions.label.format=Format\:
financer.upload-transactions.format.MT940_CSV=MT940 CSV
financer.upload-transactions.label.file=File\:
financer.upload-transactions.submit=Upload transactions
financer.upload-transactions.label.new-period-on-recurring-transaction-summary=New period on encountering a certain recurring transaction
financer.upload-transactions.label.new-period-on-recurring-transaction=Recurring transaction\:
financer.upload-transactions.label.new-period-on-recurring-transaction-enable=Enable\:
financer.upload-transactions.format.MT940_CSV=MT940 CSV
financer.upload-transactions.format.VB_CSV=Volksbank CSV
financer.create-upload-transactions.title=financer\: create uploaded transactions
financer.create-upload-transactions.table-header.create=Create
financer.create-upload-transactions.table-header.fromAccount=From account

4
financer-web-client/src/main/resources/i18n/message_de_DE.properties
View File

@@ -175,13 +175,15 @@ financer.search-transactions.show-query-options.between=Filtern von Werten (Zeic
financer.upload-transactions.title=financer\: Buchungen hochladen
financer.upload-transactions.label.format=Format\:
financer.upload-transactions.format.MT940_CSV=MT940 CSV
financer.upload-transactions.label.file=Datei\:
financer.upload-transactions.submit=Buchungen hochladen
financer.upload-transactions.label.new-period-on-recurring-transaction-summary=Neue Periode bei wiederkehrender Buchung
financer.upload-transactions.label.new-period-on-recurring-transaction=Wiederkehrende Buchung\:
financer.upload-transactions.label.new-period-on-recurring-transaction-enable=Aktiv\:
financer.upload-transactions.format.MT940_CSV=MT940 CSV
financer.upload-transactions.format.VB_CSV=Volksbank CSV
financer.create-upload-transactions.title=financer\: Erstelle hochgeladene Buchungen
financer.create-upload-transactions.table-header.create=Erstellen
financer.create-upload-transactions.table-header.fromAccount=Von Konto

1
financer-web-client/src/main/resources/static/changelog.txt
View File

@@ -8,6 +8,7 @@ v48 -> v49:
- Added a new column to the transaction upload creation form that is a consecutive number
- #24 Added the possibility to add regular expressions to accounts so that transactions uploaded can be automatically
matched to accounts
- #25 Added Volksbank CSV as transaction upload format
v47 -> v48:
- #20 Added new property 'transaction type' to a transaction, denoting the type of the transaction, e.g. asset swap,