#28 Transaction import: Volksbank CSV format

financer-common/src/main/java/de/financer/dto/TransactionUploadFormat.java

@@ -1,5 +1,6 @@
 package de.financer.dto;
 
 public enum TransactionUploadFormat {
-    MT940_CSV;
+    MT940_CSV,
+    VB_CSV
 }

financer-web-client/src/main/java/de/financer/config/FinancerConfig.java

@@ -98,6 +98,9 @@ public class FinancerConfig {
         public static class Format {
             private String dateFormat;
             private Locale locale;
+            private Integer skipLinesHead;
+            private Integer skipLinesTail;
+            private boolean removeEmptyLines;
 
             public String getDateFormat() {
                 return dateFormat;
@@ -114,6 +117,30 @@ public class FinancerConfig {
             public void setLocale(Locale locale) {
                 this.locale = locale;
             }
+
+            public Integer getSkipLinesHead() {
+                return skipLinesHead;
+            }
+
+            public void setSkipLinesHead(Integer skipLinesHead) {
+                this.skipLinesHead = skipLinesHead;
+            }
+
+            public Integer getSkipLinesTail() {
+                return skipLinesTail;
+            }
+
+            public void setSkipLinesTail(Integer skipLinesTail) {
+                this.skipLinesTail = skipLinesTail;
+            }
+
+            public boolean isRemoveEmptyLines() {
+                return removeEmptyLines;
+            }
+
+            public void setRemoveEmptyLines(boolean removeEmptyLines) {
+                this.removeEmptyLines = removeEmptyLines;
+            }
         }
     }
 }

financer-web-client/src/main/java/de/financer/config/FormatConverter.java

@@ -13,7 +13,7 @@ public class FormatConverter implements Converter<String, FinancerConfig.Transac
     public FinancerConfig.TransactionUpload.Format convert(String source) {
         final String[] data = source.split(",");
 
-        if (data.length != 2) {
+        if (data.length != 5) {
             throw new IllegalArgumentException("Wrong format of Format definition!");
         }
 
@@ -21,6 +21,9 @@ public class FormatConverter implements Converter<String, FinancerConfig.Transac
 
         format.setDateFormat(data[0]);
         format.setLocale(new Locale(data[1]));
+        format.setSkipLinesHead(Integer.valueOf(data[2]));
+        format.setSkipLinesTail(Integer.valueOf(data[3]));
+        format.setRemoveEmptyLines(Boolean.parseBoolean(data[4]));
 
         return format;
     }

financer-web-client/src/main/java/de/financer/transactionUpload/AbstractTransactionUploadWorker.java

@@ -5,8 +5,13 @@ import de.financer.config.FinancerConfig;
 import de.financer.dto.TransactionUploadFormat;
 import de.financer.model.Account;
 import org.apache.commons.collections4.IterableUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
+import java.text.NumberFormat;
+import java.text.ParseException;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Optional;
 import java.util.regex.Matcher;
@@ -27,10 +32,6 @@ public abstract class AbstractTransactionUploadWorker implements TransactionUplo
         return this.financerConfig.getTransactionUpload().getFormats().get(this.format);
     }
 
-    protected String removeQuotes(String value) {
-        return CharMatcher.is('\"').trimFrom(value);
-    }
-
     public FinancerConfig getFinancerConfig() {
         return financerConfig;
     }
@@ -52,4 +53,21 @@ public abstract class AbstractTransactionUploadWorker implements TransactionUplo
 
         return Pair.of(null, rawDescription);
     }
+
+    protected Long formatAmount(String amountString) {
+        try {
+            return Math.abs(NumberFormat.getNumberInstance(this.getFormatConfig().getLocale())
+                                        .parse(StringUtils.replace(amountString, ".", ""))
+                                        .longValue());
+        } catch (ParseException e) {
+            // Make it unchecked because of the usage in the lambda pipeline
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected String formatDate(String rawDate) {
+        // Format to common format
+        return LocalDate.parse(rawDate, DateTimeFormatter.ofPattern(this.getFormatConfig().getDateFormat()))
+                        .format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+    }
 }

financer-web-client/src/main/java/de/financer/transactionUpload/MT940CSVTransactionUploadWorker.java

@@ -13,14 +13,14 @@ import org.springframework.web.multipart.MultipartFile;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.text.NumberFormat;
 import java.text.ParseException;
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.Optional;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWorker {
     /**
@@ -43,29 +43,34 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
             final CSVParser parser = CSVFormat.DEFAULT.builder()
                                                       .setDelimiter(";")
                                                       .setNullString(null)
+                                                      .setIgnoreEmptyLines(this.getFormatConfig().isRemoveEmptyLines())
                                                       .build()
                                                       .parse(new InputStreamReader(is));
 
-            retVal.addAll(parser.stream().skip(1)
+            final List<CSVRecord> records = parser.getRecords();
-                                .map(r -> {
+            final long limit = records.size() - (this.getFormatConfig().getSkipLinesHead() +
-                                            final Pair<Account, String> accountAndDescription =
+                                                 this.getFormatConfig().getSkipLinesTail());
-                                                    this.getAccountAndDescription(buildDescription(r));
 
-                                            return new UploadedTransaction(
+            retVal.addAll(records.stream()
-                                                // Amount
+                                 .skip(this.getFormatConfig().getSkipLinesHead())
-                                                Math.abs(formatAmount(removeQuotes(r.get(AMOUNT_INDEX)))),
+                                 .limit(limit)
-                                                // Description
+                                 .map(r -> {
-                                                accountAndDescription.getRight(),
+                                             final Pair<Account, String> accountAndDescription =
-                                                // Date
+                                                     this.getAccountAndDescription(buildDescription(r));
-                                                formatDate(LocalDate.parse(removeQuotes(r.get(DATE_INDEX)),
+
-                                                                DateTimeFormatter.ofPattern(this.getFormatConfig()
+                                             return new UploadedTransaction(
-                                                                                                .getDateFormat()))),
+                                                 // Amount
-                                                // To account key
+                                                 formatAmount(r.get(AMOUNT_INDEX)),
-                                                Optional.ofNullable(accountAndDescription.getLeft())
+                                                 // Description
-                                                        .map(Account::getKey).orElse(null));
+                                                 accountAndDescription.getRight(),
-                                          }
+                                                 // Date
-                                )
+                                                 formatDate(r.get(DATE_INDEX)),
-                                .collect(Collectors.toList()));
+                                                 // To account key
+                                                 Optional.ofNullable(accountAndDescription.getLeft())
+                                                         .map(Account::getKey).orElse(null));
+                                           }
+                                 )
+                                 .collect(Collectors.toList()));
         } catch (IOException | RuntimeException e) {
             if (e instanceof RuntimeException) {
                 if (e.getCause() instanceof ParseException) {
@@ -80,11 +85,6 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
         return retVal;
     }
 
-    private String formatDate(LocalDate date) {
-        // Format to common format
-        return date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
-    }
-
     private String buildDescription(CSVRecord r) {
         final String description1 = r.get(DESCRIPTION1_INDEX);
         final String description2 = r.get(DESCRIPTION2_INDEX);
@@ -95,15 +95,4 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
 
         return description2;
     }
-
-    private Long formatAmount(String amountString) {
-        try {
-            return NumberFormat.getNumberInstance(this.getFormatConfig().getLocale())
-                               .parse(amountString)
-                               .longValue();
-        } catch (ParseException e) {
-            // Make it unchecked because of the usage in the lambda pipeline
-            throw new RuntimeException(e);
-        }
-    }
 }

financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java

@@ -18,6 +18,10 @@ public class TransactionUploadWorkerFactory {
             case MT940_CSV:
                 worker = new MT940CSVTransactionUploadWorker(this.financerConfig, accounts);
 
+                break;
+            case VB_CSV:
+                worker = new VBCSVTransactionUploadWorker(this.financerConfig, accounts);
+
                 break;
             default:
                 worker = null;

financer-web-client/src/main/java/de/financer/transactionUpload/VBCSVTransactionUploadWorker.java

@@ -0,0 +1,91 @@
+package de.financer.transactionUpload;
+
+import de.financer.config.FinancerConfig;
+import de.financer.dto.TransactionUploadFormat;
+import de.financer.model.Account;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class VBCSVTransactionUploadWorker extends AbstractTransactionUploadWorker {
+    private static final int DATE_INDEX = 1;
+    private static final int DESCRIPTION1_INDEX = 8;
+    private static final int AMOUNT_INDEX = 11;
+
+    protected VBCSVTransactionUploadWorker(FinancerConfig financerConfig, Iterable<Account> accounts) {
+        super(TransactionUploadFormat.VB_CSV, financerConfig, accounts);
+    }
+
+    @Override
+    public Collection<UploadedTransaction> process(MultipartFile file) throws Exception {
+        final Collection<UploadedTransaction> retVal = new ArrayList<>();
+
+        try (InputStream is = file.getInputStream()) {
+            final CSVParser parser = CSVFormat.DEFAULT.builder()
+                                                      .setDelimiter(";")
+                                                      .setIgnoreEmptyLines(this.getFormatConfig().isRemoveEmptyLines())
+                                                      .build()
+                                                      .parse(new InputStreamReader(is));
+
+            final List<CSVRecord> records = parser.getRecords();
+            final long limit = records.size() - (this.getFormatConfig().getSkipLinesHead() +
+                                                 this.getFormatConfig().getSkipLinesTail());
+
+            retVal.addAll(records.stream()
+                                .skip(this.getFormatConfig().getSkipLinesHead())
+                                .limit(limit)
+                                .map(r -> {
+                                            final Pair<Account, String> accountAndDescription =
+                                                    this.getAccountAndDescription(buildDescription(r));
+
+                                            return new UploadedTransaction(
+                                                    // Amount
+                                                    formatAmount(r.get(AMOUNT_INDEX)),
+                                                    // Description
+                                                    accountAndDescription.getRight(),
+                                                    // Date
+                                                    formatDate(r.get(DATE_INDEX)),
+                                                    // To account key
+                                                    Optional.ofNullable(accountAndDescription.getLeft())
+                                                            .map(Account::getKey).orElse(null));
+                                        }
+                                )
+                                .collect(Collectors.toList()));
+        } catch (IOException | RuntimeException e) {
+            if (e instanceof RuntimeException) {
+                if (e.getCause() instanceof ParseException) {
+                    // Make it checked again
+                    throw (ParseException) e.getCause();
+                }
+            }
+
+            throw e;
+        }
+
+        return retVal;
+    }
+
+    private String buildDescription(CSVRecord r) {
+        final String description1 = r.get(DESCRIPTION1_INDEX);
+
+        if (StringUtils.isEmpty(description1)) {
+            return "";
+        }
+
+        // \R is shorthand for line break chars
+        return description1.replaceAll("\\R", " ");
+    }
+}

financer-web-client/src/main/resources/config/application.properties

@@ -62,4 +62,8 @@ financer.darkMode=true
 # The list-style property has to be compliant to the format
 #  1 -> date format
 #  2 -> locale
-financer.transactionUpload.formats.MT940_CSV=dd.MM.yy,de_DE
+#  3 -> skip number of lines HEAD
+#  4 -> skip number of lines TAIL
+#  5 -> remove empty lines
+financer.transactionUpload.formats.MT940_CSV=dd.MM.yy,de_DE,1,0,false
+financer.transactionUpload.formats.VB_CSV=dd.MM.yyyy,de_DE,9,2,true

financer-web-client/src/main/resources/i18n/message.properties

@@ -175,13 +175,15 @@ financer.search-transactions.show-query-options.between=Filtering of values (str
 
 financer.upload-transactions.title=financer\: upload transactions
 financer.upload-transactions.label.format=Format\:
-financer.upload-transactions.format.MT940_CSV=MT940 CSV
 financer.upload-transactions.label.file=File\:
 financer.upload-transactions.submit=Upload transactions
 financer.upload-transactions.label.new-period-on-recurring-transaction-summary=New period on encountering a certain recurring transaction
 financer.upload-transactions.label.new-period-on-recurring-transaction=Recurring transaction\:
 financer.upload-transactions.label.new-period-on-recurring-transaction-enable=Enable\:
 
+financer.upload-transactions.format.MT940_CSV=MT940 CSV
+financer.upload-transactions.format.VB_CSV=Volksbank CSV
+
 financer.create-upload-transactions.title=financer\: create uploaded transactions
 financer.create-upload-transactions.table-header.create=Create
 financer.create-upload-transactions.table-header.fromAccount=From account

financer-web-client/src/main/resources/i18n/message_de_DE.properties

@@ -175,13 +175,15 @@ financer.search-transactions.show-query-options.between=Filtern von Werten (Zeic
 
 financer.upload-transactions.title=financer\: Buchungen hochladen
 financer.upload-transactions.label.format=Format\:
-financer.upload-transactions.format.MT940_CSV=MT940 CSV
 financer.upload-transactions.label.file=Datei\:
 financer.upload-transactions.submit=Buchungen hochladen
 financer.upload-transactions.label.new-period-on-recurring-transaction-summary=Neue Periode bei wiederkehrender Buchung
 financer.upload-transactions.label.new-period-on-recurring-transaction=Wiederkehrende Buchung\:
 financer.upload-transactions.label.new-period-on-recurring-transaction-enable=Aktiv\:
 
+financer.upload-transactions.format.MT940_CSV=MT940 CSV
+financer.upload-transactions.format.VB_CSV=Volksbank CSV
+
 financer.create-upload-transactions.title=financer\: Erstelle hochgeladene Buchungen
 financer.create-upload-transactions.table-header.create=Erstellen
 financer.create-upload-transactions.table-header.fromAccount=Von Konto

financer-web-client/src/main/resources/static/changelog.txt

@@ -8,6 +8,7 @@ v48 -> v49:
 - Added a new column to the transaction upload creation form that is a consecutive number
 - #24 Added the possibility to add regular expressions to accounts so that transactions uploaded can be automatically
   matched to accounts
+- #25 Added Volksbank CSV as transaction upload format
 
 v47 -> v48:
 - #20 Added new property 'transaction type' to a transaction, denoting the type of the transaction, e.g. asset swap,