MK13/financer
1
0
Fork 0
Code Issues 10 Actions Activity

#24 Transaction import: rules for automatic account matching

Browse Source
This commit is contained in:
MK13
2021-09-02 02:30:24 +02:00
parent 70218ad7dc
commit d88e2583b5
22 changed files with 184 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
financer-web-client/src/main/java/de/financer/controller/AccountController.java
View File

@@ -28,9 +28,12 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.util.UriComponentsBuilder;
import javax.servlet.http.HttpServletRequest;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -90,12 +93,13 @@ public class AccountController {
}
@PostMapping("/saveAccount")
public String saveAccount(NewAccountForm form, Model model) {
public String saveAccount(NewAccountForm form, Model model) throws UnsupportedEncodingException {
final UriComponentsBuilder builder = UriComponentsBuilder
.fromHttpUrl(ControllerUtils.buildUrl(this.financerConfig, Function.ACC_CREATE_ACCOUNT))
.queryParam("key", form.getKey())
.queryParam("accountGroupName", form.getGroup())
.queryParam("type", form.getType());
.queryParam("type", form.getType())
.queryParam("regexps", URLEncoder.encode(form.getRegexps(), StandardCharsets.UTF_8.name()));
final ResponseEntity<String> response = new StringTemplate().exchange(builder);
final ResponseReason responseReason = ResponseReason.fromResponseEntity(response);
@@ -230,12 +234,12 @@ public class AccountController {
@GetMapping("/editAccount")
public String editAccount(Model model, String key) {
_editAccount(model, key, Optional.empty(), Optional.empty());
_editAccount(model, key, Optional.empty(), Optional.empty(), Optional.empty());
return "account/editAccount";
}
private void _editAccount(Model model, String originalKey, Optional<String> newKey, Optional<String> newGroup) {
private void _editAccount(Model model, String originalKey, Optional<String> newKey, Optional<String> newGroup, Optional<String> regexps) {
final ResponseEntity<Account> exchange = new GetAccountByKeyTemplate().exchange(this.financerConfig, originalKey);
final ResponseEntity<Iterable<AccountGroup>> accountGroupResponse = new GetAllAccountGroupsTemplate()
.exchange(this.financerConfig);
@@ -249,6 +253,7 @@ public class AccountController {
form.setGroup(newGroup.orElse(Optional.ofNullable(account.getAccountGroup()).map(AccountGroup::getName).orElse(null)));
form.setId(account.getId().toString());
form.setOriginalKey(originalKey);
form.setRegexps(regexps.orElse(account.getUploadMatchRegexps()));
model.addAttribute("form", form);
@@ -258,18 +263,19 @@ public class AccountController {
}
@PostMapping("/editAccount")
public String editAccount(Model model, EditAccountForm form) {
public String editAccount(Model model, EditAccountForm form) throws UnsupportedEncodingException {
final UriComponentsBuilder editBuilder = UriComponentsBuilder
.fromHttpUrl(ControllerUtils.buildUrl(this.financerConfig, Function.ACC_EDIT_ACCOUNT))
.queryParam("id", form.getId())
.queryParam("key", form.getKey())
.queryParam("accountGroupName", form.getGroup());
.queryParam("accountGroupName", form.getGroup())
.queryParam("regexps", URLEncoder.encode(form.getRegexps(), StandardCharsets.UTF_8.name()));
final ResponseEntity<String> closeResponse = new StringTemplate().exchange(editBuilder);
final ResponseReason responseReason = ResponseReason.fromResponseEntity(closeResponse);
if (!ResponseReason.OK.equals(responseReason)) {
_editAccount(model, form.getOriginalKey(), Optional.of(form.getKey()), Optional.of(form.getGroup()));
_editAccount(model, form.getOriginalKey(), Optional.of(form.getKey()), Optional.of(form.getGroup()), Optional.of(form.getRegexps()));
return "account/editAccount";
}

11
financer-web-client/src/main/java/de/financer/controller/TransactionController.java
View File

@@ -211,8 +211,12 @@ public class TransactionController {
// TODO ENCODING??
try {
final Iterable<Account> allAccounts =
FinancerRestTemplate.exchangeGet(this.financerConfig,
Function.ACC_GET_ALL,
new ParameterizedTypeReference<Iterable<Account>>() {});
final TransactionUploadWorker worker = this.workerFactory
.getWorker(TransactionUploadFormat.valueOf(form.getFormat()));
.getWorker(TransactionUploadFormat.valueOf(form.getFormat()), allAccounts);
final Collection<UploadedTransaction> uploadedTransactions = worker.process(form.getFile());
@@ -229,6 +233,7 @@ public class TransactionController {
}
catch(Exception e) {
// TODO
e.printStackTrace();
final ResponseEntity<Iterable<RecurringTransaction>> response = new GetAllRecurringTransactionsTemplate()
.exchange(this.financerConfig);
final List<RecurringTransaction> recurringTransactionList = new ArrayList<>();
@@ -248,7 +253,7 @@ public class TransactionController {
ControllerUtils.addCurrencySymbol(model, this.financerConfig);
ControllerUtils.addDarkMode(model, this.financerConfig);
return "transaction/createUploadedTransactions";
return "transaction/uploadTransactions";
}
}
@@ -303,6 +308,8 @@ public class TransactionController {
}
catch(Exception e) {
// TODO
e.printStackTrace();
return _uploadTransactions(model, Optional.empty(), form);
}

1
financer-web-client/src/main/java/de/financer/form/CreateUploadedTransactionForm.java
View File

@@ -54,6 +54,7 @@ public class CreateUploadedTransactionForm {
entry.setAmount(uploadedTransaction.getAmount());
entry.setDescription(uploadedTransaction.getDescription());
entry.setDate(uploadedTransaction.getDate());
entry.setToAccountKey(uploadedTransaction.getToAccountKey());
return entry;
}

9
financer-web-client/src/main/java/de/financer/form/EditAccountForm.java
View File

@@ -5,6 +5,7 @@ public class EditAccountForm {
private String group;
private String id;
private String originalKey;
private String regexps;
public String getKey() {
return key;
@@ -37,4 +38,12 @@ public class EditAccountForm {
public void setOriginalKey(String originalKey) {
this.originalKey = originalKey;
}
public String getRegexps() {
return regexps;
}
public void setRegexps(String regexps) {
this.regexps = regexps;
}
}

9
financer-web-client/src/main/java/de/financer/form/NewAccountForm.java
View File

@@ -4,6 +4,7 @@ public class NewAccountForm {
private String key;
private String type;
private String group;
private String regexps;
public String getKey() {
return key;
@@ -28,4 +29,12 @@ public class NewAccountForm {
public void setGroup(String group) {
this.group = group;
}
public String getRegexps() {
return regexps;
}
public void setRegexps(String regexps) {
this.regexps = regexps;
}
}

34
financer-web-client/src/main/java/de/financer/transactionUpload/AbstractTransactionUploadWorker.java
View File

@@ -3,14 +3,24 @@ package de.financer.transactionUpload;
import com.google.common.base.CharMatcher;
import de.financer.config.FinancerConfig;
import de.financer.dto.TransactionUploadFormat;
import de.financer.model.Account;
import org.apache.commons.collections4.IterableUtils;
import org.apache.commons.lang3.tuple.Pair;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public abstract class AbstractTransactionUploadWorker implements TransactionUploadWorker {
private FinancerConfig financerConfig;
private TransactionUploadFormat format;
private final FinancerConfig financerConfig;
private final TransactionUploadFormat format;
private final List<Account> accounts;
protected AbstractTransactionUploadWorker(TransactionUploadFormat format, FinancerConfig financerConfig) {
protected AbstractTransactionUploadWorker(TransactionUploadFormat format, FinancerConfig financerConfig, Iterable<Account> accounts) {
this.format = format;
this.financerConfig = financerConfig;
this.accounts = IterableUtils.toList(accounts);
}
protected FinancerConfig.TransactionUpload.Format getFormatConfig() {
@@ -24,4 +34,22 @@ public abstract class AbstractTransactionUploadWorker implements TransactionUplo
public FinancerConfig getFinancerConfig() {
return financerConfig;
}
protected Pair<Account, String> getAccountAndDescription(String rawDescription) {
for (Account a : this.accounts) {
final String[] regexps = Optional.ofNullable(a.getUploadMatchRegexps()).orElse("").split("\r\n");
if (regexps.length > 0) {
for(String regex : regexps) {
Matcher matcher = Pattern.compile(regex).matcher(rawDescription);
if(matcher.matches()) {
return Pair.of(a, matcher.group(1));
}
}
}
}
return Pair.of(null, rawDescription);
}
}

22
financer-web-client/src/main/java/de/financer/transactionUpload/MT940CSVTransactionUploadWorker.java
View File

@@ -2,10 +2,12 @@ package de.financer.transactionUpload;
import de.financer.config.FinancerConfig;
import de.financer.dto.TransactionUploadFormat;
import de.financer.model.Account;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
@@ -17,6 +19,7 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Optional;
import java.util.stream.Collectors;
public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWorker {
@@ -28,8 +31,8 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
private static final int DESCRIPTION2_INDEX = 4;
private static final int AMOUNT_INDEX = 8;
protected MT940CSVTransactionUploadWorker(FinancerConfig financerConfig) {
super(TransactionUploadFormat.MT940_CSV, financerConfig);
protected MT940CSVTransactionUploadWorker(FinancerConfig financerConfig, Iterable<Account> accounts) {
super(TransactionUploadFormat.MT940_CSV, financerConfig, accounts);
}
@Override
@@ -44,16 +47,23 @@ public class MT940CSVTransactionUploadWorker extends AbstractTransactionUploadWo
.parse(new InputStreamReader(is));
retVal.addAll(parser.stream().skip(1)
.map(r -> new UploadedTransaction(
.map(r -> {
final Pair<Account, String> accountAndDescription =
this.getAccountAndDescription(buildDescription(r));
return new UploadedTransaction(
// Amount
Math.abs(formatAmount(removeQuotes(r.get(AMOUNT_INDEX)))),
// Description
buildDescription(r),
accountAndDescription.getRight(),
// Date
formatDate(LocalDate.parse(removeQuotes(r.get(DATE_INDEX)),
DateTimeFormatter.ofPattern(this.getFormatConfig()
.getDateFormat())))
)
.getDateFormat()))),
// To account key
Optional.ofNullable(accountAndDescription.getLeft())
.map(Account::getKey).orElse(null));
}
)
.collect(Collectors.toList()));
} catch (IOException | RuntimeException e) {

1
financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorker.java
View File

@@ -1,5 +1,6 @@
package de.financer.transactionUpload;
import de.financer.model.Account;
import org.springframework.web.multipart.MultipartFile;
import java.util.Collection;

5
financer-web-client/src/main/java/de/financer/transactionUpload/TransactionUploadWorkerFactory.java
View File

@@ -2,6 +2,7 @@ package de.financer.transactionUpload;
import de.financer.config.FinancerConfig;
import de.financer.dto.TransactionUploadFormat;
import de.financer.model.Account;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -10,12 +11,12 @@ public class TransactionUploadWorkerFactory {
@Autowired
private FinancerConfig financerConfig;
public TransactionUploadWorker getWorker(TransactionUploadFormat format) {
public TransactionUploadWorker getWorker(TransactionUploadFormat format, Iterable<Account> accounts) {
AbstractTransactionUploadWorker worker;
switch (format) {
case MT940_CSV:
worker = new MT940CSVTransactionUploadWorker(this.financerConfig);
worker = new MT940CSVTransactionUploadWorker(this.financerConfig, accounts);
break;
default:

12
financer-web-client/src/main/java/de/financer/transactionUpload/UploadedTransaction.java
View File

@@ -4,11 +4,13 @@ public class UploadedTransaction {
private Long amount;
private String description;
private String date;
private String toAccountKey;
public UploadedTransaction(Long amount, String description, String date) {
public UploadedTransaction(Long amount, String description, String date, String toAccountKey) {
this.amount = amount;
this.description = description;
this.date = date;
this.toAccountKey = toAccountKey;
}
public Long getAmount() {
@@ -34,4 +36,12 @@ public class UploadedTransaction {
public void setDate(String date) {
this.date = date;
}
public String getToAccountKey() {
return toAccountKey;
}
public void setToAccountKey(String toAccountKey) {
this.toAccountKey = toAccountKey;
}
}

2
financer-web-client/src/main/resources/i18n/message.properties
View File

@@ -34,12 +34,14 @@ financer.account-new.title=financer\: create new account
financer.account-new.label.key=Key\:
financer.account-new.label.type=Type\:
financer.account-new.label.group=Group\:
financer.account-new.label.upload-match-regexps=Import match regexps\:
financer.account-new.submit=Create account
financer.account-edit.title=financer\: edit account
financer.account-edit.label.key=Key\:
financer.account-edit.label.group=Group\:
financer.account-edit.submit=Edit account
financer.account-edit.label.upload-match-regexps=Import match regexps\:
financer.account-group-new.title=financer\: create new account group
financer.account-group-new.label.name=Name\:

2
financer-web-client/src/main/resources/i18n/message_de_DE.properties
View File

@@ -34,6 +34,7 @@ financer.account-new.title=financer\: Neues Konto erstellen
financer.account-new.label.key=Schl\u00FCssel\:
financer.account-new.label.type=Typ\:
financer.account-new.label.group=Gruppe\:
financer.account-new.label.upload-match-regexps=Zuordnungsregexps\:
financer.account-new.submit=Konto erstellen
financer.account-group-new.title=financer\: Neue Konto-Gruppe erstellen
@@ -44,6 +45,7 @@ financer.account-edit.title=financer\: Bearbeite Konto
financer.account-edit.label.key=Schl\u00FCssel\:
financer.account-edit.label.group=Gruppe\:
financer.account-edit.submit=Konto bearbeiten
financer.account-edit.label.upload-match-regexps=Zuordnungsregexps\:
financer.transaction-new.title=financer\: Neue Buchung erstellen
financer.transaction-new.label.from-account=Von Konto\:

3
financer-web-client/src/main/resources/static/changelog.txt
View File

@@ -5,6 +5,9 @@ v48 -> v49:
- #25 Added the possibility to edit accounts
- #23 Added an option to specify a recurring transaction during transaction upload that will close and open expense
periods
- Added a new column to the transaction upload creation form that is a consecutive number
- #24 Added the possibility to add regular expressions to accounts so that transactions uploaded can be automatically
matched to accounts
v47 -> v48:
- #20 Added new property 'transaction type' to a transaction, denoting the type of the transaction, e.g. asset swap,

28
financer-web-client/src/main/resources/static/readme.txt
View File

@@ -13,17 +13,17 @@
6. Account groups
7. Transactions
8. Recurring transactions
9. Reporting
10. FQL
11. Setup
12. Links
9. Transaction upload
10. Reporting
11. FQL
12. Setup
13. Links
1. About
========
This is the manual for the financer application - a simple app to support you in managing your personal finances.
The main goal of the financer application is to keep things simple by not attempting to provide sophisticated
automation. Instead it is merely a tool that provides basic key values to support you.
The main goal of the financer application is to keep things simple.
2. Overview
===========
@@ -183,18 +183,22 @@
8. Recurring transactions
=========================
9. Reporting
============
9. Transaction upload
=====================
File types + regex
10. FQL
10. Reporting
=============
11. FQL
=======
11. Setup
12. Setup
=========
This chapter explains how to setup a financer instance. It requires PostgreSQL as a database backend and a Java
Servlet Container (e.g. Apache Tomcat) as a runtime environment.
11.1 Database setup
12.1 Database setup
-------------------
First install PostgreSQL. Then create a user for financer:
sudo -iu postgres
@@ -210,7 +214,7 @@
\q
exit
12. Links
13. Links
=========
This chapter contains useful links:
- financer web page: https://financer.dev/

2
financer-web-client/src/main/resources/templates/account/editAccount.html
View File

@@ -20,6 +20,8 @@
<select id="selectGroup" th:field="*{group}">
<option th:each="group : ${accountGroups}" th:value="${group.name}" th:text="${group.name}"/>
</select>
<label for="inputRegexps" th:text="#{financer.account-edit.label.upload-match-regexps}"/>
<textarea type="text" id="inputRegexps" th:field="*{regexps}" rows="8" cols="70"/>
<input type="hidden" id="inputId" th:field="*{id}"/>
<input type="hidden" id="originalKey" th:field="*{originalKey}"/>
<input type="submit" th:value="#{financer.account-edit.submit}" />

2
financer-web-client/src/main/resources/templates/account/newAccount.html
View File

@@ -24,6 +24,8 @@
<select id="selectGroup" th:field="*{group}">
<option th:each="group : ${accountGroups}" th:value="${group.name}" th:text="${group.name}"/>
</select>
<label for="inputRegexps" th:text="#{financer.account-new.label.upload-match-regexps}"/>
<textarea type="text" id="inputRegexps" th:field="*{regexps}" rows="8" cols="70"/>
<input type="submit" th:value="#{financer.account-new.submit}" />
</form>
<div th:replace="includes/footer :: footer"/>

2
financer-web-client/src/main/resources/templates/transaction/createUploadedTransactions.html
View File

@@ -17,6 +17,7 @@
method="post" enctype="multipart/form-data">
<table id="create-upload-transactions-table">
<tr>
<th />
<th th:text="#{financer.create-upload-transactions.table-header.create}"/>
<th th:text="#{financer.create-upload-transactions.table-header.fromAccount}"/>
<th th:text="#{financer.create-upload-transactions.table-header.toAccount}"/>
@@ -28,6 +29,7 @@
<th th:text="#{financer.create-upload-transactions.table-header.file}"/>
</tr>
<tr th:each="entry, i : ${form.entries}">
<td th:text="${i.count}"/>
<td>
<input type="checkbox" th:field="*{entries[__${i.index}__].create}"/>
</td>