Add FQL IN clause support

This commit is contained in:
2020-12-05 02:17:37 +01:00
parent 5a8a444009
commit c193faaf5a
18 changed files with 525 additions and 63 deletions

View File

@@ -11,17 +11,17 @@ orExpression
andExpression
: expression (AND expression)* ;
expression
: (regularExpression | betweenExpression | periodExpression)
: (regularExpression | betweenExpression | periodExpression | specialExpression)
| parenthesisExpression ;
parenthesisExpression
: L_PAREN expr=orExpression R_PAREN ;
orderByExpression
: ORDER_BY field=IDENTIFIER order=(DESC | ASC ) ;
: ORDER_BY field=IDENTIFIER order=(DESC | ASC) ;
// regular expressions - NOT regex
regularExpression
: (stringExpression | intExpression | booleanExpression | dateExpression | likeExpression) ;
: (stringExpression | intExpression | booleanExpression | dateExpression) ;
stringExpression
: field=IDENTIFIER operator=STRING_OPERATOR value=STRING_VALUE ;
intExpression
@@ -32,7 +32,18 @@ booleanExpression
dateExpression
: field=IDENTIFIER operator=STRING_OPERATOR value=DATE_VALUE ;
likeExpression : field=IDENTIFIER LIKE value=STRING_VALUE ;
// special expressions
specialExpression
: (likeExpression | inExpression) ;
likeExpression
: field=IDENTIFIER LIKE value=STRING_VALUE ;
inExpression
: (inIntExpression | inStringExpression) ;
inIntExpression
: field=IDENTIFIER IN L_PAREN value=intValueList R_PAREN ;
inStringExpression
: field=IDENTIFIER IN L_PAREN value=stringValueList R_PAREN ;
// BETWEEN expressions
betweenExpression
@@ -52,6 +63,12 @@ periodConstExpression
LAST_YEAR |
GRAND_TOTAL) ;
// util expressions
stringValueList
: STRING_VALUE (COMMA STRING_VALUE)* ;
intValueList
: INT_VALUE (COMMA INT_VALUE)* ;
/*
* Lexer rules
*/
@@ -86,6 +103,8 @@ ASC : A S C ;
TRUE : T R U E ;
FALSE : F A L S E ;
LIKE : L I K E ;
IN : I N ;
COMMA : ',' ;
// Constant values
CURRENT : C U R R E N T ;

View File

@@ -9,6 +9,8 @@ import de.financer.fql.FQLVisitorImpl;
import de.financer.model.*;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Repository;
import javax.persistence.EntityManager;
@@ -20,6 +22,8 @@ import java.util.List;
@Repository
public class TransactionRepositoryCustomImpl implements TransactionRepositoryCustom {
private static final Logger LOGGER = LoggerFactory.getLogger(TransactionRepositoryCustomImpl.class);
@PersistenceContext
private EntityManager entityManager;
@@ -31,7 +35,8 @@ public class TransactionRepositoryCustomImpl implements TransactionRepositoryCus
Order order,
Boolean taxRelevant,
boolean accountsAnd,
Boolean hasFile) {
Boolean hasFile
) {
final CriteriaBuilder criteriaBuilder = this.entityManager.getCriteriaBuilder();
final CriteriaQuery<SearchTransactionsResponseDto> criteriaQuery = criteriaBuilder
.createQuery(SearchTransactionsResponseDto.class);
@@ -63,15 +68,12 @@ public class TransactionRepositoryCustomImpl implements TransactionRepositoryCus
if (fromAccountPredicate != null && toAccountPredicate != null) {
if (accountsAnd) {
predicates.add(criteriaBuilder.and(fromAccountPredicate, toAccountPredicate));
}
else {
} else {
predicates.add(criteriaBuilder.or(fromAccountPredicate, toAccountPredicate));
}
}
else if (fromAccountPredicate != null) {
} else if (fromAccountPredicate != null) {
predicates.add(fromAccountPredicate);
}
else if (toAccountPredicate != null) {
} else if (toAccountPredicate != null) {
predicates.add(toAccountPredicate);
}
// else: both null, nothing to do
@@ -79,15 +81,14 @@ public class TransactionRepositoryCustomImpl implements TransactionRepositoryCus
if (hasFile != null) {
if (hasFile) {
predicates.add(criteriaBuilder.isNotNull(fileJoin.get(File_.id)));
}
else {
} else {
predicates.add(criteriaBuilder.isNull(fileJoin.get(File_.id)));
}
}
criteriaQuery.where(predicates.toArray(new Predicate[]{}));
switch(order) {
switch (order) {
case TRANSACTIONS_BY_DATE_DESC:
// either leave case as last before default if new cases arrive
// or copy the expression

View File

@@ -0,0 +1,11 @@
package de.financer.fql;
public class FQLException extends RuntimeException {
public FQLException(Exception e) {
super(e);
}
public FQLException(String message) {
super(message);
}
}

View File

@@ -1,11 +1,8 @@
package de.financer.fql;
import de.financer.dto.SearchTransactionsResponseDto;
import de.financer.fql.field_handler.BetweenIntHandler;
import de.financer.fql.field_handler.BetweenStringHandler;
import de.financer.fql.field_handler.PeriodConstHandler;
import de.financer.fql.field_handler.*;
import de.financer.fql.join_handler.JoinKey;
import de.financer.fql.field_handler.IntHandler;
import de.financer.model.*;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.commons.lang3.StringUtils;
@@ -146,8 +143,35 @@ public class FQLVisitorImpl extends FQLBaseVisitor<Predicate> {
fieldMapping.getJoinHandler().apply(this.froms, fieldMapping);
return fieldMapping.getFieldHandler()
.apply(fieldMapping, this.froms, this.criteriaBuilder, ctx.value.getText());
// Overwrite the actual field handler of the field
return new LikeHandler().apply(fieldMapping, this.froms, this.criteriaBuilder, ctx.value.getText());
}
@Override
public Predicate visitInExpression(FQLParser.InExpressionContext ctx) {
return ctx.getChild(0).accept(this);
}
@Override
public Predicate visitInIntExpression(FQLParser.InIntExpressionContext ctx) {
final FieldMapping fieldMapping = FieldMapping.findByFieldName(ctx.field.getText());
final InHandler.InIntHandlerParameterContainer con = InHandler.InIntHandlerParameterContainer.of(ctx.value.getText());
fieldMapping.getJoinHandler().apply(this.froms, fieldMapping);
// Overwrite the actual field handler of the field
return new InHandler().apply(fieldMapping, this.froms, this.criteriaBuilder, con);
}
@Override
public Predicate visitInStringExpression(FQLParser.InStringExpressionContext ctx) {
final FieldMapping fieldMapping = FieldMapping.findByFieldName(ctx.field.getText());
final InHandler.InStringHandlerParameterContainer con = InHandler.InStringHandlerParameterContainer.of(ctx.value.getText());
fieldMapping.getJoinHandler().apply(this.froms, fieldMapping);
// Overwrite the actual field handler of the field
return new InHandler().apply(fieldMapping, this.froms, this.criteriaBuilder, con);
}
@Override
@@ -179,6 +203,7 @@ public class FQLVisitorImpl extends FQLBaseVisitor<Predicate> {
@Override
public Predicate visitBetweenStringExpression(FQLParser.BetweenStringExpressionContext ctx) {
final FieldMapping fieldMapping = FieldMapping.findByFieldName(ctx.field.getText());
final BetweenStringHandler.BetweenStringHandlerParameterContainer con = BetweenStringHandler.BetweenStringHandlerParameterContainer
.of(ctx.left.getText(), ctx.right.getText());
@@ -191,6 +216,7 @@ public class FQLVisitorImpl extends FQLBaseVisitor<Predicate> {
@Override
public Predicate visitBetweenIntExpression(FQLParser.BetweenIntExpressionContext ctx) {
final FieldMapping fieldMapping = FieldMapping.findByFieldName(ctx.field.getText());
final BetweenIntHandler.BetweenIntHandlerParameterContainer con = BetweenIntHandler.BetweenIntHandlerParameterContainer
.of(ctx.left.getText(), ctx.right.getText());

View File

@@ -35,16 +35,10 @@ public enum FieldMapping {
HAS_FILE("hasFile", File_.ID, File.class, FileJoinHandler.class, JoinKey.of(File.class), null, NotNullSyntheticHandler.class),
DESCRIPTION("description", Transaction_.DESCRIPTION, Transaction.class, NoopJoinHandler.class, JoinKey.of(Transaction.class),
null, LikeHandler.class);
null, StringHandler.class);
/**
* The name of the field as used in FQL
*/
private final String fieldName;
/**
* The name of the ORM field
*/
private final String attributeName;
/**
@@ -90,6 +84,9 @@ public enum FieldMapping {
.orElseThrow(() -> new IllegalArgumentException("Unknown field: " + fqlFieldName));
}
/**
* The name of the ORM field
*/
public String getAttributeName() {
return attributeName;
}

View File

@@ -27,12 +27,7 @@ public class BetweenStringHandler implements FieldHandler<BetweenStringHandler.B
public Predicate apply(FieldMapping fieldMapping, Map<JoinKey, From<?, ?>> froms, CriteriaBuilder criteriaBuilder, BetweenStringHandlerParameterContainer con) {
return criteriaBuilder
.between(froms.get(fieldMapping.getJoinKey()).get(fieldMapping.getAttributeName()),
removeQuotes(con.left),
removeQuotes(con.right));
FieldHandlerUtils.removeQuotes(con.left),
FieldHandlerUtils.removeQuotes(con.right));
}
private String removeQuotes(String value) {
return value.replaceAll("'", "");
}
}

View File

@@ -0,0 +1,7 @@
package de.financer.fql.field_handler;
public class FieldHandlerUtils {
public static String removeQuotes(String value) {
return value.replaceAll("'", "");
}
}

View File

@@ -0,0 +1,89 @@
package de.financer.fql.field_handler;
import de.financer.fql.FQLException;
import de.financer.fql.FieldMapping;
import de.financer.fql.join_handler.JoinKey;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.From;
import javax.persistence.criteria.Predicate;
import java.util.*;
import java.util.stream.Collectors;
public class InHandler implements FieldHandler<InHandler.InHandlerParameterContainer<?>> {
public interface InHandlerParameterContainer<T> {
List<T> getValueList();
void setValueList(List<T> valueList);
}
public static class InIntHandlerParameterContainer implements InHandlerParameterContainer<Long> {
private List<Long> valueList;
@Override
public List<Long> getValueList() {
return this.valueList;
}
@Override
public void setValueList(List<Long> valueList) {
this.valueList = valueList;
}
public static InIntHandlerParameterContainer of(String value) {
final InIntHandlerParameterContainer con = new InIntHandlerParameterContainer();
final String[] values = value.split(",");
final List<String> valueListTmp = Arrays.asList(values);
List<Long> valueList;
try {
valueList = valueListTmp.stream().map(s -> Long.valueOf(s)).collect(Collectors.toList());
}
catch (NumberFormatException e) {
throw new FQLException(e);
}
con.setValueList(valueList);
return con;
}
}
public static class InStringHandlerParameterContainer implements InHandlerParameterContainer<String> {
private List<String> valueList;
@Override
public List<String> getValueList() {
return this.valueList;
}
@Override
public void setValueList(List<String> valueList) {
this.valueList = valueList;
}
public static InStringHandlerParameterContainer of(String value) {
final InStringHandlerParameterContainer con = new InStringHandlerParameterContainer();
final String[] values = value.split(",");
List<String> valueList = Arrays.asList(values);
valueList = valueList.stream().map(s -> FieldHandlerUtils.removeQuotes(s)).collect(Collectors.toList());
// STRING_VALUE originates from ANTLR and is added if it encounters a missing STRING_VALUE ('...', )
final Optional<String> stringValueOptional = valueList.stream().filter(s -> s.contains("STRING_VALUE")).findAny();
if (stringValueOptional.isPresent()) {
throw new FQLException("Passed empty value in IN clause");
}
con.setValueList(valueList);
return con;
}
}
@Override
public Predicate apply(FieldMapping fieldMapping, Map<JoinKey, From<?, ?>> froms, CriteriaBuilder criteriaBuilder, InHandlerParameterContainer<?> value) {
return froms.get(fieldMapping.getJoinKey()).get(fieldMapping.getAttributeName()).in(value.getValueList());
}
}

View File

@@ -12,10 +12,6 @@ public class LikeHandler implements FieldHandler<String> {
@Override
public Predicate apply(FieldMapping fieldMapping, Map<JoinKey, From<?, ?>> froms, CriteriaBuilder criteriaBuilder, String value) {
return criteriaBuilder
.like(criteriaBuilder.lower(froms.get(fieldMapping.getJoinKey()).get(fieldMapping.getAttributeName())), removeQuotes(value).toLowerCase());
}
private String removeQuotes(String value) {
return value.replaceAll("'", "");
.like(criteriaBuilder.lower(froms.get(fieldMapping.getJoinKey()).get(fieldMapping.getAttributeName())), FieldHandlerUtils.removeQuotes(value).toLowerCase());
}
}

View File

@@ -12,10 +12,6 @@ public class StringHandler implements FieldHandler<String> {
@Override
public Predicate apply(FieldMapping fieldMapping, Map<JoinKey, From<?, ?>> froms, CriteriaBuilder criteriaBuilder, String value) {
return criteriaBuilder
.equal(froms.get(fieldMapping.getJoinKey()).get(fieldMapping.getAttributeName()), removeQuotes(value));
}
private String removeQuotes(String value) {
return value.replaceAll("'", "");
.equal(froms.get(fieldMapping.getJoinKey()).get(fieldMapping.getAttributeName()), FieldHandlerUtils.removeQuotes(value));
}
}

View File

@@ -6,6 +6,7 @@ import de.financer.dba.TransactionRepository;
import de.financer.dto.ExpensePeriodTotal;
import de.financer.dto.Order;
import de.financer.dto.SearchTransactionsResponseDto;
import de.financer.fql.FQLException;
import de.financer.model.*;
import de.financer.service.exception.FinancerServiceException;
import de.financer.service.parameter.SearchTransactionsParameter;
@@ -396,8 +397,9 @@ public class TransactionService {
try {
return this.transactionRepository.searchTransactions(fql);
}
catch(IllegalArgumentException iae) {
LOGGER.error("Error while parsing FQL", iae);
// Exception handling in the FQLVisitorImpl is a bit messy...
catch(IllegalArgumentException | ClassCastException | NullPointerException | FQLException e) {
LOGGER.error("Error while parsing FQL", e);
throw new FinancerServiceException(ResponseReason.FQL_MALFORMED);
}

View File

@@ -0,0 +1,332 @@
package de.financer.controller.integration;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.financer.FinancerApplication;
import de.financer.ResponseReason;
import de.financer.dto.SearchTransactionsResponseDto;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.MvcResult;
import org.springframework.test.web.servlet.ResultMatcher;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@RunWith(SpringRunner.class)
@SpringBootTest(classes = FinancerApplication.class)
@AutoConfigureMockMvc
@TestPropertySource(
locations = "classpath:application-integrationtest.properties")
// This class does not test the SQL generation and whether these SQLs make sense and return useful data
// but whether the FQL parsing works, that's why every test just has an Assert for an empty transaction list.
// The test DB does not contain transactions. If we reach this Assert the parsing worked.
// Arrange is done by the SpringBootTest annotation, so it is empty for every case as well.
public class TransactionController_FQLIntegrationTest {
@Autowired
private MockMvc mockMvc;
@Autowired
private ObjectMapper objectMapper;
@Test
public void test_period_CURRENT_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("period = CURRENT");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_period_LAST_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("period = LAST");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_period_CURRENT_YEAR_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("period = CURRENT_YEAR");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_period_LAST_YEAR_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("period = LAST_YEAR");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_period_GRAND_TOTAL_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("period = GRAND_TOTAL");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccount_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount = 'Convenience'");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccount_LIKE_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount LIKE 'Conve%'");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccountGroup_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccountGroup = 'car'");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccountGroup_LIKE_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount LIKE 'Ca%'");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccount_int_FAIL() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount = 1", status().is(400));
ResponseReason responseReason = convertException(mvcResult);
// Assert
Assert.assertEquals(ResponseReason.FQL_MALFORMED, responseReason);
}
@Test
public void test_amount_LIKE_FAIL() {
// Arrange
// Act
final MvcResult mvcResult = perform("amount LIKE 1", status().is(400));
ResponseReason responseReason = convertException(mvcResult);
// Assert
Assert.assertEquals(ResponseReason.FQL_MALFORMED, responseReason);
}
@Test
// While this doesn't make any sense whatsoever it is still possible with the FQL grammar
// Maybe this will be fixed in the future
public void test_toAccount_BETWEEN_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount BETWEEN 'Convenience' AND 'Another account'");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccount_IN_SINGLE_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount IN ('Convenience')");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccount_IN_MULTIPLE_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount IN ('Convenience', 'Another account')");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_toAccount_IN_SINGLE_FAIL() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount IN ('Convenience',)", status().is(400));
ResponseReason responseReason = convertException(mvcResult);
// Assert
Assert.assertEquals(ResponseReason.FQL_MALFORMED, responseReason);
}
@Test
public void test_toAccount_IN_MULTIPLE_FAIL() {
// Arrange
// Act
final MvcResult mvcResult = perform("toAccount IN ('Convenience', 'Another account',)", status().is(400));
ResponseReason responseReason = convertException(mvcResult);
// Assert
Assert.assertEquals(ResponseReason.FQL_MALFORMED, responseReason);
}
@Test
public void test_amount_IN_SINGLE_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("amount IN (100)");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_amount_IN_MULTIPLE_OK() {
// Arrange
// Act
final MvcResult mvcResult = perform("amount IN (100, 200)");
final List<SearchTransactionsResponseDto> transactions = convertResult(mvcResult);
// Assert
Assert.assertEquals(0, transactions.size());
}
@Test
public void test_amount_IN_SINGLE_FAIL() {
// Arrange
// Act
final MvcResult mvcResult = perform("amount IN (100,)", status().is(400));
ResponseReason responseReason = convertException(mvcResult);
// Assert
Assert.assertEquals(ResponseReason.FQL_MALFORMED, responseReason);
}
@Test
public void test_amount_IN_MULTIPLE_FAIL() {
// Arrange
// Act
final MvcResult mvcResult = perform("amount IN (100, 200,)", status().is(400));
ResponseReason responseReason = convertException(mvcResult);
// Assert
Assert.assertEquals(ResponseReason.FQL_MALFORMED, responseReason);
}
private MvcResult perform(String fql) {
return perform(fql, status().isOk());
}
private MvcResult perform(String fql, ResultMatcher resultMatcher) {
try {
final MvcResult mvcResult = this.mockMvc
.perform(get("/transactionsByFql")
.contentType(MediaType.APPLICATION_JSON)
.param("fql", URLEncoder.encode(fql, "UTF-8")))
.andExpect(resultMatcher)
.andReturn();
return mvcResult;
}
catch(Exception e) {
Assert.fail("Exception while performing call! " + e.getMessage());
return null;
}
}
private List<SearchTransactionsResponseDto> convertResult(MvcResult mvcResult) {
try {
return this.objectMapper
.readValue(mvcResult.getResponse().getContentAsByteArray(), new TypeReference<List<SearchTransactionsResponseDto>>() {});
} catch (IOException e) {
Assert.fail("Exception while converting result!" + e.getMessage());
return null;
}
}
private ResponseReason convertException(MvcResult mvcResult) {
try {
return ResponseReason.fromResponseEntity(new ResponseEntity<String>(mvcResult.getResponse().getContentAsString(), HttpStatus
.valueOf(mvcResult.getResponse().getStatus())));
} catch (UnsupportedEncodingException e) {
Assert.fail("Exception while converting exception!" + e.getMessage());
return null;
}
}
}

View File

@@ -2,4 +2,4 @@ spring.profiles.active=hsqldb,dev
spring.datasource.url=jdbc:hsqldb:mem:.
spring.datasource.username=sa
spring.flyway.locations=classpath:/database/hsqldb,classpath:/database/hsqldb/integration,classpath:/database/common
spring.flyway.locations=classpath:/database/hsqldb,classpath:/database/hsqldb/integration,classpath:/database/common