[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[libeufin] 01/02: Disk-space policy: fix database query.
From: |
gnunet |
Subject: |
[libeufin] 01/02: Disk-space policy: fix database query. |
Date: |
Tue, 13 Dec 2022 23:12:58 +0100 |
This is an automated email from the git hooks/post-receive script.
ms pushed a commit to branch master
in repository libeufin.
commit 812122ba1de2b7eb4bb52c11df05db3c9bf01675
Author: MS <ms@taler.net>
AuthorDate: Tue Dec 13 22:58:32 2022 +0100
Disk-space policy: fix database query.
Revert use of index-based search, because the
deletion of bank messages is now optional. That
avoids to process stored messages more than once.
---
nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt | 4 +++
.../tech/libeufin/nexus/bankaccount/BankAccount.kt | 42 ++++++++++++++--------
.../kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt | 1 +
nexus/src/test/kotlin/MakeEnv.kt | 2 ++
4 files changed, 35 insertions(+), 14 deletions(-)
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
index 4c6dc18e..a120da7b 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/DB.kt
@@ -285,6 +285,8 @@ object NexusBankAccountsTable : LongIdTable() {
val lastStatementCreationTimestamp =
long("lastStatementCreationTimestamp").nullable()
val lastReportCreationTimestamp =
long("lastReportCreationTimestamp").nullable()
val lastNotificationCreationTimestamp =
long("lastNotificationCreationTimestamp").nullable()
+ // Highest bank message ID that this bank account is aware of.
+ val highestSeenBankMessageSerialId = long("highestSeenBankMessageSerialId")
val pain001Counter = long("pain001counter").default(1)
}
@@ -294,11 +296,13 @@ class NexusBankAccountEntity(id: EntityID<Long>) :
LongEntity(id) {
return find { NexusBankAccountsTable.bankAccountName eq name
}.firstOrNull()
}
}
+
var bankAccountName by NexusBankAccountsTable.bankAccountName
var accountHolder by NexusBankAccountsTable.accountHolder
var iban by NexusBankAccountsTable.iban
var bankCode by NexusBankAccountsTable.bankCode
var defaultBankConnection by NexusBankConnectionEntity
optionalReferencedOn NexusBankAccountsTable.defaultBankConnection
+ var highestSeenBankMessageSerialId by
NexusBankAccountsTable.highestSeenBankMessageSerialId
var pain001Counter by NexusBankAccountsTable.pain001Counter
var lastStatementCreationTimestamp by
NexusBankAccountsTable.lastStatementCreationTimestamp
var lastReportCreationTimestamp by
NexusBankAccountsTable.lastReportCreationTimestamp
diff --git
a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
index 6025e5ee..87150a24 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/bankaccount/BankAccount.kt
@@ -39,14 +39,14 @@ import java.time.Instant
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
-// Defaults to false. Gets true if defined as "yes".
private val keepBankMessages: String? =
System.getenv("LIBEUFIN_NEXUS_KEEP_BANK_MESSAGES")
-
fun requireBankAccount(call: ApplicationCall, parameterKey: String):
NexusBankAccountEntity {
val name = call.parameters[parameterKey]
- if (name == null) {
- throw NexusError(HttpStatusCode.InternalServerError, "no parameter for
bank account")
- }
+ if (name == null)
+ throw NexusError(
+ HttpStatusCode.InternalServerError,
+ "no parameter for bank account"
+ )
val account = transaction { NexusBankAccountEntity.findByName(name) }
if (account == null) {
throw NexusError(HttpStatusCode.NotFound, "bank connection '$name' not
found")
@@ -291,7 +291,10 @@ fun processCamtMessage(
* Create new transactions for an account based on bank messages it
* did not see before.
*/
-fun ingestBankMessagesIntoAccount(bankConnectionId: String, bankAccountId:
String): CamtTransactionsCount {
+fun ingestBankMessagesIntoAccount(
+ bankConnectionId: String,
+ bankAccountId: String
+): CamtTransactionsCount {
var totalNew = 0
var downloadedTransactions = 0
transaction {
@@ -304,25 +307,35 @@ fun ingestBankMessagesIntoAccount(bankConnectionId:
String, bankAccountId: Strin
if (acct == null) {
throw NexusError(HttpStatusCode.InternalServerError, "account not
found")
}
+ var lastId = acct.highestSeenBankMessageSerialId
NexusBankMessageEntity.find {
- (NexusBankMessagesTable.bankConnection eq conn.id) and not(
- NexusBankMessagesTable.errors
- )
+ (NexusBankMessagesTable.bankConnection eq conn.id) and
+ (NexusBankMessagesTable.id greater
acct.highestSeenBankMessageSerialId) and
+ // Wrong messages got already skipped by the
+ // index check above. Below is a extra check.
+ not(NexusBankMessagesTable.errors)
}.orderBy(Pair(NexusBankMessagesTable.id, SortOrder.ASC)).forEach {
val doc =
XMLUtil.parseStringIntoDom(it.message.bytes.toString(Charsets.UTF_8))
val processingResult = processCamtMessage(bankAccountId, doc,
it.code)
if (processingResult.newTransactions == -1) {
it.errors = true
+ lastId = it.id.value
return@forEach
}
- /**
- * The XML document from the bank parsed correctly,
- * remove now from the database.
- */
- if (keepBankMessages == null || keepBankMessages != "yes")
it.delete()
totalNew += processingResult.newTransactions
downloadedTransactions += processingResult.downloadedTransactions
+ /**
+ * Disk-space conservative check: only store if "yes" was
+ * explicitly set into the environment variable. Any other
+ * value or non given falls back to deletion.
+ */
+ if (keepBankMessages == null || keepBankMessages != "yes") {
+ it.delete()
+ return@forEach
+ }
+ lastId = it.id.value
}
+ acct.highestSeenBankMessageSerialId = lastId
}
// return totalNew
return CamtTransactionsCount(
@@ -449,6 +462,7 @@ fun importBankAccount(call: ApplicationCall,
offeredBankAccountId: String, nexus
iban = offeredAccount[OfferedBankAccountsTable.iban]
bankCode =
offeredAccount[OfferedBankAccountsTable.bankCode]
defaultBankConnection = conn
+ highestSeenBankMessageSerialId = 0
accountHolder =
offeredAccount[OfferedBankAccountsTable.accountHolder]
}
logger.info("Account ${newImportedAccount.id} gets
imported")
diff --git a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
index 0cc58106..a8069977 100644
--- a/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
+++ b/nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt
@@ -305,6 +305,7 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
reason = "bank gave no BIC"
)
defaultBankConnection = conn
+ highestSeenBankMessageSerialId = 0
}
}
}
diff --git a/nexus/src/test/kotlin/MakeEnv.kt b/nexus/src/test/kotlin/MakeEnv.kt
index 1eab2016..58b2affb 100644
--- a/nexus/src/test/kotlin/MakeEnv.kt
+++ b/nexus/src/test/kotlin/MakeEnv.kt
@@ -89,6 +89,7 @@ fun prepNexusDb() {
iban = FOO_USER_IBAN
bankCode = "SANDBOXX"
defaultBankConnection = c
+ highestSeenBankMessageSerialId = 0
accountHolder = "foo"
}
val b = NexusBankAccountEntity.new {
@@ -96,6 +97,7 @@ fun prepNexusDb() {
iban = BAR_USER_IBAN
bankCode = "SANDBOXX"
defaultBankConnection = c
+ highestSeenBankMessageSerialId = 0
accountHolder = "bar"
}
}
--
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.