[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] 02/02: cache NICK results in namestore to avoid un
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] 02/02: cache NICK results in namestore to avoid unnecessary load on the DB; improve sqlite DB style, use WAL for sqlite DB |
|
Date: |
Tue, 11 Dec 2018 19:27:38 +0100 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a commit to branch master
in repository gnunet.
commit c1dd4d02762d51d1ac4078407cda2796aff8faf4
Author: Christian Grothoff <address@hidden>
AuthorDate: Tue Dec 11 19:27:34 2018 +0100
cache NICK results in namestore to avoid unnecessary load on the DB;
improve sqlite DB style, use WAL for sqlite DB
---
src/arm/gnunet-arm.c | 17 ++++-
src/namecache/plugin_namecache_sqlite.c | 63 +++++++-----------
src/namestore/gnunet-service-namestore.c | 106 ++++++++++++++++++++++++++++++-
src/namestore/plugin_namestore_sqlite.c | 1 +
4 files changed, 142 insertions(+), 45 deletions(-)
diff --git a/src/arm/gnunet-arm.c b/src/arm/gnunet-arm.c
index 28d1f7592..fa0374bc5 100644
--- a/src/arm/gnunet-arm.c
+++ b/src/arm/gnunet-arm.c
@@ -74,7 +74,7 @@ static char *term;
/**
* Set to the name of the config file used.
*/
-static const char *config_file;
+static char *config_file;
/**
* Set to the directory where runtime files are stored.
@@ -172,6 +172,7 @@ delete_files ()
static void
shutdown_task (void *cls)
{
+ (void) cls;
if (NULL != al_task)
{
GNUNET_SCHEDULER_cancel (al_task);
@@ -284,6 +285,7 @@ conn_status (void *cls,
{
static int once;
+ (void) cls;
if ( (GNUNET_SYSERR == connected) &&
(0 == once) )
{
@@ -312,6 +314,7 @@ start_callback (void *cls,
enum GNUNET_ARM_RequestStatus rs,
enum GNUNET_ARM_Result result)
{
+ (void) cls;
op = NULL;
if (GNUNET_ARM_REQUEST_SENT_OK != rs)
{
@@ -355,6 +358,7 @@ stop_callback (void *cls,
{
char *msg;
+ (void) cls;
op = NULL;
if (GNUNET_ARM_REQUEST_SENT_OK != rs)
{
@@ -405,6 +409,7 @@ init_callback (void *cls,
enum GNUNET_ARM_RequestStatus rs,
enum GNUNET_ARM_Result result)
{
+ (void) cls;
op = NULL;
if (GNUNET_ARM_REQUEST_SENT_OK != rs)
{
@@ -452,6 +457,7 @@ term_callback (void *cls,
{
char *msg;
+ (void) cls;
op = NULL;
if (GNUNET_ARM_REQUEST_SENT_OK != rs)
{
@@ -501,6 +507,7 @@ list_callback (void *cls,
{
unsigned int i;
+ (void) cls;
op = NULL;
if (GNUNET_ARM_REQUEST_SENT_OK != rs)
{
@@ -538,6 +545,7 @@ list_callback (void *cls,
static void
action_loop (void *cls)
{
+ (void) cls;
al_task = NULL;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Running requested actions\n");
@@ -637,6 +645,7 @@ srv_status (void *cls,
{
const char *msg;
+ (void) cls;
switch (status)
{
case GNUNET_ARM_SERVICE_MONITORING_STARTED:
@@ -679,6 +688,7 @@ srv_status (void *cls,
static void
timeout_task_cb (void *cls)
{
+ (void) cls;
timeout_task = NULL;
ret = 2;
GNUNET_SCHEDULER_shutdown ();
@@ -699,8 +709,9 @@ run (void *cls,
const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *c)
{
- char *armconfig;
-
+ (void) cls;
+ (void) args;
+ (void) cfgfile;
cfg = GNUNET_CONFIGURATION_dup (c);
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_string (cfg,
diff --git a/src/namecache/plugin_namecache_sqlite.c
b/src/namecache/plugin_namecache_sqlite.c
index f67831708..b355d37ba 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -168,6 +168,18 @@ create_indices (sqlite3 * dbh)
static int
database_setup (struct Plugin *plugin)
{
+ struct GNUNET_SQ_ExecuteStatement es[] = {
+ GNUNET_SQ_make_try_execute ("PRAGMA temp_store=MEMORY"),
+ GNUNET_SQ_make_try_execute ("PRAGMA synchronous=NORMAL"),
+ GNUNET_SQ_make_try_execute ("PRAGMA legacy_file_format=OFF"),
+ GNUNET_SQ_make_try_execute ("PRAGMA auto_vacuum=INCREMENTAL"),
+ GNUNET_SQ_make_try_execute ("PRAGMA encoding=\"UTF-8\""),
+ GNUNET_SQ_make_try_execute ("PRAGMA locking_mode=EXCLUSIVE"),
+ GNUNET_SQ_make_try_execute ("PRAGMA page_size=4092"),
+ GNUNET_SQ_make_try_execute ("PRAGMA journal_mode=WAL"),
+ GNUNET_SQ_EXECUTE_STATEMENT_END
+ };
+
sqlite3_stmt *stmt;
char *afsdir;
#if ENULL_DEFINED
@@ -208,47 +220,16 @@ database_setup (struct Plugin *plugin)
sqlite3_errmsg (plugin->dbh));
return GNUNET_SYSERR;
}
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA temp_store=MEMORY",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA synchronous=NORMAL",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA legacy_file_format=OFF",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA auto_vacuum=INCREMENTAL",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA encoding=\"UTF-8\"",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA locking_mode=EXCLUSIVE",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA page_size=4092",
- NULL, NULL,
- ENULL));
- CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh,
- "PRAGMA journal_mode=WAL",
- NULL, NULL,
- ENULL));
-
+ if (GNUNET_OK !=
+ GNUNET_SQ_exec_statements (plugin->dbh,
+ es))
+ {
+ GNUNET_break (0);
+ LOG (GNUNET_ERROR_TYPE_ERROR,
+ _("Failed to setup database at `%s'\n"),
+ plugin->fn);
+ return GNUNET_SYSERR;
+ }
CHECK (SQLITE_OK ==
sqlite3_busy_timeout (plugin->dbh,
BUSY_TIMEOUT_MS));
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index d9fa5b3fc..d9e9a9e82 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -46,6 +46,10 @@
*/
#define MONITOR_STALL_WARN_DELAY GNUNET_TIME_UNIT_MINUTES
+/**
+ * Size of the cache used by #get_nick_record()
+ */
+#define NC_SIZE 16
/**
* A namestore client
@@ -292,6 +296,33 @@ struct StoreActivity
/**
+ * Entry in list of cached nick resolutions.
+ */
+struct NickCache
+{
+ /**
+ * Zone the cache entry is for.
+ */
+ struct GNUNET_CRYPTO_EcdsaPrivateKey zone;
+
+ /**
+ * Cached record data.
+ */
+ struct GNUNET_GNSRECORD_Data *rd;
+
+ /**
+ * Timestamp when this cache entry was used last.
+ */
+ struct GNUNET_TIME_Absolute last_used;
+};
+
+
+/**
+ * We cache nick records to reduce DB load.
+ */
+static struct NickCache nick_cache[NC_SIZE];
+
+/**
* Public key of all zeros.
*/
static const struct GNUNET_CRYPTO_EcdsaPrivateKey zero;
@@ -481,6 +512,48 @@ lookup_nick_it (void *cls,
/**
+ * Add entry to the cache for @a zone and @a nick
+ *
+ * @param zone zone key to cache under
+ * @param nick nick entry to cache
+ */
+static void
+cache_nick (const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
+ const struct GNUNET_GNSRECORD_Data *nick)
+{
+ struct NickCache *oldest;
+
+ oldest = NULL;
+ for (unsigned int i=0;i<NC_SIZE;i++)
+ {
+ struct NickCache *pos = &nick_cache[i];
+
+ if ( (NULL == oldest) ||
+ (oldest->last_used.abs_value_us >
+ pos->last_used.abs_value_us) )
+ oldest = pos;
+ if (0 == memcmp (zone,
+ &pos->zone,
+ sizeof (*zone)))
+ {
+ oldest = pos;
+ break;
+ }
+ }
+ GNUNET_free_non_null (oldest->rd);
+ oldest->zone = *zone;
+ oldest->rd = GNUNET_malloc (sizeof (*nick) +
+ nick->data_size);
+ *oldest->rd = *nick;
+ oldest->rd->data = &oldest->rd[1];
+ memcpy (&oldest->rd[1],
+ nick->data,
+ nick->data_size);
+ oldest->last_used = GNUNET_TIME_absolute_get ();
+}
+
+
+/**
* Return the NICK record for the zone (if it exists).
*
* @param zone private key for the zone to look for nick
@@ -493,6 +566,27 @@ get_nick_record (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *zone)
struct GNUNET_GNSRECORD_Data *nick;
int res;
+ /* check cache first */
+ for (unsigned int i=0;i<NC_SIZE;i++)
+ {
+ struct NickCache *pos = &nick_cache[i];
+ if ( (NULL != pos->rd) &&
+ (0 == memcmp (zone,
+ &pos->zone,
+ sizeof (*zone))) )
+ {
+ nick = GNUNET_malloc (sizeof (*nick) +
+ pos->rd->data_size);
+ *nick = *pos->rd;
+ nick->data = &nick[1];
+ memcpy (&nick[1],
+ pos->rd->data,
+ pos->rd->data_size);
+ pos->last_used = GNUNET_TIME_absolute_get ();
+ return nick;
+ }
+ }
+
nick = NULL;
res = GSN_database->lookup_records (GSN_database->cls,
zone,
@@ -508,6 +602,10 @@ get_nick_record (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *zone)
GNUNET_GNSRECORD_z2s (&pub));
return NULL;
}
+
+ /* update cache */
+ cache_nick (zone,
+ nick);
return nick;
}
@@ -663,7 +761,7 @@ send_lookup_response (struct NamestoreClient *nc,
GNUNET_SERVICE_client_drop (nc->client);
return;
}
- if (rd_ser_len >= UINT16_MAX - name_len - sizeof (*zir_msg))
+ if (((size_t) rd_ser_len) >= UINT16_MAX - name_len - sizeof (*zir_msg))
{
GNUNET_break (0);
GNUNET_SERVICE_client_drop (nc->client);
@@ -1445,6 +1543,12 @@ handle_record_store (void *cls,
conv_name)) ||
(GNUNET_GNSRECORD_TYPE_NICK != rd[i].record_type) )
rd_clean_off++;
+
+ if ( (0 == strcmp (GNUNET_GNS_EMPTY_LABEL_AT,
+ conv_name)) &&
+ (GNUNET_GNSRECORD_TYPE_NICK == rd[i].record_type) )
+ cache_nick (&rp_msg->private_key,
+ &rd[i]);
}
res = GSN_database->store_records (GSN_database->cls,
&rp_msg->private_key,
diff --git a/src/namestore/plugin_namestore_sqlite.c
b/src/namestore/plugin_namestore_sqlite.c
index 23af3960d..e68a47a7b 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -122,6 +122,7 @@ database_setup (struct Plugin *plugin)
GNUNET_SQ_make_try_execute ("PRAGMA auto_vacuum=INCREMENTAL"),
GNUNET_SQ_make_try_execute ("PRAGMA encoding=\"UTF-8\""),
GNUNET_SQ_make_try_execute ("PRAGMA locking_mode=EXCLUSIVE"),
+ GNUNET_SQ_make_try_execute ("PRAGMA journal_mode=WAL"),
GNUNET_SQ_make_try_execute ("PRAGMA page_size=4092"),
GNUNET_SQ_make_execute ("CREATE TABLE IF NOT EXISTS ns098records ("
" uid INTEGER PRIMARY KEY,"
--
To stop receiving notification emails like this one, please contact
address@hidden