[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] 02/02: misc zoneimporter fixes
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] 02/02: misc zoneimporter fixes |
|
Date: |
Thu, 26 Apr 2018 16:40:14 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a commit to branch master
in repository gnunet.
commit 823215c974ccb1ef3cad9eb1082999cd1b910416
Author: Christian Grothoff <address@hidden>
AuthorDate: Thu Apr 26 16:40:09 2018 +0200
misc zoneimporter fixes
---
src/namecache/plugin_namecache_sqlite.c | 75 ++++++++-----
src/namestore/gnunet-service-namestore.c | 14 ++-
src/namestore/gnunet-zoneimport.c | 172 ++++++++++++++++-------------
src/zonemaster/gnunet-service-zonemaster.c | 29 ++++-
4 files changed, 173 insertions(+), 117 deletions(-)
diff --git a/src/namecache/plugin_namecache_sqlite.c
b/src/namecache/plugin_namecache_sqlite.c
index 6f5f2d952..e0f64a6b0 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -136,13 +136,16 @@ create_indices (sqlite3 * dbh)
{
/* create indices */
if ( (SQLITE_OK !=
- sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_query_hash ON
ns096blocks (query,expiration_time)",
+ sqlite3_exec (dbh,
+ "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks
(query,expiration_time)",
NULL, NULL, NULL)) ||
(SQLITE_OK !=
- sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_block_expiration ON
ns096blocks (expiration_time)",
+ sqlite3_exec (dbh,
+ "CREATE INDEX IF NOT EXISTS ir_block_expiration ON
ns096blocks (expiration_time)",
NULL, NULL, NULL)) )
LOG (GNUNET_ERROR_TYPE_ERROR,
- "Failed to create indices: %s\n", sqlite3_errmsg (dbh));
+ "Failed to create indices: %s\n",
+ sqlite3_errmsg (dbh));
}
@@ -208,28 +211,44 @@ database_setup (struct Plugin *plugin)
return GNUNET_SYSERR;
}
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA temp_store=MEMORY",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA synchronous=NORMAL", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA synchronous=NORMAL",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL,
NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA legacy_file_format=OFF",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
- NULL, ENULL));
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA auto_vacuum=INCREMENTAL",
+ NULL, NULL,
+ ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"", NULL,
- NULL, ENULL));
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA encoding=\"UTF-8\"",
+ NULL, NULL,
+ ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL,
NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA locking_mode=EXCLUSIVE",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA page_size=4092",
+ NULL, NULL,
ENULL));
- CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
+ CHECK (SQLITE_OK ==
+ sqlite3_busy_timeout (plugin->dbh,
+ BUSY_TIMEOUT_MS));
/* Create tables */
@@ -237,17 +256,19 @@ database_setup (struct Plugin *plugin)
sq_prepare (plugin->dbh,
"SELECT 1 FROM sqlite_master WHERE tbl_name =
'ns096blocks'",
&stmt));
- if ((sqlite3_step (stmt) == SQLITE_DONE) &&
- (sqlite3_exec
- (plugin->dbh,
- "CREATE TABLE ns096blocks ("
- " query BLOB NOT NULL,"
- " block BLOB NOT NULL,"
- " expiration_time INT8 NOT NULL"
- ")",
- NULL, NULL, NULL) != SQLITE_OK))
+ if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
+ (SQLITE_OK !=
+ sqlite3_exec (plugin->dbh,
+ "CREATE TABLE ns096blocks ("
+ " query BLOB NOT NULL,"
+ " block BLOB NOT NULL,"
+ " expiration_time INT8 NOT NULL"
+ ")",
+ NULL, NULL, NULL)) )
{
- LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
+ LOG_SQLITE (plugin,
+ GNUNET_ERROR_TYPE_ERROR,
+ "sqlite3_exec");
sqlite3_finalize (stmt);
return GNUNET_SYSERR;
}
@@ -411,11 +432,11 @@ namecache_sqlite_cache_block (void *cls,
GNUNET_CRYPTO_hash (&block->derived_key,
sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
&query);
- fprintf (stderr, // GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Caching new version of block %s (expires %s)\n",
- GNUNET_h2s (&query),
- GNUNET_STRINGS_absolute_time_to_string (expiration));
expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Caching new version of block %s (expires %s)\n",
+ GNUNET_h2s (&query),
+ GNUNET_STRINGS_absolute_time_to_string (expiration));
if (block_size > 64 * 65536)
{
GNUNET_break (0);
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index 994eae2c8..06a50132b 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -708,11 +708,15 @@ refresh_block (struct NamestoreClient *nc,
&res);
GNUNET_free (nick);
}
-
- exp_time = (0 == res_count)
- ? GNUNET_TIME_UNIT_ZERO_ABS
- : GNUNET_GNSRECORD_record_get_expiration_time (res_count,
- res);
+ if (0 == res_count)
+ {
+ send_store_response (nc,
+ GNUNET_OK,
+ rid);
+ return; /* no data, no need to update cache */
+ }
+ exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
+ res);
if (cache_keys)
block = GNUNET_GNSRECORD_block_create2 (zone_key,
exp_time,
diff --git a/src/namestore/gnunet-zoneimport.c
b/src/namestore/gnunet-zoneimport.c
index 6622a7676..503262487 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -48,6 +48,16 @@
#define MAX_RETRIES 5
/**
+ * How many DNS requests do we at most issue in rapid series?
+ */
+#define MAX_SERIES 10
+
+/**
+ * How long do we wait at least between series of requests?
+ */
+#define SERIES_DELAY GNUNET_TIME_relative_multiply
(GNUNET_TIME_UNIT_MICROSECONDS, 10)
+
+/**
* How many requests do we request from NAMESTORE in one batch
* during our initial iteration?
*/
@@ -200,11 +210,16 @@ static struct GNUNET_NAMESTORE_Handle *ns;
static struct GNUNET_DNSSTUB_Context *ctx;
/**
- * The number of queries that are outstanding
+ * The number of DNS queries that are outstanding
*/
static unsigned int pending;
/**
+ * The number of NAMESTORE record store operations that are outstanding
+ */
+static unsigned int pending_rs;
+
+/**
* Number of lookups we performed overall.
*/
static unsigned int lookups;
@@ -692,7 +707,7 @@ process_record (void *cls,
req->hostname))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "DNS returned record for `%s' of type %u while resolving
`%s'\n",
+ "DNS returned record from zone `%s' of type %u while resolving
`%s'\n",
rec->name,
(unsigned int) rec->type,
req->hostname);
@@ -910,7 +925,11 @@ store_completed_cb (void *cls,
struct Request *req = cls;
req->qe = NULL;
- pending--;
+ pending_rs--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
if (GNUNET_SYSERR == success)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -967,6 +986,10 @@ process_result (void *cls,
req_tail,
req);
pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Stub gave up on DNS reply for `%s'\n",
req->hostname);
@@ -981,12 +1004,17 @@ process_result (void *cls,
return;
}
if (req->id != dns->id)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "DNS ID did not match request, ignoring reply\n");
return;
+ }
GNUNET_CONTAINER_DLL_remove (req_head,
req_tail,
req);
GNUNET_DNSSTUB_resolve_cancel (req->rs);
req->rs = NULL;
+ pending--;
p = GNUNET_DNSPARSER_parse ((const char *) dns,
dns_len);
if (NULL == p)
@@ -998,11 +1026,17 @@ process_result (void *cls,
{
failures++;
insert_sorted (req);
- pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
return;
}
insert_sorted (req);
- pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
return;
}
/* import new records */
@@ -1047,6 +1081,7 @@ process_result (void *cls,
/* convert linked list into array */
for (rec = req->rec_head; NULL != rec; rec =rec->next)
rd[off++] = rec->grd;
+ pending_rs++;
req->qe = GNUNET_NAMESTORE_records_store (ns,
&req->zone->key,
get_label (req),
@@ -1054,69 +1089,13 @@ process_result (void *cls,
rd,
&store_completed_cb,
req);
+ GNUNET_assert (NULL != req->qe);
}
insert_sorted (req);
}
/**
- * Submit a request to DNS unless we need to slow down because
- * we are at the rate limit.
- *
- * @param req request to submit
- * @return #GNUNET_OK if request was submitted
- * #GNUNET_NO if request was already submitted
- * #GNUNET_SYSERR if we are at the rate limit
- */
-static int
-submit_req (struct Request *req)
-{
- static struct GNUNET_TIME_Absolute last_request;
- struct GNUNET_TIME_Absolute now;
- void *raw;
- size_t raw_size;
-
- if (NULL != req->qe)
- return GNUNET_NO; /* namestore op still pending */
- if (NULL != req->rs)
- {
- GNUNET_break (0);
- return GNUNET_NO; /* already submitted */
- }
- now = GNUNET_TIME_absolute_get ();
- if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) ||
- (pending >= THRESH) )
- return GNUNET_SYSERR;
- GNUNET_CONTAINER_DLL_insert (req_head,
- req_tail,
- req);
- GNUNET_assert (NULL == req->rs);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Requesting resolution for `%s'\n",
- req->hostname);
- raw = build_dns_query (req,
- &raw_size);
- if (NULL == raw)
- {
- GNUNET_break (0);
- free_request (req);
- return GNUNET_SYSERR;
- }
- req->rs = GNUNET_DNSSTUB_resolve (ctx,
- raw,
- raw_size,
- &process_result,
- req);
- GNUNET_assert (NULL != req->rs);
- req->issue_num++;
- last_request = now;
- lookups++;
- pending++;
- return GNUNET_OK;
-}
-
-
-/**
* Process as many requests as possible from the queue.
*
* @param cls NULL
@@ -1125,23 +1104,60 @@ static void
process_queue (void *cls)
{
struct Request *req;
+ unsigned int series;
+ void *raw;
+ size_t raw_size;
(void) cls;
+ series = 0;
t = NULL;
- while (1)
+ while (pending + pending_rs < THRESH)
{
req = GNUNET_CONTAINER_heap_peek (req_heap);
if (NULL == req)
break;
+ if (NULL != req->qe)
+ return; /* namestore op still pending */
+ if (NULL != req->rs)
+ {
+ GNUNET_break (0);
+ return; /* already submitted */
+ }
if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
break;
- if (GNUNET_OK != submit_req (req))
- break;
GNUNET_assert (req ==
GNUNET_CONTAINER_heap_remove_root (req_heap));
req->hn = NULL;
+ GNUNET_CONTAINER_DLL_insert (req_head,
+ req_tail,
+ req);
+ GNUNET_assert (NULL == req->rs);
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Requesting resolution for `%s'\n",
+ req->hostname);
+ raw = build_dns_query (req,
+ &raw_size);
+ if (NULL == raw)
+ {
+ GNUNET_break (0);
+ free_request (req);
+ continue;
+ }
+ req->rs = GNUNET_DNSSTUB_resolve (ctx,
+ raw,
+ raw_size,
+ &process_result,
+ req);
+ GNUNET_assert (NULL != req->rs);
+ req->issue_num++;
+ lookups++;
+ pending++;
+ series++;
+ if (series > MAX_SERIES)
+ break;
}
-
+ if (pending + pending_rs >= THRESH)
+ return; /* wait for replies */
req = GNUNET_CONTAINER_heap_peek (req_heap);
if (NULL == req)
return;
@@ -1156,17 +1172,15 @@ process_queue (void *cls)
t = GNUNET_SCHEDULER_add_at (req->expires,
&process_queue,
NULL);
+ return;
}
- else
- {
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Throttling for 1ms\n");
- if (NULL != t)
- GNUNET_SCHEDULER_cancel (t);
- t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
- &process_queue,
- NULL);
- }
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Throttling\n");
+ if (NULL != t)
+ GNUNET_SCHEDULER_cancel (t);
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
}
@@ -1512,7 +1526,6 @@ iterate_zones (void *cls)
GNUNET_assert (NULL != zone_tail);
if (zone_tail == last)
{
- GNUNET_assert (NULL == t);
/* Done iterating over relevant zones in NAMESTORE, move
rest of hash map to work queue as well. */
GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
@@ -1750,12 +1763,13 @@ main (int argc,
NULL);
GNUNET_free ((void*) argv);
fprintf (stderr,
- "Rejected %u names, did %u lookups, found %u records, %u lookups
failed, %u pending on shutdown\n",
+ "Rejected %u names, did %u lookups, found %u records, %u lookups
failed, %u/%u pending on shutdown\n",
rejects,
lookups,
records,
failures,
- pending);
+ pending,
+ pending_rs);
return 0;
}
diff --git a/src/zonemaster/gnunet-service-zonemaster.c
b/src/zonemaster/gnunet-service-zonemaster.c
index f40f25c3f..518d5f572 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -181,6 +181,13 @@ static struct GNUNET_SCHEDULER_Task *zone_publish_task;
static int first_zone_iteration;
/**
+ * Optimize block insertion by caching map of private keys to
+ * public keys in memory?
+ */
+static int cache_keys;
+
+
+/**
* Task run during shutdown.
*
* @param cls unused
@@ -388,11 +395,18 @@ perform_dht_put (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *key,
expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
rd_public);
- block = GNUNET_GNSRECORD_block_create (key,
- expire,
- label,
- rd_public,
- rd_public_count);
+ if (cache_keys)
+ block = GNUNET_GNSRECORD_block_create2 (key,
+ expire,
+ label,
+ rd_public,
+ rd_public_count);
+ else
+ block = GNUNET_GNSRECORD_block_create (key,
+ expire,
+ label,
+ rd_public,
+ rd_public_count);
if (NULL == block)
{
GNUNET_break (0);
@@ -585,6 +599,7 @@ put_gns_record (void *cls,
static void
publish_zone_dht_start (void *cls)
{
+ (void) cls;
zone_publish_task = NULL;
GNUNET_STATISTICS_update (statistics,
"Full zone iterations launched",
@@ -736,7 +751,9 @@ run (void *cls,
GNUNET_SCHEDULER_shutdown ();
return;
}
-
+ cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
+ "namestore",
+ "CACHE_KEYS");
put_interval = INITIAL_PUT_INTERVAL;
zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
if (GNUNET_OK ==
--
To stop receiving notification emails like this one, please contact
address@hidden