[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] branch master updated: add option to disable namec
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] branch master updated: add option to disable namecache, add velocity calculation and delay correction to zonemaster, fix some ftbfs from recent NAMESTORE API change |
|
Date: |
Sun, 29 Apr 2018 10:56:22 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a commit to branch master
in repository gnunet.
The following commit(s) were added to refs/heads/master by this push:
new ce2864cfa add option to disable namecache, add velocity calculation
and delay correction to zonemaster, fix some ftbfs from recent NAMESTORE API
change
ce2864cfa is described below
commit ce2864cfaa27e55096b480bf35db5f8cee2a5e7e
Author: Christian Grothoff <address@hidden>
AuthorDate: Sun Apr 29 10:56:15 2018 +0200
add option to disable namecache, add velocity calculation and delay
correction to zonemaster, fix some ftbfs from recent NAMESTORE API change
---
contrib/timeout_watchdog.c | 7 +-
src/credential/gnunet-service-credential.c | 4 +-
src/gns/gns.conf.in | 3 -
src/gns/gnunet-service-gns.c | 10 ++-
src/gns/gnunet-service-gns_resolver.c | 19 ++---
.../gnunet-service-identity-provider.c | 21 +++--
src/identity-provider/plugin_rest_openid_connect.c | 7 +-
src/namecache/namecache.conf.in | 5 ++
src/namestore/gnunet-service-namestore.c | 36 ++++++--
src/namestore/gnunet-zoneimport.c | 16 ++--
src/namestore/plugin_namestore_flat.c | 2 +-
src/zonemaster/gnunet-service-zonemaster.c | 95 ++++++++++++++++++++--
12 files changed, 178 insertions(+), 47 deletions(-)
diff --git a/contrib/timeout_watchdog.c b/contrib/timeout_watchdog.c
index 7a0376b9f..fc61a7cc7 100644
--- a/contrib/timeout_watchdog.c
+++ b/contrib/timeout_watchdog.c
@@ -33,12 +33,14 @@
static pid_t child;
+
static void
sigchld_handler (int val)
{
int status = 0;
int ret = 0;
+ (void) val;
waitpid (child, &status, 0);
if (WIFEXITED (status) != 0)
{
@@ -53,6 +55,7 @@ sigchld_handler (int val)
exit (ret);
}
+
static void
sigint_handler (int val)
{
@@ -60,8 +63,10 @@ sigint_handler (int val)
exit (val);
}
+
int
-main (int argc, char *argv[])
+main (int argc,
+ char *argv[])
{
int timeout = 0;
pid_t gpid = 0;
diff --git a/src/credential/gnunet-service-credential.c
b/src/credential/gnunet-service-credential.c
index a2c339363..399371a2e 100644
--- a/src/credential/gnunet-service-credential.c
+++ b/src/credential/gnunet-service-credential.c
@@ -961,9 +961,11 @@ collect_next (void *cls)
struct VerifyRequestHandle *vrh = cls;
vrh->collect_next_task = NULL;
GNUNET_assert (NULL != vrh->cred_collection_iter);
- GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter);
+ GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter,
+ 1);
}
+
static void
handle_cred_collection_cb (void *cls,
const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
diff --git a/src/gns/gns.conf.in b/src/gns/gns.conf.in
index d48a213e9..2e49a4c60 100644
--- a/src/gns/gns.conf.in
+++ b/src/gns/gns.conf.in
@@ -22,9 +22,6 @@ MAX_PARALLEL_BACKGROUND_QUERIES = 1000
# called via NSS or other mechanisms).
INTERCEPT_DNS = YES
-# Using caching (start with namestore), or always ask DHT?
-USE_CACHE = YES
-
# PREFIX = valgrind --leak-check=full --track-origins=yes
# Zones
diff --git a/src/gns/gnunet-service-gns.c b/src/gns/gnunet-service-gns.c
index 90cd47e1d..c376ddfcc 100644
--- a/src/gns/gnunet-service-gns.c
+++ b/src/gns/gnunet-service-gns.c
@@ -229,14 +229,14 @@ GNS_get_tld (const char *name)
/**
* Task run during shutdown.
*
- * @param cls unused
- * @param tc unused
+ * @param cls unused, NULL
*/
static void
shutdown_task (void *cls)
{
struct GNS_TopLevelDomain *tld;
+ (void) cls;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Shutting down!\n");
GNS_interceptor_done ();
@@ -283,6 +283,7 @@ client_disconnect_cb (void *cls,
struct ClientLookupHandle *clh;
struct GnsClient *gc = app_ctx;
+ (void) cls;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Client %p disconnected\n",
client);
@@ -314,6 +315,8 @@ client_connect_cb (void *cls,
struct GNUNET_MQ_Handle *mq)
{
struct GnsClient *gc;
+
+ (void) cls;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Client %p connected\n",
client);
@@ -383,6 +386,7 @@ check_lookup (void *cls,
size_t msg_size;
const char* name;
+ (void) cls;
msg_size = ntohs (l_msg->header.size);
if (msg_size < sizeof (struct LookupMessage))
{
@@ -479,6 +483,8 @@ read_service_conf (void *cls,
struct GNUNET_CRYPTO_EcdsaPublicKey pk;
struct GNS_TopLevelDomain *tld;
+ (void) cls;
+ (void) section;
if (option[0] != '.')
return;
if (GNUNET_OK !=
diff --git a/src/gns/gnunet-service-gns_resolver.c
b/src/gns/gnunet-service-gns_resolver.c
index 745a2f3bd..92e03bc69 100644
--- a/src/gns/gnunet-service-gns_resolver.c
+++ b/src/gns/gnunet-service-gns_resolver.c
@@ -501,7 +501,7 @@ static struct CacheOps *co_tail;
/**
* Use namecache
*/
-static int use_cache;
+static int disable_cache;
/**
* Global configuration.
@@ -2478,7 +2478,7 @@ recursive_gns_resolution_namecache (struct
GNS_ResolverHandle *rh)
GNUNET_GNSRECORD_query_from_public_key (&ac->authority_info.gns_authority,
ac->label,
&query);
- if (GNUNET_YES == use_cache)
+ if (GNUNET_YES != disable_cache)
{
rh->namecache_qe
= GNUNET_NAMECACHE_lookup_block (namecache_handle,
@@ -2489,7 +2489,8 @@ recursive_gns_resolution_namecache (struct
GNS_ResolverHandle *rh)
}
else
{
- start_dht_request (rh, &query);
+ start_dht_request (rh,
+ &query);
}
}
@@ -2816,13 +2817,11 @@ GNS_resolver_init (struct GNUNET_NAMECACHE_Handle *nc,
dht_lookup_heap =
GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
max_allowed_background_queries = max_bg_queries;
- if (GNUNET_SYSERR == (use_cache =
- GNUNET_CONFIGURATION_get_value_yesno (c,
- "gns",
- "USE_CACHE")))
- use_cache = GNUNET_YES;
- if (GNUNET_NO == use_cache)
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ disable_cache = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+ "namecache",
+ "DISABLE");
+ if (GNUNET_YES == disable_cache)
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
"Namecache disabled\n");
vpn_handle = GNUNET_VPN_connect (cfg);
}
diff --git a/src/identity-provider/gnunet-service-identity-provider.c
b/src/identity-provider/gnunet-service-identity-provider.c
index 572202c3f..6afb7bb05 100644
--- a/src/identity-provider/gnunet-service-identity-provider.c
+++ b/src/identity-provider/gnunet-service-identity-provider.c
@@ -2190,12 +2190,15 @@ attr_iter_cb (void *cls,
if (rd_count != 1)
{
- GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+ GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+ 1);
return;
}
- if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type) {
- GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+ if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type)
+ {
+ GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+ 1);
return;
}
attr_ver = ntohl(*((uint32_t*)rd->data));
@@ -2209,8 +2212,10 @@ attr_iter_cb (void *cls,
rd->data_size-sizeof (uint32_t),
key,
(void**)&attr_ser);
- if (GNUNET_SYSERR == msg_extra_len) {
- GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+ if (GNUNET_SYSERR == msg_extra_len)
+ {
+ GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+ 1);
return;
}
@@ -2253,13 +2258,15 @@ iterate_after_abe_bootstrap (void *cls,
ai);
}
-void
+
+static void
iterate_next_after_abe_bootstrap (void *cls,
struct GNUNET_ABE_AbeMasterKey *abe_key)
{
struct AttributeIterator *ai = cls;
ai->abe_key = abe_key;
- GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+ GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+ 1);
}
diff --git a/src/identity-provider/plugin_rest_openid_connect.c
b/src/identity-provider/plugin_rest_openid_connect.c
index 7e8054e60..125bb98f6 100644
--- a/src/identity-provider/plugin_rest_openid_connect.c
+++ b/src/identity-provider/plugin_rest_openid_connect.c
@@ -1076,16 +1076,19 @@ namestore_iteration_callback (
}
}
- GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it);
+ GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it,
+ 1);
}
+
/**
* Iteration over all results finished, build final
* response.
*
* @param cls the `struct RequestHandle`
*/
-static void namestore_iteration_finished (void *cls)
+static void
+namestore_iteration_finished (void *cls)
{
struct RequestHandle *handle = cls;
struct GNUNET_HashCode cache_key;
diff --git a/src/namecache/namecache.conf.in b/src/namecache/namecache.conf.in
index 236cafecd..becd34187 100644
--- a/src/namecache/namecache.conf.in
+++ b/src/namecache/namecache.conf.in
@@ -11,6 +11,11 @@ ACCEPT_FROM = 127.0.0.1;
ACCEPT_FROM6 = ::1;
DATABASE = sqlite
+# Disables use of caching by GNS. Useful for systems that
+# publish very large zones and are CPU bound, if they do not
+# also do a large number of lookups.
+DISABLE = NO
+
[namecache-sqlite]
FILENAME = $GNUNET_DATA_HOME/namecache/sqlite.db
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index 06a50132b..b9061ed9f 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -257,6 +257,12 @@ static struct GNUNET_NotificationContext *monitor_nc;
*/
static int cache_keys;
+/**
+ * Use the namecache? Doing so creates additional cryptographic
+ * operations whenever we touch a record.
+ */
+static int disable_namecache;
+
/**
* Task run during shutdown.
@@ -281,8 +287,11 @@ cleanup_task (void *cls)
cop);
GNUNET_free (cop);
}
- GNUNET_NAMECACHE_disconnect (namecache);
- namecache = NULL;
+ if (NULL != namecache)
+ {
+ GNUNET_NAMECACHE_disconnect (namecache);
+ namecache = NULL;
+ }
GNUNET_break (NULL == GNUNET_PLUGIN_unload (db_lib_name,
GSN_database));
GNUNET_free (db_lib_name);
@@ -715,6 +724,14 @@ refresh_block (struct NamestoreClient *nc,
rid);
return; /* no data, no need to update cache */
}
+ if (GNUNET_YES == disable_namecache)
+ {
+ GNUNET_STATISTICS_update (statistics,
+ "Namecache updates skipped (NC disabled)",
+ 1,
+ GNUNET_NO);
+ return;
+ }
exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
res);
if (cache_keys)
@@ -1692,13 +1709,20 @@ run (void *cls,
(void) service;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Starting namestore service\n");
- GSN_cfg = cfg;
- monitor_nc = GNUNET_notification_context_create (1);
- namecache = GNUNET_NAMECACHE_connect (cfg);
- /* Loading database plugin */
cache_keys = GNUNET_CONFIGURATION_get_value_yesno (cfg,
"namestore",
"CACHE_KEYS");
+ disable_namecache = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+ "namecache",
+ "DISABLE");
+ GSN_cfg = cfg;
+ monitor_nc = GNUNET_notification_context_create (1);
+ if (GNUNET_NO == disable_namecache)
+ {
+ namecache = GNUNET_NAMECACHE_connect (cfg);
+ GNUNET_assert (NULL != namecache);
+ }
+ /* Loading database plugin */
if (GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_string (cfg,
"namestore",
diff --git a/src/namestore/gnunet-zoneimport.c
b/src/namestore/gnunet-zoneimport.c
index 503262487..55779e06a 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -951,10 +951,10 @@ store_completed_cb (void *cls,
delta = GNUNET_TIME_absolute_get_duration (last);
last = GNUNET_TIME_absolute_get ();
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Processed 1000 records in %s\n",
- GNUNET_STRINGS_relative_time_to_string (delta,
- GNUNET_YES));
+ fprintf (stderr,
+ "Processed 1000 records in %s\n",
+ GNUNET_STRINGS_relative_time_to_string (delta,
+ GNUNET_YES));
}
}
free_records (req);
@@ -1590,10 +1590,10 @@ process_stdin (void *cls)
delta = GNUNET_TIME_absolute_get_duration (last);
last = GNUNET_TIME_absolute_get ();
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Imported 1000 records in %s\n",
- GNUNET_STRINGS_relative_time_to_string (delta,
- GNUNET_YES));
+ fprintf (stderr,
+ "Imported 1000 records in %s\n",
+ GNUNET_STRINGS_relative_time_to_string (delta,
+ GNUNET_YES));
}
queue (hn);
}
diff --git a/src/namestore/plugin_namestore_flat.c
b/src/namestore/plugin_namestore_flat.c
index e9ed1cc4f..f061ab7d1 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -583,7 +583,7 @@ iterate_zones (void *cls,
(void) key;
if (0 == ic->limit)
return GNUNET_NO;
- if ( (NULL != it->zone) &&
+ if ( (NULL != ic->zone) &&
(0 != memcmp (entry->private_key,
ic->zone,
sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) )
diff --git a/src/zonemaster/gnunet-service-zonemaster.c
b/src/zonemaster/gnunet-service-zonemaster.c
index 518d5f572..25baf4396 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -272,6 +272,14 @@ publish_zone_dht_start (void *cls);
/**
+ * How often do we measure the delta between desired zone
+ * iteration speed and actual speed, and tell statistics
+ * service about it?
+ */
+#define DELTA_INTERVAL 100
+
+
+/**
* Continuation called from DHT once the PUT operation is done.
*
* @param cls closure, NULL if called from regular iteration,
@@ -283,7 +291,11 @@ dht_put_continuation (void *cls,
int success)
{
struct MonitorActivity *ma = cls;
+ static unsigned long long put_cnt;
+ static struct GNUNET_TIME_Absolute last_put_100;
+ static struct GNUNET_TIME_Relative sub_delta;
struct GNUNET_TIME_Relative next_put_interval;
+ struct GNUNET_TIME_Relative delay;
num_public_records++;
if (NULL == ma)
@@ -300,20 +312,91 @@ dht_put_continuation (void *cls,
LATE_ITERATION_SPEEDUP_FACTOR);
}
else
+ {
next_put_interval = put_interval;
+ }
next_put_interval = GNUNET_TIME_relative_min (next_put_interval,
MAXIMUM_ZONE_ITERATION_INTERVAL);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"PUT complete, next PUT in %s!\n",
GNUNET_STRINGS_relative_time_to_string (next_put_interval,
GNUNET_YES));
-
- GNUNET_STATISTICS_set (statistics,
- "Current zone iteration interval (ms)",
- next_put_interval.rel_value_us / 1000LL,
- GNUNET_NO);
+ /* compute velocities and delay corrections to apply */
+ if (0 == put_cnt)
+ last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
+ put_cnt++;
+ if (0 == put_cnt % DELTA_INTERVAL)
+ {
+ struct GNUNET_TIME_Relative delta;
+ unsigned long long pct = 0;
+
+ /* How fast were we really? */
+ delta = GNUNET_TIME_absolute_get_duration (last_put_100);
+ delta.rel_value_us /= DELTA_INTERVAL;
+ last_put_100 = GNUNET_TIME_absolute_get ();
+ /* Tell statistics actual vs. desired speed */
+ GNUNET_STATISTICS_set (statistics,
+ "Target zone iteration velocity (μs)",
+ next_put_interval.rel_value_us,
+ GNUNET_NO);
+ GNUNET_STATISTICS_set (statistics,
+ "Current zone iteration velocity (μs)",
+ delta.rel_value_us,
+ GNUNET_NO);
+ /* update "sub_delta" based on difference, taking
+ previous sub_delta into account! */
+ if (next_put_interval.rel_value_us > delta.rel_value_us)
+ {
+ /* We were too fast, reduce sub_delta! */
+ struct GNUNET_TIME_Relative corr;
+
+ corr = GNUNET_TIME_relative_subtract (next_put_interval,
+ delta);
+ if (sub_delta.rel_value_us > delta.rel_value_us)
+ {
+ /* Reduce sub_delta by corr */
+ sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
+ corr);
+ }
+ else
+ {
+ /* We're doing fine with waiting the full time, this
+ should theoretically only happen if we run at
+ infinite speed. */
+ sub_delta = GNUNET_TIME_UNIT_ZERO;
+ }
+ }
+ else if (next_put_interval.rel_value_us < delta.rel_value_us)
+ {
+ /* We were too slow, increase sub_delta! */
+ struct GNUNET_TIME_Relative corr;
+
+ corr = GNUNET_TIME_relative_subtract (delta,
+ next_put_interval);
+ sub_delta = GNUNET_TIME_relative_add (sub_delta,
+ corr);
+ if (sub_delta.rel_value_us > next_put_interval.rel_value_us)
+ {
+ /* CPU overload detected, we cannot go at desired speed,
+ as this would mean using a negative delay. */
+ sub_delta = next_put_interval;
+ /* compute how much faster we would want to be for
+ the desired velocity */
+ if (0 == next_put_interval.rel_value_us)
+ pct = UINT64_MAX; /* desired speed is infinity ... */
+ else
+ pct = sub_delta.rel_value_us * 100 / next_put_interval.rel_value_us;
+ }
+ }
+ GNUNET_STATISTICS_set (statistics,
+ "% speed increase needed for target velocity",
+ pct,
+ GNUNET_NO);
+ } /* end of periodic velocity calculations */
+ delay = GNUNET_TIME_relative_subtract (next_put_interval,
+ sub_delta);
GNUNET_assert (NULL == zone_publish_task);
- zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval,
+ zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
&publish_zone_dht_next,
NULL);
}
--
To stop receiving notification emails like this one, please contact
address@hidden
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] [gnunet] branch master updated: add option to disable namecache, add velocity calculation and delay correction to zonemaster, fix some ftbfs from recent NAMESTORE API change,
gnunet <=