[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r34769 - gnunet/src/datastore
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r34769 - gnunet/src/datastore |
|
Date: |
Tue, 23 Dec 2014 21:37:23 +0100 |
Author: grothoff
Date: 2014-12-23 21:37:22 +0100 (Tue, 23 Dec 2014)
New Revision: 34769
Modified:
gnunet/src/datastore/gnunet-service-datastore.c
Log:
fixing datastore-statistics interaction to ensure stats are properly written to
disk on exit
Modified: gnunet/src/datastore/gnunet-service-datastore.c
===================================================================
--- gnunet/src/datastore/gnunet-service-datastore.c 2014-12-23 05:07:02 UTC
(rev 34768)
+++ gnunet/src/datastore/gnunet-service-datastore.c 2014-12-23 20:37:22 UTC
(rev 34769)
@@ -1,6 +1,6 @@
/*
This file is part of GNUnet
- (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other
contributing authors)
+ (C) 2004-2014 Christian Grothoff (and other contributing authors)
GNUnet is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published
@@ -142,19 +142,19 @@
static struct GNUNET_CONTAINER_BloomFilter *filter;
/**
- * How much space are we allowed to use?
+ * Name of our plugin.
*/
-static unsigned long long quota;
+static char *plugin_name;
/**
- * Should the database be dropped on exit?
+ * Our configuration.
*/
-static int do_drop;
+static const struct GNUNET_CONFIGURATION_Handle *cfg;
/**
- * Name of our plugin.
+ * Handle for reporting statistics.
*/
-static char *plugin_name;
+static struct GNUNET_STATISTICS_Handle *stats;
/**
* How much space are we using for the cache? (space available for
@@ -176,42 +176,47 @@
static unsigned long long payload;
/**
- * Number of updates that were made to the
- * payload value since we last synchronized
- * it with the statistics service.
+ * Identity of the task that is used to delete
+ * expired content.
*/
-static unsigned int lastSync;
+static GNUNET_SCHEDULER_TaskIdentifier expired_kill_task;
/**
- * Did we get an answer from statistics?
+ * Minimum time that content should have to not be discarded instantly
+ * (time stamp of any content that we've been discarding recently to
+ * stay below the quota). FOREVER if we had to expire content with
+ * non-zero priority.
*/
-static int stats_worked;
+static struct GNUNET_TIME_Absolute min_expiration;
/**
- * Identity of the task that is used to delete
- * expired content.
+ * How much space are we allowed to use?
*/
-static GNUNET_SCHEDULER_TaskIdentifier expired_kill_task;
+static unsigned long long quota;
/**
- * Our configuration.
+ * Should the database be dropped on exit?
*/
-const struct GNUNET_CONFIGURATION_Handle *cfg;
+static int do_drop;
/**
- * Minimum time that content should have to not be discarded instantly
- * (time stamp of any content that we've been discarding recently to
- * stay below the quota). FOREVER if we had to expire content with
- * non-zero priority.
+ * Should we refresh the BF when the DB is loaded?
*/
-static struct GNUNET_TIME_Absolute min_expiration;
+static int refresh_bf;
/**
- * Handle for reporting statistics.
+ * Number of updates that were made to the
+ * payload value since we last synchronized
+ * it with the statistics service.
*/
-static struct GNUNET_STATISTICS_Handle *stats;
+static unsigned int last_sync;
+/**
+ * Did we get an answer from statistics?
+ */
+static int stats_worked;
+
/**
* Synchronize our utilization statistics with the
* statistics service.
@@ -221,11 +226,10 @@
{
GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
GNUNET_STATISTICS_set (stats, "# utilization by current datastore", payload,
GNUNET_NO);
- lastSync = 0;
+ last_sync = 0;
}
-
/**
* Context for transmitting replies to clients.
*/
@@ -292,7 +296,8 @@
* @param tc task context
*/
static void
-delete_expired (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+delete_expired (void *cls,
+ const struct GNUNET_SCHEDULER_TaskContext *tc);
/**
@@ -311,15 +316,20 @@
* @param uid unique identifier for the datum;
* maybe 0 if no unique identifier is available
*
- * @return GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue
+ * @return #GNUNET_SYSERR to abort the iteration, #GNUNET_OK to continue
* (continue on call to "next", of course),
- * GNUNET_NO to delete the item and continue (if supported)
+ * #GNUNET_NO to delete the item and continue (if supported)
*/
static int
-expired_processor (void *cls, const struct GNUNET_HashCode * key, uint32_t
size,
- const void *data, enum GNUNET_BLOCK_Type type,
- uint32_t priority, uint32_t anonymity,
- struct GNUNET_TIME_Absolute expiration, uint64_t uid)
+expired_processor (void *cls,
+ const struct GNUNET_HashCode *key,
+ uint32_t size,
+ const void *data,
+ enum GNUNET_BLOCK_Type type,
+ uint32_t priority,
+ uint32_t anonymity,
+ struct GNUNET_TIME_Absolute expiration,
+ uint64_t uid)
{
struct GNUNET_TIME_Absolute now;
@@ -564,7 +574,6 @@
}
-
/**
* Function that will transmit the given datastore entry
* to the client.
@@ -719,7 +728,8 @@
* @param message the actual message
*/
static void
-handle_release_reserve (void *cls, struct GNUNET_SERVER_Client *client,
+handle_release_reserve (void *cls,
+ struct GNUNET_SERVER_Client *client,
const struct GNUNET_MessageHeader *message)
{
const struct ReleaseReserveMessage *msg =
@@ -730,7 +740,8 @@
int rid = ntohl (msg->rid);
unsigned long long rem;
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n",
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Processing `%s' request\n",
"RELEASE_RESERVE");
next = reservations;
prev = NULL;
@@ -748,7 +759,8 @@
((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) *
pos->entries;
GNUNET_assert (reserved >= rem);
reserved -= rem;
- GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved,
+ GNUNET_STATISTICS_set (stats,
+ gettext_noop ("# reserved"), reserved,
GNUNET_NO);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Returning %llu remaining reserved bytes to storage pool\n",
@@ -835,7 +847,8 @@
GNUNET_TIME_absolute_ntoh (dm->expiration), &msg);
if (GNUNET_OK == ret)
{
- GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes stored"), size,
+ GNUNET_STATISTICS_update (stats,
+ gettext_noop ("# bytes stored"), size,
GNUNET_YES);
GNUNET_CONTAINER_bloomfilter_add (filter, &dm->key);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -961,7 +974,8 @@
pos->entries--;
pos->amount -= size;
reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
- GNUNET_STATISTICS_set (stats, gettext_noop ("# reserved"), reserved,
+ GNUNET_STATISTICS_set (stats,
+ gettext_noop ("# reserved"), reserved,
GNUNET_NO);
}
}
@@ -1007,7 +1021,8 @@
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Processing `%s' request for `%s' of type %u\n", "GET",
GNUNET_h2s (&msg->key), ntohl (msg->type));
- GNUNET_STATISTICS_update (stats, gettext_noop ("# GET requests received"), 1,
+ GNUNET_STATISTICS_update (stats,
+ gettext_noop ("# GET requests received"), 1,
GNUNET_NO);
GNUNET_SERVER_client_keep (client);
if ((size == sizeof (struct GetMessage)) &&
@@ -1196,7 +1211,9 @@
handle_drop (void *cls, struct GNUNET_SERVER_Client *client,
const struct GNUNET_MessageHeader *message)
{
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing `%s' request\n", "DROP");
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Processing `%s' request\n",
+ "DROP");
do_drop = GNUNET_YES;
GNUNET_SERVER_receive_done (client, GNUNET_OK);
}
@@ -1211,21 +1228,25 @@
* 0 for "reset to empty"
*/
static void
-disk_utilization_change_cb (void *cls, int delta)
+disk_utilization_change_cb (void *cls,
+ int delta)
{
if ((delta < 0) && (payload < -delta))
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
- _
- ("Datastore payload inaccurate (%lld < %lld). Trying to
fix.\n"),
- (long long) payload, (long long) -delta);
+ _("Datastore payload must have been inaccurate (%lld < %lld).
Recomputing it.\n"),
+ (long long) payload,
+ (long long) -delta);
payload = plugin->api->estimate_size (plugin->api->cls);
- sync_stats ();
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ _("New payload: %lld\n"),
+ (long long) payload);
+ sync_stats ();
return;
}
payload += delta;
- lastSync++;
- if (lastSync >= MAX_STAT_SYNC_LAG)
+ last_sync++;
+ if (last_sync >= MAX_STAT_SYNC_LAG)
sync_stats ();
}
@@ -1237,14 +1258,17 @@
* @param subsystem name of subsystem that created the statistic
* @param name the name of the datum
* @param value the current value
- * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
- * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if
not
+ * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
*/
static int
-process_stat_in (void *cls, const char *subsystem, const char *name,
- uint64_t value, int is_persistent)
+process_stat_in (void *cls,
+ const char *subsystem,
+ const char *name,
+ uint64_t value,
+ int is_persistent)
{
- GNUNET_assert (stats_worked == GNUNET_NO);
+ GNUNET_assert (GNUNET_NO == stats_worked);
stats_worked = GNUNET_YES;
payload += value;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1254,17 +1278,6 @@
}
-static void
-process_stat_done (void *cls, int success)
-{
- struct DatastorePlugin *plugin = cls;
-
- stat_get = NULL;
- if (stats_worked == GNUNET_NO)
- payload = plugin->api->estimate_size (plugin->api->cls);
-}
-
-
/**
* Load the datastore plugin.
*/
@@ -1278,16 +1291,20 @@
ret->env.cfg = cfg;
ret->env.duc = &disk_utilization_change_cb;
ret->env.cls = NULL;
- GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"),
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ _("Loading `%s' datastore plugin\n"),
plugin_name);
- GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name);
+ GNUNET_asprintf (&libname,
+ "libgnunet_plugin_datastore_%s",
+ plugin_name);
ret->short_name = GNUNET_strdup (plugin_name);
ret->lib_name = libname;
ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
- if (ret->api == NULL)
+ if (NULL == ret->api)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _("Failed to load datastore plugin for `%s'\n"), plugin_name);
+ _("Failed to load datastore plugin for `%s'\n"),
+ plugin_name);
GNUNET_free (ret->short_name);
GNUNET_free (libname);
GNUNET_free (ret);
@@ -1312,50 +1329,85 @@
GNUNET_free (plug->lib_name);
GNUNET_free (plug->short_name);
GNUNET_free (plug);
- GNUNET_free (quota_stat_name);
- quota_stat_name = NULL;
}
/**
- * Final task run after shutdown. Unloads plugins and disconnects us from
- * statistics.
+ * Adds a given @a key to the bloomfilter in @a cls @a count times.
+ *
+ * @param cls the bloomfilter
+ * @param key key to add
+ * @param count number of times to add key
*/
static void
-unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+add_key_to_bloomfilter (void *cls,
+ const struct GNUNET_HashCode *key,
+ unsigned int count)
{
- if (lastSync > 0)
- sync_stats ();
- if (GNUNET_YES == do_drop)
- plugin->api->drop (plugin->api->cls);
- unload_plugin (plugin);
- plugin = NULL;
- if (filter != NULL)
+ struct GNUNET_CONTAINER_BloomFilter *bf = cls;
+
+ while (0 < count--)
+ GNUNET_CONTAINER_bloomfilter_add (bf, key);
+}
+
+
+/**
+ * We finished receiving the statistic. Initialize the plugin; if
+ * loading the statistic failed, run the estimator.
+ *
+ * @param cls NULL
+ * @param success #GNUNET_NO if we failed to read the stat
+ */
+static void
+process_stat_done (void *cls, int success)
+{
+ stat_get = NULL;
+ plugin = load_plugin ();
+ if (NULL == plugin)
{
GNUNET_CONTAINER_bloomfilter_free (filter);
filter = NULL;
+ if (NULL != stats)
+ {
+ GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
+ stats = NULL;
+ }
+ return;
}
- if (stat_get != NULL)
+ if (GNUNET_NO == stats_worked)
{
- GNUNET_STATISTICS_get_cancel (stat_get);
- stat_get = NULL;
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Failed to obtain value from statistics service, recomputing
it\n");
+ payload = plugin->api->estimate_size (plugin->api->cls);
}
- if (stats != NULL)
+ if (GNUNET_YES == refresh_bf)
{
- GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
- stats = NULL;
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ _("Rebuilding bloomfilter. Please be patient.\n"));
+ if (NULL != plugin->api->get_keys)
+ plugin->api->get_keys (plugin->api->cls,
+ &add_key_to_bloomfilter,
+ filter);
+ else
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ _("Plugin does not support get_keys function. Please
fix!\n"));
+
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ _("Bloomfilter construction complete.\n"));
}
- GNUNET_free_non_null (plugin_name);
- plugin_name = NULL;
+ expired_kill_task
+ = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
+ &delete_expired,
+ NULL);
}
/**
- * Last task run during shutdown. Disconnects us from
- * the transport and core.
+ * Task run during shutdown.
*/
static void
-cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+cleaning_task (void *cls,
+ const struct GNUNET_SCHEDULER_TaskContext *tc)
{
struct TransmitCallbackContext *tcc;
@@ -1376,8 +1428,31 @@
GNUNET_SCHEDULER_cancel (expired_kill_task);
expired_kill_task = GNUNET_SCHEDULER_NO_TASK;
}
- GNUNET_SCHEDULER_add_continuation (&unload_task, NULL,
- GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+ if (GNUNET_YES == do_drop)
+ plugin->api->drop (plugin->api->cls);
+ unload_plugin (plugin);
+ plugin = NULL;
+ if (NULL != filter)
+ {
+ GNUNET_CONTAINER_bloomfilter_free (filter);
+ filter = NULL;
+ }
+ if (NULL != stat_get)
+ {
+ GNUNET_STATISTICS_get_cancel (stat_get);
+ stat_get = NULL;
+ }
+ GNUNET_free_non_null (plugin_name);
+ plugin_name = NULL;
+ if (last_sync > 0)
+ sync_stats ();
+ if (NULL != stats)
+ {
+ GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
+ stats = NULL;
+ }
+ GNUNET_free (quota_stat_name);
+ quota_stat_name = NULL;
}
@@ -1390,13 +1465,14 @@
* @param client identification of the client
*/
static void
-cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client)
+cleanup_reservations (void *cls,
+ struct GNUNET_SERVER_Client *client)
{
struct ReservationList *pos;
struct ReservationList *prev;
struct ReservationList *next;
- if (client == NULL)
+ if (NULL == client)
return;
prev = NULL;
pos = reservations;
@@ -1424,24 +1500,6 @@
/**
- * Adds a given key to the bloomfilter 'count' times.
- *
- * @param cls the bloomfilter
- * @param key key to add
- * @param count number of times to add key
- */
-static void
-add_key_to_bloomfilter (void *cls,
- const struct GNUNET_HashCode *key,
- unsigned int count)
-{
- struct GNUNET_CONTAINER_BloomFilter *bf = cls;
- while (0 < count--)
- GNUNET_CONTAINER_bloomfilter_add (bf, key);
-}
-
-
-/**
* Process datastore requests.
*
* @param cls closure
@@ -1449,7 +1507,8 @@
* @param c configuration to use
*/
static void
-run (void *cls, struct GNUNET_SERVER_Handle *server,
+run (void *cls,
+ struct GNUNET_SERVER_Handle *server,
const struct GNUNET_CONFIGURATION_Handle *c)
{
static const struct GNUNET_SERVER_MessageHandler handlers[] = {
@@ -1476,7 +1535,6 @@
char *fn;
char *pfn;
unsigned int bf_size;
- int refresh_bf;
cfg = c;
if (GNUNET_OK !=
@@ -1516,11 +1574,11 @@
{
GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
_("Could not use specified filename `%s' for bloomfilter.\n"),
- fn != NULL ? fn : "");
+ NULL != fn ? fn : "");
GNUNET_free_non_null (fn);
fn = NULL;
}
- if (fn != NULL)
+ if (NULL != fn)
{
GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
@@ -1576,52 +1634,31 @@
refresh_bf = GNUNET_YES;
}
GNUNET_free_non_null (fn);
- if (filter == NULL)
+ if (NULL == filter)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_("Failed to initialize bloomfilter.\n"));
- if (stats != NULL)
+ if (NULL != stats)
{
GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
stats = NULL;
}
return;
}
- plugin = load_plugin ();
- if (NULL == plugin)
- {
- GNUNET_CONTAINER_bloomfilter_free (filter);
- filter = NULL;
- if (stats != NULL)
- {
- GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
- stats = NULL;
- }
- return;
- }
stat_get =
- GNUNET_STATISTICS_get (stats, "datastore", quota_stat_name,
- GNUNET_TIME_UNIT_SECONDS, &process_stat_done,
- &process_stat_in, plugin);
- GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
+ GNUNET_STATISTICS_get (stats,
+ "datastore",
+ quota_stat_name,
+ GNUNET_TIME_UNIT_SECONDS,
+ &process_stat_done,
+ &process_stat_in,
+ NULL);
+ GNUNET_SERVER_disconnect_notify (server,
+ &cleanup_reservations,
+ NULL);
GNUNET_SERVER_add_handlers (server, handlers);
- if (GNUNET_YES == refresh_bf)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- _("Rebuilding bloomfilter. Please be patient.\n"));
- if (NULL != plugin->api->get_keys)
- plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter,
filter);
- else
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _("Plugin does not support get_keys function. Please
fix!\n"));
-
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- _("Bloomfilter construction complete.\n"));
- }
- expired_kill_task =
- GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
- &delete_expired, NULL);
- GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &cleaning_task,
+ GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
+ &cleaning_task,
NULL);
}
@@ -1634,13 +1671,15 @@
* @return 0 ok, 1 on error
*/
int
-main (int argc, char *const *argv)
+main (int argc,
+ char *const *argv)
{
int ret;
ret =
(GNUNET_OK ==
- GNUNET_SERVICE_run (argc, argv, "datastore", GNUNET_SERVICE_OPTION_NONE,
+ GNUNET_SERVICE_run (argc, argv, "datastore",
+ GNUNET_SERVICE_OPTION_NONE,
&run, NULL)) ? 0 : 1;
return ret;
}
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r34769 - gnunet/src/datastore,
gnunet <=