[GNUnet-SVN] r8948 - gnunet/src/fs

[gnunet]

Author: grothoff
Date: 2009-09-05 14:40:29 -0600 (Sat, 05 Sep 2009)
New Revision: 8948

Modified:
   gnunet/src/fs/fs.h
   gnunet/src/fs/fs_download.c
Log:
towards having download

Modified: gnunet/src/fs/fs.h
===================================================================
--- gnunet/src/fs/fs.h  2009-09-05 20:38:19 UTC (rev 8947)
+++ gnunet/src/fs/fs.h  2009-09-05 20:40:29 UTC (rev 8948)
@@ -337,12 +337,12 @@
   /**
    * Desired anonymity level.
    */
-  unsigned int anonymity;
+  uint32_t anonymity;
 
   /**
    * Desired priority (for keeping the content in the DB).
    */
-  unsigned int priority;
+  uint32_t priority;
 
 };
 
@@ -428,12 +428,6 @@
   char *nuid;
 
   /**
-   * ID of the task performing the upload. NO_TASK
-   * if the upload has completed.
-   */
-  GNUNET_SCHEDULER_TaskIdentifier upload_task;
-
-  /**
    * Our own client handle for the FS service;
    * only briefly used when we start to index a
    * file, otherwise NULL.
@@ -441,6 +435,23 @@
   struct GNUNET_CLIENT_Connection *client;
 
   /**
+   * Current position in the file-tree for the
+   * upload.
+   */
+  struct GNUNET_FS_FileInformation *fi_pos;
+
+  /**
+   * Connection to the datastore service.
+   */
+  struct GNUNET_DATASTORE_Handle *dsh;
+
+  /**
+   * ID of the task performing the upload. NO_TASK
+   * if the upload has completed.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier upload_task;
+
+  /**
    * Typically GNUNET_NO.  Set to GNUNET_YES if
    * "upload_task" is GNUNET_SCHEDULER_NO_TASK
    * and we're waiting for a response from the
@@ -461,17 +472,6 @@
   enum GNUNET_FS_PublishOptions options;
 
   /**
-   * Current position in the file-tree for the
-   * upload.
-   */
-  struct GNUNET_FS_FileInformation *fi_pos;
-
-  /**
-   * Connection to the datastore service.
-   */
-  struct GNUNET_DATASTORE_Handle *dsh;
-
-  /**
    * Space reservation ID with datastore service
    * for this upload.
    */
@@ -765,10 +765,127 @@
 
 
 /**
+ * Information about an active download request.
+ */ 
+struct DownloadRequest
+{
+  /**
+   * While pending, we keep all download requests
+   * in a linked list.
+   */
+  struct DownloadRequest *next;
+
+  /**
+   * CHK for the request.
+   */
+  struct ContentHashKey chk;
+
+  /**
+   * Offset of the corresponding block.
+   */
+  uint64_t offset;
+
+  /**
+   * Depth of the corresponding block in the tree.
+   */
+  unsigned int depth;
+
+  /**
+   * Set if this request is currently in the linked list of pending
+   * requests.  Needed in case we get a response for a request that we
+   * have not yet send (due to FS bug or two blocks with identical
+   * content); in this case, we would need to remove the block from
+   * the pending list (and need a fast way to check if the block is on
+   * it).
+   */
+  int is_pending;
+
+};
+
+
+/**
  * Context for controlling a download.
  */
 struct GNUNET_FS_DownloadContext
 {
+  
+  /**
+   * Global FS context.
+   */ 
+  struct GNUNET_FS_Handle *h;
+  
+  /**
+   * Connection to the FS service.
+   */
+  struct GNUNET_CLIENT_Connection *client;
+
+  /**
+   * Parent download (used when downloading files
+   * in directories).
+   */
+  struct GNUNET_FS_DownloadContext *parent;
+
+  /**
+   * Context kept for the client.
+   */
+  void *client_info;
+
+  /**
+   * URI that identifies the file that
+   * we are downloading.
+   */
+  struct GNUNET_FS_Uri *uri;
+
+  /**
+   * Where are we writing the data (name of the
+   * file, can be NULL!).
+   */
+  char *filename;
+
+  /**
+   * Map of active requests (those waiting
+   * for a response).  The key is the hash
+   * of the encryped block (aka query).
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *active;
+
+  /**
+   * Linked list of pending requests.
+   */
+  struct DownloadRequest *pending;
+
+  /**
+   * ID of a task that is using this struct
+   * and that must be cancelled when the download
+   * is being stopped (if not GNUNET_SCHEDULER_NO_TASK).
+   * Used for the task that adds some artificial
+   * delay when trying to reconnect to the FS
+   * service.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier task;
+
+  /**
+   * What is the first offset that we're interested
+   * in?
+   */
+  uint64_t offset;
+
+  /**
+   * How many bytes starting from offset are desired?
+   * This is NOT the overall length of the file!
+   */
+  uint64_t length;
+
+  /**
+   * Desired level of anonymity.
+   */
+  uint32_t anonymity;
+
+  /**
+   * Options for the download.
+   */
+  enum GNUNET_FS_DownloadOptions options;
+
 };
 
 struct GNUNET_FS_Namespace

Modified: gnunet/src/fs/fs_download.c
===================================================================
--- gnunet/src/fs/fs_download.c 2009-09-05 20:38:19 UTC (rev 8947)
+++ gnunet/src/fs/fs_download.c 2009-09-05 20:40:29 UTC (rev 8948)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008 Christian Grothoff (and 
other contributing authors)
+     (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff 
(and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,19 +19,290 @@
 */
 /**
  * @file fs/fs_download.c
- * @brief DOWNLOAD helper methods (which do the real work).
+ * @brief download methods
  * @author Christian Grothoff
+ *
+ * TODO:
+ * - process replies
+ * - callback signaling
+ * - location URI suppport (can wait)
+ * - persistence (can wait)
  */
-
 #include "platform.h"
+#include "gnunet_constants.h"
 #include "gnunet_fs_service.h"
 #include "fs.h"
 
 #define DEBUG_DOWNLOAD GNUNET_YES
 
 
+/**
+ * Schedule the download of the specified
+ * block in the tree.
+ *
+ * @param dc overall download this block belongs to
+ * @param chk content-hash-key of the block
+ * @param offset offset of the block in the file
+ *         (for IBlocks, the offset is the lowest
+ *          offset of any DBlock in the subtree under
+ *          the IBlock)
+ * @param depth depth of the block, 0 is the root of the tree
+ */
+static void
+schedule_block_download (struct GNUNET_FS_DownloadContext *dc,
+                        const struct ContentHashKey *chk,
+                        uint64_t offset,
+                        unsigned int depth)
+{
+  struct DownloadRequest *sm;
 
+  sm = GNUNET_malloc (sizeof (struct DownloadRequest));
+  sm->chk = *chk;
+  sm->offset = offset;
+  sm->depth = depth;
+  sm->is_pending = GNUNET_YES;
+  sm->next = dc->pending;
+  dc->pending = sm;
+  GNUNET_CONTAINER_multihashmap_put (dc->active,
+                                    &chk->query,
+                                    sm,
+                                    
GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+}
+
+
 /**
+ * We've lost our connection with the FS service.
+ * Re-establish it and re-transmit all of our
+ * pending requests.
+ *
+ * @param dc download context that is having trouble
+ */
+static void
+try_reconnect (struct GNUNET_FS_DownloadContext *dc);
+
+
+/**
+ * Process a search result.
+ *
+ * @param sc our search context
+ * @param type type of the result
+ * @param data the (encrypted) response
+ * @param size size of data
+ */
+static void
+process_result (struct GNUNET_FS_DownloadContext *dc,
+               uint32_t type,
+               const void *data,
+               size_t size)
+{
+  GNUNET_HashCode query;
+  struct DownloadRequest *sm;
+  struct GNUNET_CRYPTO_AesSessionKey skey;
+  struct GNUNET_CRYPTO_AesInitializationVector iv;
+  char pt[size];
+
+  GNUNET_CRYPTO_hash (data, size, &query);
+  sm = GNUNET_CONTAINER_multihashmap_get (dc->active,
+                                         &query);
+  if (NULL == sm)
+    {
+      GNUNET_break (0);
+      return;
+    }
+  GNUNET_assert (GNUNET_YES ==
+                GNUNET_CONTAINER_multihashmap_remove (dc->active,
+                                                      &query,
+                                                      sm));
+  GNUNET_CRYPTO_hash_to_aes_key (&sm->chk.key, &skey, &iv);
+  GNUNET_CRYPTO_aes_decrypt (data,
+                            size,
+                            &skey,
+                            &iv,
+                            pt);
+  // FIXME: save to disk
+  // FIXME: make persistent
+  // FIXME: call progress callback
+  // FIXME: trigger next block (if applicable)  
+}
+
+
+/**
+ * Type of a function to call when we receive a message
+ * from the service.
+ *
+ * @param cls closure
+ * @param msg message received, NULL on timeout or fatal error
+ */
+static void 
+receive_results (void *cls,
+                const struct GNUNET_MessageHeader * msg)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  const struct ContentMessage *cm;
+  uint16_t msize;
+
+  if ( (NULL == msg) ||
+       (ntohs (msg->type) != GNUNET_MESSAGE_TYPE_FS_CONTENT) ||
+       (ntohs (msg->size) <= sizeof (struct ContentMessage)) )
+    {
+      try_reconnect (dc);
+      return;
+    }
+  msize = ntohs (msg->size);
+  cm = (const struct ContentMessage*) msg;
+  process_result (dc, 
+                 ntohl (cm->type),
+                 &cm[1],
+                 msize - sizeof (struct ContentMessage));
+  /* continue receiving */
+  GNUNET_CLIENT_receive (dc->client,
+                        &receive_results,
+                        dc,
+                        GNUNET_TIME_UNIT_FOREVER_REL);
+}
+
+
+
+/**
+ * We're ready to transmit a search request to the
+ * file-sharing service.  Do it.  If there is 
+ * more than one request pending, try to send 
+ * multiple or request another transmission.
+ *
+ * @param cls closure
+ * @param size number of bytes available in buf
+ * @param buf where the callee should write the message
+ * @return number of bytes written to buf
+ */
+static size_t
+transmit_download_request (void *cls,
+                          size_t size, 
+                          void *buf)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  size_t msize;
+  struct SearchMessage *sm;
+
+  if (NULL == buf)
+    {
+      try_reconnect (dc);
+      return 0;
+    }
+  GNUNET_assert (size >= sizeof (struct SearchMessage));
+  msize = 0;
+  sm = buf;
+  while ( (dc->pending == NULL) &&
+         (size > msize + sizeof (struct SearchMessage)) )
+    {
+      memset (sm, 0, sizeof (struct SearchMessage));
+      sm->header.size = htons (sizeof (struct SearchMessage));
+      sm->header.type = htons (GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
+      sm->anonymity_level = htonl (dc->anonymity);
+      // FIXME: support 'loc' URIs (set sm->target) 
+      sm->query = dc->pending->chk.query;
+      dc->pending->is_pending = GNUNET_NO;
+      dc->pending = dc->pending->next;
+      msize += sizeof (struct SearchMessage);
+      sm++;
+    }
+  return msize;
+}
+
+
+/**
+ * Reconnect to the FS service and transmit
+ * our queries NOW.
+ *
+ * @param cls our download context
+ * @param tc unused
+ */
+static void
+do_reconnect (void *cls,
+             const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct GNUNET_CLIENT_Connection *client;
+  
+  dc->task = GNUNET_SCHEDULER_NO_TASK;
+  client = GNUNET_CLIENT_connect (dc->h->sched,
+                                 "fs",
+                                 dc->h->cfg);
+  if (NULL == client)
+    {
+      try_reconnect (dc);
+      return;
+    }
+  dc->client = client;
+  GNUNET_CLIENT_notify_transmit_ready (client,
+                                      sizeof (struct SearchMessage),
+                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+                                      &transmit_download_request,
+                                      dc);  
+  GNUNET_CLIENT_receive (client,
+                        &receive_results,
+                        dc,
+                        GNUNET_TIME_UNIT_FOREVER_REL);
+}
+
+
+/**
+ * Add entries that are not yet pending back to
+ * the pending list.
+ *
+ * @param cls our download context
+ * @param key unused
+ * @param entry entry of type "struct DownloadRequest"
+ * @return GNUNET_OK
+ */
+static int
+retry_entry (void *cls,
+            const GNUNET_HashCode *key,
+            void *entry)
+{
+  struct GNUNET_FS_DownloadContext *dc = cls;
+  struct DownloadRequest *dr = entry;
+
+  if (! dr->is_pending)
+    {
+      dr->next = dc->pending;
+      dr->is_pending = GNUNET_YES;
+      dc->pending = entry;
+    }
+  return GNUNET_OK;
+}
+
+
+/**
+ * We've lost our connection with the FS service.
+ * Re-establish it and re-transmit all of our
+ * pending requests.
+ *
+ * @param dc download context that is having trouble
+ */
+static void
+try_reconnect (struct GNUNET_FS_DownloadContext *dc)
+{
+  
+  if (NULL != dc->client)
+    {
+      GNUNET_CONTAINER_multihashmap_iterate (dc->active,
+                                            &retry_entry,
+                                            dc);
+      GNUNET_CLIENT_disconnect (dc->client);
+      dc->client = NULL;
+    }
+  dc->task
+    = GNUNET_SCHEDULER_add_delayed (dc->h->sched,
+                                   GNUNET_NO,
+                                   GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                   GNUNET_SCHEDULER_NO_TASK,
+                                   GNUNET_TIME_UNIT_SECONDS,
+                                   &do_reconnect,
+                                   dc);
+}
+
+
+/**
  * Download parts of a file.  Note that this will store
  * the blocks at the respective offset in the given file.  Also, the
  * download is still using the blocking of the underlying FS
@@ -50,9 +321,7 @@
  * @param offset at what offset should we start the download (typically 0)
  * @param length how many bytes should be downloaded starting at offset
  * @param anonymity anonymity level to use for the download
- * @param no_temporaries set to GNUNET_YES to disallow generation of temporary 
files
- * @param recursive should this be a recursive download (useful for directories
- *        to automatically trigger download of files in the directories)
+ * @param options various options
  * @param parent parent download to associate this download with (use NULL
  *        for top-level downloads; useful for manually-triggered recursive 
downloads)
  * @return context that can be used to control this download
@@ -61,31 +330,120 @@
 GNUNET_FS_file_download_start (struct GNUNET_FS_Handle *h,
                               const struct GNUNET_FS_Uri *uri,
                               const char *filename,
-                              unsigned long long offset,
-                              unsigned long long length,
-                              unsigned int anonymity,
-                              int no_temporaries,      
-                              int recursive,
+                              uint64_t offset,
+                              uint64_t length,
+                              uint32_t anonymity,
+                              enum GNUNET_FS_DownloadOptions options,
                               struct GNUNET_FS_DownloadContext *parent)
 {
-  return NULL;
+  struct GNUNET_FS_DownloadContext *dc;
+  struct GNUNET_CLIENT_Connection *client;
+
+  client = GNUNET_CLIENT_connect (h->sched,
+                                 "fs",
+                                 h->cfg);
+  if (NULL == client)
+    return NULL;
+  // FIXME: add support for "loc" URIs!
+  GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
+  if ( (dc->offset + dc->length < dc->offset) ||
+       (dc->offset + dc->length > uri->data.chk.file_length) )
+    {
+      GNUNET_break (0);
+      return NULL;
+    }
+  dc = GNUNET_malloc (sizeof(struct GNUNET_FS_DownloadContext));
+  dc->h = h;
+  dc->client = client;
+  dc->parent = parent;
+  dc->uri = GNUNET_FS_uri_dup (uri);
+  dc->filename = (NULL == filename) ? NULL : GNUNET_strdup (filename);
+  dc->offset = offset;
+  dc->length = length;
+  dc->anonymity = anonymity;
+  dc->options = options;
+  dc->active = GNUNET_CONTAINER_multihashmap_create (1 + (length / 
DBLOCK_SIZE));
+  // FIXME: make persistent
+  schedule_block_download (dc, 
+                          &dc->uri->data.chk.chk,
+                          0, 
+                          0);
+  GNUNET_CLIENT_notify_transmit_ready (client,
+                                      sizeof (struct SearchMessage),
+                                       GNUNET_CONSTANTS_SERVICE_TIMEOUT,
+                                      &transmit_download_request,
+                                      dc);  
+  GNUNET_CLIENT_receive (client,
+                        &receive_results,
+                        dc,
+                        GNUNET_TIME_UNIT_FOREVER_REL);
+  // FIXME: signal download start
+  return dc;
 }
 
+
 /**
+ * Free entries in the map.
+ *
+ * @param cls unused (NULL)
+ * @param key unused
+ * @param entry entry of type "struct DownloadRequest" which is freed
+ * @return GNUNET_OK
+ */
+static int
+free_entry (void *cls,
+           const GNUNET_HashCode *key,
+           void *entry)
+{
+  GNUNET_free (entry);
+  return GNUNET_OK;
+}
+
+
+/**
  * Stop a download (aborts if download is incomplete).
  *
- * @param rm handle for the download
+ * @param dc handle for the download
  * @param do_delete delete files of incomplete downloads
  */
 void
-GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *rm,
+GNUNET_FS_file_download_stop (struct GNUNET_FS_DownloadContext *dc,
                              int do_delete)
 {
+  // FIXME: make unpersistent
+  // FIXME: signal download end
+  
+  if (GNUNET_SCHEDULER_NO_TASK != dc->task)
+    GNUNET_SCHEDULER_cancel (dc->h->sched,
+                            dc->task);
+  if (NULL != dc->client)
+    GNUNET_CLIENT_disconnect (dc->client);
+  GNUNET_CONTAINER_multihashmap_iterate (dc->active,
+                                        &free_entry,
+                                        NULL);
+  GNUNET_CONTAINER_multihashmap_destroy (dc->active);
+  GNUNET_FS_uri_destroy (dc->uri);
+  GNUNET_free_non_null (dc->filename);
+  GNUNET_free (dc);
 }
 
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 #if 0
 
 /**