[Emacs-diffs] emacs-24 b54f572: Port memory-full checking to GnuTLS 3.3

[Paul Eggert]

branch: emacs-24
commit b54f5721bfb6bf21cac5402cf34a8130e11bfb70
Author: Paul Eggert <address@hidden>
Commit: Paul Eggert <address@hidden>

    Port memory-full checking to GnuTLS 3.3
    
    Instead of using gnutls_global_set_mem_functions, check every call
    to a GnuTLS function that might return an indication of memory
    exhaustion.  Suggested by Dmitry Antipov in:
    http://lists.gnu.org/archive/html/emacs-devel/2014-12/msg02056.html
    * src/gnutls.c (gnutls_global_set_mem_functions) [WINDOWSNT]: Remove.
    (init_gnutls_functions): Do not load gnutls_global_set_mem_functions.
    (fn_gnutls_global_set_mem_functions) [!WINDOWSNT]: Remove.
    All uses removed.
    (check_memory_full): New function.
    (emacs_gnutls_handshake, emacs_gnutls_handle_error)
    (gnutls_make_error, Fgnutls_boot): Use it.
    (emacs_gnutls_global_init): Avoid gnutls_global_set_mem_functions.
---
 src/ChangeLog |   16 ++++++++++++++++
 src/gnutls.c  |   40 +++++++++++++++++++++++++---------------
 2 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/src/ChangeLog b/src/ChangeLog
index 2df8308..3ea6057 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,19 @@
+2014-12-28  Paul Eggert  <address@hidden>
+
+       Port memory-full checking to GnuTLS 3.3
+       Instead of using gnutls_global_set_mem_functions, check every call
+       to a GnuTLS function that might return an indication of memory
+       exhaustion.  Suggested by Dmitry Antipov in:
+       http://lists.gnu.org/archive/html/emacs-devel/2014-12/msg02056.html
+       * gnutls.c (gnutls_global_set_mem_functions) [WINDOWSNT]: Remove.
+       (init_gnutls_functions): Do not load gnutls_global_set_mem_functions.
+       (fn_gnutls_global_set_mem_functions) [!WINDOWSNT]: Remove.
+       All uses removed.
+       (check_memory_full): New function.
+       (emacs_gnutls_handshake, emacs_gnutls_handle_error)
+       (gnutls_make_error, Fgnutls_boot): Use it.
+       (emacs_gnutls_global_init): Avoid gnutls_global_set_mem_functions.
+
 2014-12-25  Eli Zaretskii  <address@hidden>
 
        * xdisp.c (set_iterator_to_next) <GET_FROM_STRING>: Limit search in
diff --git a/src/gnutls.c b/src/gnutls.c
index ffa3c98..f093568 100644
--- a/src/gnutls.c
+++ b/src/gnutls.c
@@ -116,10 +116,6 @@ DEF_GNUTLS_FN (void, gnutls_global_set_log_function, 
(gnutls_log_func));
 DEF_GNUTLS_FN (void, gnutls_global_set_audit_log_function, 
(gnutls_audit_log_func));
 #endif
 DEF_GNUTLS_FN (void, gnutls_global_set_log_level, (int));
-DEF_GNUTLS_FN (void, gnutls_global_set_mem_functions,
-              (gnutls_alloc_function, gnutls_alloc_function,
-               gnutls_is_secure_function, gnutls_realloc_function,
-               gnutls_free_function));
 DEF_GNUTLS_FN (int, gnutls_handshake, (gnutls_session_t));
 DEF_GNUTLS_FN (int, gnutls_init, (gnutls_session_t *, unsigned int));
 DEF_GNUTLS_FN (int, gnutls_priority_set_direct,
@@ -184,7 +180,6 @@ init_gnutls_functions (void)
   LOAD_GNUTLS_FN (library, gnutls_global_set_audit_log_function);
 #endif
   LOAD_GNUTLS_FN (library, gnutls_global_set_log_level);
-  LOAD_GNUTLS_FN (library, gnutls_global_set_mem_functions);
   LOAD_GNUTLS_FN (library, gnutls_handshake);
   LOAD_GNUTLS_FN (library, gnutls_init);
   LOAD_GNUTLS_FN (library, gnutls_priority_set_direct);
@@ -244,7 +239,6 @@ init_gnutls_functions (void)
 #define fn_gnutls_global_set_audit_log_function        
gnutls_global_set_audit_log_function
 #endif
 #define fn_gnutls_global_set_log_level         gnutls_global_set_log_level
-#define fn_gnutls_global_set_mem_functions     gnutls_global_set_mem_functions
 #define fn_gnutls_handshake                    gnutls_handshake
 #define fn_gnutls_init                         gnutls_init
 #define fn_gnutls_priority_set_direct          gnutls_priority_set_direct
@@ -264,6 +258,17 @@ init_gnutls_functions (void)
 #endif /* !WINDOWSNT */
 
 
+/* Report memory exhaustion if ERR is an out-of-memory indication.  */
+static void
+check_memory_full (int err)
+{
+  /* When GnuTLS exhausts memory, it doesn't say how much memory it
+     asked for, so tell the Emacs allocator that GnuTLS asked for no
+     bytes.  This isn't accurate, but it's good enough.  */
+  if (err == GNUTLS_E_MEMORY_ERROR)
+    memory_full (0);
+}
+
 #ifdef HAVE_GNUTLS3
 /* Function to log a simple audit message.  */
 static void
@@ -360,7 +365,7 @@ emacs_gnutls_handshake (struct Lisp_Process *proc)
     }
   else
     {
-      fn_gnutls_alert_send_appropriate (state, ret);
+      check_memory_full (fn_gnutls_alert_send_appropriate (state, ret));
     }
   return ret;
 }
@@ -477,6 +482,8 @@ emacs_gnutls_handle_error (gnutls_session_t session, int 
err)
   if (err >= 0)
     return 1;
 
+  check_memory_full (err);
+
   max_log_level = global_gnutls_log_level;
 
   /* TODO: use gnutls-error-fatalp and gnutls-error-string.  */
@@ -542,6 +549,7 @@ gnutls_make_error (int err)
       return Qgnutls_e_invalid_session;
     }
 
+  check_memory_full (err);
   return make_number (err);
 }
 
@@ -682,11 +690,8 @@ emacs_gnutls_global_init (void)
   int ret = GNUTLS_E_SUCCESS;
 
   if (!gnutls_global_initialized)
-    {
-      fn_gnutls_global_set_mem_functions (xmalloc, xmalloc, NULL,
-                                         xrealloc, xfree);
-      ret = fn_gnutls_global_init ();
-    }
+    ret = fn_gnutls_global_init ();
+
   gnutls_global_initialized = 1;
 
   return gnutls_make_error (ret);
@@ -854,7 +859,8 @@ one trustfile (usually a CA bundle).  */)
       unsigned int gnutls_verify_flags = GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT;
 
       GNUTLS_LOG (2, max_log_level, "allocating x509 credentials");
-      fn_gnutls_certificate_allocate_credentials (&x509_cred);
+      check_memory_full ((fn_gnutls_certificate_allocate_credentials
+                         (&x509_cred)));
       XPROCESS (proc)->gnutls_x509_cred = x509_cred;
 
       verify_flags = Fplist_get (proplist, QCgnutls_bootprop_verify_flags);
@@ -873,7 +879,8 @@ one trustfile (usually a CA bundle).  */)
   else /* Qgnutls_anon: */
     {
       GNUTLS_LOG (2, max_log_level, "allocating anon credentials");
-      fn_gnutls_anon_allocate_client_credentials (&anon_cred);
+      check_memory_full ((fn_gnutls_anon_allocate_client_credentials
+                         (&anon_cred)));
       XPROCESS (proc)->gnutls_anon_cred = anon_cred;
     }
 
@@ -1105,7 +1112,10 @@ one trustfile (usually a CA bundle).  */)
          return gnutls_make_error (ret);
        }
 
-      if (!fn_gnutls_x509_crt_check_hostname (gnutls_verify_cert, c_hostname))
+      int err
+       = fn_gnutls_x509_crt_check_hostname (gnutls_verify_cert, c_hostname);
+      check_memory_full (err);
+      if (!err)
        {
           if (verify_error_all
               || !NILP (Fmember (QCgnutls_bootprop_hostname, verify_error)))