Make sure to return correct error codes for secmem failures.

ERRNO was not always set and thus it could happen that a misleading
error code was returned form a malloc functions.  Fix was to set
ERRNO.

At one place we also switched to the newer gpg_err_code_from_syserror
which makes sure to return a special error code in case ERRNO is not
set at all.

5 files changed, 32 insertions, 6 deletions

diff --git a/cipher/ChangeLog b/cipher/ChangeLog
index 2a28b204..fb051416 100644
--- a/cipher/ChangeLog
+++ b/cipher/ChangeLog
@@ -1,3 +1,8 @@
+2011-04-19  Werner Koch  <wk@g10code.com>
+
+	* cipher.c (gcry_cipher_open): Replace gpg_err_code_from_errno by
+	gpg_err_code_from_syserror.
+
 2011-04-11  Werner Koch  <wk@g10code.com>
 
 	* pubkey.c (gcry_pk_get_keygrip): Avoid double free of L2.
diff --git a/cipher/cipher.c b/cipher/cipher.c
index 90fdb173..b0a532af 100644
--- a/cipher/cipher.c
+++ b/cipher/cipher.c
@@ -786,7 +786,7 @@ gcry_cipher_open (gcry_cipher_hd_t *handle,
 	h = gcry_calloc (1, size);
 
       if (! h)
-	err = gpg_err_code_from_errno (errno);
+	err = gpg_err_code_from_syserror ();
       else
 	{
           size_t off = 0;
diff --git a/src/ChangeLog b/src/ChangeLog
index 9476e82e..2907e854 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,10 @@
+2011-04-19  Werner Koch  <wk@g10code.com>
+
+	* stdmem.c (_gcry_private_malloc_secure, _gcry_private_malloc):
+	Set ERRNO on failure.
+	* secmem.c (mb_get_new): Set ERRNO on failure.
+	(_gcry_secmem_malloc_internal): Ditto.
+
 2011-04-01  Werner Koch  <wk@g10code.com>
 
 	* global.c (gcry_realloc): Divert to gcry_malloc or gcry_free.
diff --git a/src/secmem.c b/src/secmem.c
index 30c27774..2beb234a 100644
--- a/src/secmem.c
+++ b/src/secmem.c
@@ -217,7 +217,10 @@ mb_get_new (memblock_t *block, size_t size)
       }
 
   if (! ptr_into_pool_p (mb))
-    mb = NULL;
+    {
+      gpg_err_set_errno (ENOMEM);
+      mb = NULL;
+    }
 
   return mb;
 }
@@ -516,12 +519,14 @@ _gcry_secmem_malloc_internal (size_t size)
         {
           log_info (_("operation is not possible without "
                       "initialized secure memory\n"));
+          gpg_err_set_errno (ENOMEM);
           return NULL;
         }
     }
   if (not_locked && fips_mode ())
     {
       log_info (_("secure memory pool is not locked while in FIPS mode\n"));
+      gpg_err_set_errno (ENOMEM);
       return NULL;
     }
   if (show_warning && !suspend_warning)
diff --git a/src/stdmem.c b/src/stdmem.c
index c20c4b4f..189da372 100644
--- a/src/stdmem.c
+++ b/src/stdmem.c
@@ -49,6 +49,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdarg.h>
+#include <errno.h>
 
 #include "g10lib.h"
 #include "stdmem.h"
@@ -88,8 +89,12 @@ void *
 _gcry_private_malloc (size_t n)
 {
   if (!n)
-    return NULL; /* Allocating 0 bytes is undefined - we better return
-                    an error to detect such coding errors.  */
+    {
+      gpg_err_set_errno (EINVAL);
+      return NULL; /* Allocating 0 bytes is undefined - we better return
+                      an error to detect such coding errors.  */
+    }
+
   if (use_m_guard)
     {
       char *p;
@@ -118,8 +123,12 @@ void *
 _gcry_private_malloc_secure (size_t n)
 {
   if (!n)
-    return NULL; /* Allocating 0 bytes is undefined - better return an
-                    error to detect such coding errors.  */
+    {
+      gpg_err_set_errno (EINVAL);
+      return NULL; /* Allocating 0 bytes is undefined - better return an
+                      error to detect such coding errors.  */
+    }
+
   if (use_m_guard)
     {
       char *p;