Move RSA key loading and decryption functions to wsutil

Loading PEM and PKCS#11 keys was being done in static functions
in packet-ssl-utils.c. These were moved to wsutil, with prototypes
in a new <wsutil/rsa.h> header. This adds gnutls as optional
dependency to wsutil.

The RSA decryption helper was also moved and is now provided in
<wsutil/wsgcrypt.h>.

This allows more dissectors to access this functionality.

Change-Id: I6cfbbf5203f2881c82bad721747834ccd76e2033
Reviewed-on: https://code.wireshark.org/review/21941
Reviewed-by: Peter Wu <peter@lekensteyn.nl>
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot <buildbot-no-reply@wireshark.org>
Reviewed-by: Michael Mann <mmann78@netscape.net>

6 files changed, 490 insertions, 1 deletions

diff --git a/wsutil/CMakeLists.txt b/wsutil/CMakeLists.txt
index ab9bb7ba82..5eca03eddd 100644
--- a/wsutil/CMakeLists.txt
+++ b/wsutil/CMakeLists.txt
@@ -54,6 +54,7 @@ set(WSUTIL_COMMON_FILES
 	os_version_info.c
 	plugins.c
 	privileges.c
+	rsa.c
 	sober128.c
 	strnatcmp.c
 	str_util.c
@@ -177,6 +178,7 @@ set(wsutil_LIBS
 	${GCRYPT_LIBRARIES}
 	${ZLIB_LIBRARIES}
 	${WIN_WSOCK32_LIBRARY}
+	${GNUTLS_LIBRARIES}
 )
 IF(WIN32)
 	set(wsutil_LIBS ${wsutil_LIBS} "iphlpapi.lib" "ws2_32.lib")
diff --git a/wsutil/Makefile.am b/wsutil/Makefile.am
index 28f04c5be0..cb409f7f4d 100644
--- a/wsutil/Makefile.am
+++ b/wsutil/Makefile.am
@@ -26,7 +26,8 @@ AM_CPPFLAGS = $(INCLUDEDIRS) $(WS_CPPFLAGS) -DWS_BUILD_DLL \
 	-DEXTCAP_DIR=\"$(extcapdir)\" 		\
 	-DPLUGIN_INSTALL_DIR=\"$(plugindir)\"	\
 	-DJSMN_STRICT				\
-	$(GLIB_CFLAGS) $(LIBGCRYPT_CFLAGS)
+	$(GLIB_CFLAGS) $(LIBGCRYPT_CFLAGS)      \
+	$(LIBGNUTLS_CFLAGS)
 
 # Optional headers for ABI checking
 wsutil_optional_abi_includes =
@@ -144,6 +145,7 @@ libwsutil_la_SOURCES = \
 	plugins.c		\
 	privileges.c		\
 	report_message.c	\
+	rsa.c			\
 	sober128.c		\
 	str_util.c		\
 	strtoi.c		\
@@ -174,6 +176,7 @@ libwsutil_la_LIBADD = \
 	@COREFOUNDATION_FRAMEWORKS@	\
 	@GLIB_LIBS@			\
 	@LIBGCRYPT_LIBS@		\
+	@LIBGNUTLS_LIBS@	        \
 	$(wsutil_optional_objects)
 
 EXTRA_libwsutil_la_DEPENDENCIES = \
@@ -195,6 +198,7 @@ EXTRA_DIST = \
 	inet_pton.c		\
 	popcount.c		\
 	popcount.h		\
+	rsa.h			\
 	strptime.c		\
 	strptime.h		\
 	win32-utils.c		\
diff --git a/wsutil/rsa.c b/wsutil/rsa.c
new file mode 100644
index 0000000000..20719cc6e5
--- /dev/null
+++ b/wsutil/rsa.c
@@ -0,0 +1,338 @@
+/* rsa.c
+ *
+ * Functions for RSA private key reading and use
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 2007 Gerald Combs
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "config.h"
+
+#include "rsa.h"
+#include <glib.h>
+#include "filesystem.h"
+#include "file_util.h"
+#include "log.h"
+#include <errno.h>
+
+
+#ifdef HAVE_LIBGNUTLS
+
+#include <gnutls/abstract.h>
+#include <gnutls/pkcs12.h>
+
+/* RSA private key file processing {{{ */
+#define RSA_PARS 6
+gcry_sexp_t
+rsa_privkey_to_sexp(gnutls_x509_privkey_t priv_key, char **err)
+{
+    gnutls_datum_t rsa_datum[RSA_PARS]; /* m, e, d, p, q, u */
+    size_t         tmp_size;
+    gcry_error_t   gret;
+    gcry_sexp_t    rsa_priv_key = NULL;
+    gint           i;
+    gcry_mpi_t     rsa_params[RSA_PARS];
+    *err = NULL;
+
+    /* RSA get parameter */
+    if (gnutls_x509_privkey_export_rsa_raw(priv_key,
+                &rsa_datum[0],
+                &rsa_datum[1],
+                &rsa_datum[2],
+                &rsa_datum[3],
+                &rsa_datum[4],
+                &rsa_datum[5])  != 0) {
+        *err = g_strdup("can't export rsa param (is a rsa private key file ?!?)");
+        return NULL;
+    }
+
+    /* convert each rsa parameter to mpi format*/
+    for(i=0; i<RSA_PARS; i++) {
+        gret = gcry_mpi_scan(&rsa_params[i], GCRYMPI_FMT_USG, rsa_datum[i].data, rsa_datum[i].size,&tmp_size);
+        /* these buffers were allocated by gnutls_x509_privkey_export_rsa_raw() */
+        g_free(rsa_datum[i].data);
+        if (gret != 0) {
+            *err = g_strdup_printf("can't convert m rsa param to int (size %d)", rsa_datum[i].size);
+            return NULL;
+        }
+    }
+
+    /* libgcrypt expects p < q, and gnutls might not return it as such, depending on gnutls version and its crypto backend */
+    if (gcry_mpi_cmp(rsa_params[3], rsa_params[4]) > 0)
+    {
+        /* p, q = q, p */
+        gcry_mpi_swap(rsa_params[3], rsa_params[4]);
+        /* due to swapping p and q, u = p^-1 mod p which happens to be needed. */
+    }
+    /* libgcrypt expects u = p^-1 mod q (for OpenPGP), but the u parameter
+     * says u = q^-1 mod p. Recompute u = p^-1 mod q. Do this unconditionally as
+     * at least GnuTLS 2.12.23 computes an invalid value. */
+    gcry_mpi_invm(rsa_params[5], rsa_params[3], rsa_params[4]);
+
+    if  (gcry_sexp_build( &rsa_priv_key, NULL,
+                "(private-key(rsa((n%m)(e%m)(d%m)(p%m)(q%m)(u%m))))", rsa_params[0],
+                rsa_params[1], rsa_params[2], rsa_params[3], rsa_params[4],
+                rsa_params[5]) != 0) {
+        *err = g_strdup("can't build rsa private key s-exp");
+        return NULL;
+    }
+
+    for (i=0; i< 6; i++)
+        gcry_mpi_release(rsa_params[i]);
+    return rsa_priv_key;
+}
+
+gnutls_x509_privkey_t
+rsa_load_pem_key(FILE *fp, char **err)
+{
+    /* gnutls makes our work much harder, since we have to work internally with
+     * s-exp formatted data, but PEM loader exports only in "gnutls_datum_t"
+     * format, and a datum -> s-exp convertion function does not exist.
+     */
+    gnutls_x509_privkey_t priv_key;
+    gnutls_datum_t        key;
+    ws_statb64            statbuf;
+    gint                  ret;
+    guint                 bytes;
+    *err = NULL;
+
+    if (ws_fstat64(ws_fileno(fp), &statbuf) == -1) {
+        *err = g_strdup("can't ws_fstat64 file");
+        return NULL;
+    }
+    if (S_ISDIR(statbuf.st_mode)) {
+        *err = g_strdup("file is a directory");
+        errno = EISDIR;
+        return NULL;
+    }
+    if (S_ISFIFO(statbuf.st_mode)) {
+        *err = g_strdup("file is a named pipe");
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!S_ISREG(statbuf.st_mode)) {
+        *err = g_strdup("file is not a regular file");
+        errno = EINVAL;
+        return NULL;
+    }
+    /* XXX - check for a too-big size */
+    /* load all file contents into a datum buffer*/
+    key.data = (unsigned char *)g_malloc((size_t)statbuf.st_size);
+    key.size = (int)statbuf.st_size;
+    bytes = (guint) fread(key.data, 1, key.size, fp);
+    if (bytes < key.size) {
+        *err = g_strdup_printf("can't read from file %d bytes, got %d",
+                key.size, bytes);
+        g_free(key.data);
+        return NULL;
+    }
+
+    /* init private key data*/
+    gnutls_x509_privkey_init(&priv_key);
+
+    /* import PEM data*/
+    if ((ret = gnutls_x509_privkey_import(priv_key, &key, GNUTLS_X509_FMT_PEM)) != GNUTLS_E_SUCCESS) {
+        *err = g_strdup_printf("can't import pem data: %s", gnutls_strerror(ret));
+        g_free(key.data);
+        return NULL;
+    }
+
+    if (gnutls_x509_privkey_get_pk_algorithm(priv_key) != GNUTLS_PK_RSA) {
+        *err = g_strdup("private key public key algorithm isn't RSA");
+        g_free(key.data);
+        return NULL;
+    }
+
+    g_free(key.data);
+
+    return priv_key;
+}
+
+static const char *
+BAGTYPE(gnutls_pkcs12_bag_type_t x) {
+    switch (x) {
+        case GNUTLS_BAG_EMPTY:               return "Empty";
+        case GNUTLS_BAG_PKCS8_ENCRYPTED_KEY: return "PKCS#8 Encrypted key";
+        case GNUTLS_BAG_PKCS8_KEY:           return "PKCS#8 Key";
+        case GNUTLS_BAG_CERTIFICATE:         return "Certificate";
+        case GNUTLS_BAG_CRL:                 return "CRL";
+        case GNUTLS_BAG_ENCRYPTED:           return "Encrypted";
+        case GNUTLS_BAG_UNKNOWN:             return "Unknown";
+        default:                             return "<undefined>";
+    }
+}
+
+gnutls_x509_privkey_t
+rsa_load_pkcs12(FILE *fp, const gchar *cert_passwd, char **err)
+{
+    int                       i, j, ret;
+    int                       rest;
+    unsigned char            *p;
+    gnutls_datum_t            data;
+    gnutls_pkcs12_bag_t       bag = NULL;
+    gnutls_pkcs12_bag_type_t  bag_type;
+    size_t                    len;
+
+    gnutls_pkcs12_t       rsa_p12  = NULL;
+    gnutls_x509_privkey_t rsa_pkey = NULL;
+
+    gnutls_x509_privkey_t     priv_key = NULL;
+    *err = NULL;
+
+    rest = 4096;
+    data.data = (unsigned char *)g_malloc(rest);
+    data.size = rest;
+    p = data.data;
+    while ((len = fread(p, 1, rest, fp)) > 0) {
+        p += len;
+        rest -= (int) len;
+        if (!rest) {
+            rest = 1024;
+            data.data = (unsigned char *)g_realloc(data.data, data.size + rest);
+            p = data.data + data.size;
+            data.size += rest;
+        }
+    }
+    data.size -= rest;
+    if (!feof(fp)) {
+        *err = g_strdup("Error during certificate reading.");
+        g_free(data.data);
+        return 0;
+    }
+
+    ret = gnutls_pkcs12_init(&rsa_p12);
+    if (ret < 0) {
+        *err = g_strdup_printf("gnutls_pkcs12_init(&st_p12) - %s", gnutls_strerror(ret));
+        g_free(data.data);
+        return 0;
+    }
+
+    /* load PKCS#12 in DER or PEM format */
+    ret = gnutls_pkcs12_import(rsa_p12, &data, GNUTLS_X509_FMT_DER, 0);
+    if (ret < 0) {
+        ret = gnutls_pkcs12_import(rsa_p12, &data, GNUTLS_X509_FMT_PEM, 0);
+        if (ret < 0) {
+            *err = g_strdup_printf("could not load PKCS#12 in DER or PEM format: %s", gnutls_strerror(ret));
+        }
+    }
+    g_free(data.data);
+    if (ret < 0) {
+        return 0;
+    }
+
+    g_log(NULL, G_LOG_LEVEL_INFO, "rsa_privkey_to_sexp: PKCS#12 imported\n");
+
+    /* TODO: Use gnutls_pkcs12_simple_parse, since 3.1.0 (August 2012) */
+    for (i=0; ; i++) {
+
+        ret = gnutls_pkcs12_bag_init(&bag);
+        if (ret < 0) break;
+
+        ret = gnutls_pkcs12_get_bag(rsa_p12, i, bag);
+        if (ret < 0) break;
+
+        for (j=0; j<gnutls_pkcs12_bag_get_count(bag); j++) {
+
+            ret = gnutls_pkcs12_bag_get_type(bag, j);
+            if (ret < 0) goto done;
+            bag_type = (gnutls_pkcs12_bag_type_t)ret;
+            if (bag_type >= GNUTLS_BAG_UNKNOWN) goto done;
+            g_log(NULL, G_LOG_LEVEL_INFO, "Bag %d/%d: %s\n", i, j, BAGTYPE(bag_type));
+            if (bag_type == GNUTLS_BAG_ENCRYPTED) {
+                ret = gnutls_pkcs12_bag_decrypt(bag, cert_passwd);
+                if (ret == 0) {
+                    ret = gnutls_pkcs12_bag_get_type(bag, j);
+                    if (ret < 0) goto done;
+                    bag_type = (gnutls_pkcs12_bag_type_t)ret;
+                    if (bag_type >= GNUTLS_BAG_UNKNOWN) goto done;
+                    g_log(NULL, G_LOG_LEVEL_INFO, "Bag %d/%d decrypted: %s\n", i, j, BAGTYPE(bag_type));
+                }
+            }
+
+            ret = gnutls_pkcs12_bag_get_data(bag, j, &data);
+            if (ret < 0) goto done;
+
+            switch (bag_type) {
+
+                case GNUTLS_BAG_PKCS8_KEY:
+                case GNUTLS_BAG_PKCS8_ENCRYPTED_KEY:
+
+                    ret = gnutls_x509_privkey_init(&rsa_pkey);
+                    if (ret < 0) {
+                        *err = g_strdup_printf("gnutls_x509_privkey_init(&rsa_pkey) - %s", gnutls_strerror(ret));
+                        goto done;
+                    }
+                    ret = gnutls_x509_privkey_import_pkcs8(rsa_pkey, &data, GNUTLS_X509_FMT_DER, cert_passwd,
+                            (bag_type==GNUTLS_BAG_PKCS8_KEY) ? GNUTLS_PKCS_PLAIN : 0);
+                    if (ret < 0) {
+                        *err = g_strdup_printf("Can not decrypt private key - %s", gnutls_strerror(ret));
+                        goto done;
+                    }
+
+                    if (gnutls_x509_privkey_get_pk_algorithm(rsa_pkey) != GNUTLS_PK_RSA) {
+                        *err = g_strdup("private key public key algorithm isn't RSA");
+                        goto done;
+                    }
+
+                    /* Private key found, return it. */
+                    priv_key = rsa_pkey;
+                    goto done;
+                    break;
+
+                default: ;
+            }
+        }  /* j */
+        if (bag) { gnutls_pkcs12_bag_deinit(bag); bag = NULL; }
+    }  /* i */
+
+done:
+    if (!priv_key && rsa_pkey)
+        gnutls_x509_privkey_deinit(rsa_pkey);
+    if (bag)
+        gnutls_pkcs12_bag_deinit(bag);
+
+    return priv_key;
+}
+
+void
+rsa_private_key_free(gpointer key)
+{
+    gcry_sexp_release((gcry_sexp_t) key);
+}
+
+#else /* ! defined(HAVE_LIBGNUTLS) */
+
+void
+rsa_private_key_free(gpointer key _U_)
+{
+}
+
+#endif /* HAVE_LIBGNUTLS */
+
+/*
+ * Editor modelines  -  http://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 4
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=4 tabstop=8 expandtab:
+ * :indentSize=4:tabSize=8:noTabs=true:
+ */
diff --git a/wsutil/rsa.h b/wsutil/rsa.h
new file mode 100644
index 0000000000..dd5a9e316d
--- /dev/null
+++ b/wsutil/rsa.h
@@ -0,0 +1,55 @@
+/* rsa.h
+ *
+ * Functions for RSA private key reading and use
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 2007 Gerald Combs
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef __RSA_H__
+#define __RSA_H__
+
+#include "ws_symbol_export.h"
+#include "wsgcrypt.h"
+
+#ifdef HAVE_LIBGNUTLS
+#include <gnutls/abstract.h>
+WS_DLL_PUBLIC gcry_sexp_t rsa_privkey_to_sexp(gnutls_x509_privkey_t priv_key, char **err);
+
+/**
+ * Load an RSA private key from specified file
+ * @param fp the file that contain the key data
+ * @param [out] err   error message upon failure; NULL upon success
+ * @return a pointer to the loaded key on success, or NULL upon failure
+ */
+WS_DLL_PUBLIC gnutls_x509_privkey_t rsa_load_pem_key(FILE* fp, char **err);
+
+/**
+ * Load a RSA private key from a PKCS#12 file (DER or PEM format)
+ * @param fp          the file that contains the key data
+ * @param cert_passwd password to decrypt the PKCS#12 file
+ * @param [out] err   error message upon failure; NULL upon success
+ * @return a pointer to the loaded key on success; NULL upon failure
+ */
+WS_DLL_PUBLIC gnutls_x509_privkey_t rsa_load_pkcs12(FILE* fp, const char *cert_passwd, char** err);
+#endif
+
+WS_DLL_PUBLIC void rsa_private_key_free(gpointer key);
+
+
+#endif /* __RSA_H__ */
diff --git a/wsutil/wsgcrypt.c b/wsutil/wsgcrypt.c
index 869111a214..a6d527f4bd 100644
--- a/wsutil/wsgcrypt.c
+++ b/wsutil/wsgcrypt.c
@@ -71,6 +71,92 @@ void crypt_des_ecb(guint8 *output, const guint8 *buffer, const guint8 *key56)
 	gcry_cipher_close(handle);
 }
 
+size_t rsa_decrypt_inplace(const guint len, guchar* data, gcry_sexp_t pk, gboolean pkcs1_padding, char **err)
+{
+	gint        rc = 0;
+	size_t      decr_len = 0, i = 0;
+	gcry_sexp_t s_data = NULL, s_plain = NULL;
+	gcry_mpi_t  encr_mpi = NULL, text = NULL;
+
+	*err = NULL;
+
+	/* create mpi representation of encrypted data */
+	rc = gcry_mpi_scan(&encr_mpi, GCRYMPI_FMT_USG, data, len, NULL);
+	if (rc != 0 ) {
+		*err = g_strdup_printf("can't convert data to mpi (size %d):%s", len, gcry_strerror(rc));
+		return 0;
+	}
+
+	/* put the data into a simple list */
+	rc = gcry_sexp_build(&s_data, NULL, "(enc-val(rsa(a%m)))", encr_mpi);
+	if (rc != 0) {
+		*err = g_strdup_printf("can't build encr_sexp:%s", gcry_strerror(rc));
+		decr_len = 0;
+		goto out;
+	}
+
+	/* pass it to libgcrypt */
+	rc = gcry_pk_decrypt(&s_plain, s_data, pk);
+	if (rc != 0)
+	{
+		*err = g_strdup_printf("can't decrypt key:%s", gcry_strerror(rc));
+		decr_len = 0;
+		goto out;
+	}
+
+	/* convert plain text sexp to mpi format */
+	text = gcry_sexp_nth_mpi(s_plain, 0, 0);
+	if (! text) {
+		*err = g_strdup("can't convert sexp to mpi");
+		decr_len = 0;
+		goto out;
+	}
+
+	/* compute size requested for plaintext buffer */
+	rc = gcry_mpi_print(GCRYMPI_FMT_USG, NULL, 0, &decr_len, text);
+	if (rc != 0) {
+		*err = g_strdup_printf("can't compute decr size:%s", gcry_strerror(rc));
+		decr_len = 0;
+		goto out;
+	}
+
+	/* sanity check on out buffer */
+	if (decr_len > len) {
+		*err = g_strdup_printf("decrypted data is too long ?!? (%" G_GSIZE_MODIFIER "u max %d)", decr_len, len);
+		decr_len = 0;
+		goto out;
+	}
+
+	/* write plain text to newly allocated buffer */
+	rc = gcry_mpi_print(GCRYMPI_FMT_USG, data, len, &decr_len, text);
+	if (rc != 0) {
+		*err = g_strdup_printf("can't print decr data to mpi (size %" G_GSIZE_MODIFIER "u):%s", decr_len, gcry_strerror(rc));
+		decr_len = 0;
+		goto out;
+	}
+
+	if (pkcs1_padding) {
+		/* strip the padding*/
+		rc = 0;
+		for (i = 1; i < decr_len; i++) {
+			if (data[i] == 0) {
+				rc = (gint) i+1;
+				break;
+			}
+		}
+
+		decr_len -= rc;
+		memmove(data, data+rc, decr_len);
+	}
+
+out:
+	gcry_sexp_release(s_data);
+	gcry_sexp_release(s_plain);
+	gcry_mpi_release(encr_mpi);
+	gcry_mpi_release(text);
+	return decr_len;
+}
+
 /*
  * Editor modelines  -  http://www.wireshark.org/tools/modelines.html
  *
diff --git a/wsutil/wsgcrypt.h b/wsutil/wsgcrypt.h
index 90ad24081d..8ad0fcfd4f 100644
--- a/wsutil/wsgcrypt.h
+++ b/wsutil/wsgcrypt.h
@@ -51,4 +51,8 @@ WS_DLL_PUBLIC gcry_error_t ws_hmac_buffer(int algo, void *digest, const void *bu
    64 bits as key, encrypted data is returned in OUTPUT which must be at least 8 bytes large */
 WS_DLL_PUBLIC void crypt_des_ecb(guint8 *output, const guint8 *buffer, const guint8 *key56);
 
+/* Convenience function for RSA decryption. Returns decrypted length on success, 0 on failure */
+WS_DLL_PUBLIC size_t rsa_decrypt_inplace(const guint len, guchar* data, gcry_sexp_t pk, gboolean pkcs1_padding, char **err);
+
+
 #endif /* __WSGCRYPT_H__ */