Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

1 files changed, 267 insertions, 182 deletions

diff --git a/epan/radius_dict.l b/epan/radius_dict.l
index ebbf2a060a..86a8428f07 100644
--- a/epan/radius_dict.l
+++ b/epan/radius_dict.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -24,10 +29,28 @@
 %option caseless
 
 /*
- * Prefix scanner routines with "Radius" rather than "yy", so this scanner
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="Radius_scanner_state_t*"
+
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
+/*
+ * Prefix scanner routines with "Radius_" rather than "yy", so this scanner
  * can coexist with other scanners.
  */
-%option prefix="Radius"
+%option prefix="Radius_"
 
 %option outfile="radius_dict.c"
 
@@ -64,9 +87,10 @@
 #include <errno.h>
 #include <epan/packet.h>
 #include <epan/dissectors/packet-radius.h>
-#include "radius_dict_lex.h"
 #include <wsutil/file_util.h>
 
+#define YY_USER_INIT BEGIN WS_OUT;
+
 #ifdef _WIN32
 /* disable Windows VC compiler warning "signed/unsigned mismatch" associated  */
 /* with YY_INPUT code generated by flex versions such as 2.5.35.              */
@@ -76,36 +100,51 @@
 #define ECHO
 #define MAX_INCLUDE_DEPTH 10
 
-	static void add_vendor(const gchar* name, guint32 id, guint type_octets, guint length_octets, gboolean has_flags);
-	static void add_value(const gchar* attrib_name,const  gchar* repr, guint32 value);
-	static void add_tlv(const gchar* name, const  gchar* code, radius_attr_dissector_t type, const gchar* attr);
-	static void add_attribute(const gchar*,const  gchar*, radius_attr_dissector_t,const  gchar*, guint, gboolean, const gchar*);
-
-	static YY_BUFFER_STATE include_stack[10];
-	static int include_stack_ptr = 0;
-
-	static radius_dictionary_t* dict = NULL;
-	static GHashTable* value_strings = NULL; /* GArray(value_string) by attribute name */
-
-	static gchar* attr_name = NULL;
-	static gchar* attr_id = NULL;
-	static radius_attr_dissector_t* attr_type = NULL;
-	static gchar* attr_vendor = NULL;
-	static gchar* vendor_name = NULL;
-	static guint32 vendor_id = 0;
-	static guint vendor_type_octets = 1;
-	static guint vendor_length_octets = 1;
-	static gboolean vendor_has_flags = FALSE;
-	static gchar* value_repr = NULL;
-	static guint encrypted = 0;
-	static gboolean has_tag = FALSE;
-	static gchar* current_vendor = NULL;
-	static gchar* current_attr = NULL;
-
-	static GString* error = NULL;
-	static gchar* directory = NULL;
-	static int linenums[] = {1,1,1,1,1,1,1,1,1,1};
-	static gchar* fullpaths[] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL};
+typedef struct {
+	YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
+	int include_stack_ptr;
+
+	radius_dictionary_t* dict;
+	GHashTable* value_strings; /* GArray(value_string) by attribute name */
+
+	gchar* attr_name;
+	gchar* attr_id;
+	radius_attr_dissector_t* attr_type;
+	gchar* attr_vendor;
+	gchar* vendor_name;
+	guint32 vendor_id;
+	guint vendor_type_octets;
+	guint vendor_length_octets;
+	gboolean vendor_has_flags;
+	gchar* value_repr;
+	guint encrypted;
+	gboolean has_tag;
+	gchar* current_vendor;
+	gchar* current_attr;
+
+	gchar* directory;
+	gchar* fullpaths[MAX_INCLUDE_DEPTH];
+	int linenums[MAX_INCLUDE_DEPTH];
+
+	GString* error;
+} Radius_scanner_state_t;
+
+static void add_vendor(Radius_scanner_state_t* state, const gchar* name, guint32 id, guint type_octets, guint length_octets, gboolean has_flags);
+static gboolean add_attribute(Radius_scanner_state_t* state, const gchar*,const  gchar*, radius_attr_dissector_t,const  gchar*, guint, gboolean, const gchar*);
+static gboolean add_tlv(Radius_scanner_state_t* state, const gchar* name, const  gchar* code, radius_attr_dissector_t type, const gchar* attr);
+static void add_value(Radius_scanner_state_t* state, const gchar* attrib_name, const  gchar* repr, guint32 value);
+
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define Radius_alloc(size, yyscanner)		(void *)malloc(size)
+#define Radius_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define Radius_free(ptr, yyscanner)		free((char *)ptr)
 
 %}
 
@@ -132,63 +171,63 @@
 <WS_OUT>END-TLV { BEGIN END_TLV; }
 
 <BEGIN_VENDOR>[0-9a-z_-]+ {
-    if (current_vendor) {
-        g_free(current_vendor);
+    if (yyextra->current_vendor) {
+        g_free(yyextra->current_vendor);
     }
-    current_vendor = g_strdup(yytext);
+    yyextra->current_vendor = g_strdup(yytext);
     BEGIN WS_OUT;
 }
 <END_VENDOR>[^\n]* {
-    if (current_vendor) {
-        g_free(current_vendor);
-        current_vendor = NULL;
+    if (yyextra->current_vendor) {
+        g_free(yyextra->current_vendor);
+        yyextra->current_vendor = NULL;
     }
     BEGIN WS_OUT;
 }
 
 <BEGIN_TLV>[0-9a-z_-]+ {
-    if (current_attr) {
-        g_free(current_attr);
+    if (yyextra->current_attr) {
+        g_free(yyextra->current_attr);
     }
-    current_attr = g_strdup(yytext);
+    yyextra->current_attr = g_strdup(yytext);
     BEGIN WS_OUT;
 }
 <END_TLV>[^\n]* {
-    if (current_attr) {
-        g_free(current_attr);
-        current_attr = NULL;
+    if (yyextra->current_attr) {
+        g_free(yyextra->current_attr);
+        yyextra->current_attr = NULL;
     }
     BEGIN WS_OUT;
 }
 
 <VENDOR>[0-9a-z_-]+   {
-    vendor_name = g_strdup(yytext);
-    vendor_type_octets = 1;
-    vendor_length_octets = 1;
-    vendor_has_flags = FALSE;
+    yyextra->vendor_name = g_strdup(yytext);
+    yyextra->vendor_type_octets = 1;
+    yyextra->vendor_length_octets = 1;
+    yyextra->vendor_has_flags = FALSE;
     BEGIN VENDOR_W_NAME;
 }
 <VENDOR_W_NAME>[0-9]+   {
-    vendor_id = (guint32) strtoul(yytext,NULL,10);
+    yyextra->vendor_id = (guint32) strtoul(yytext,NULL,10);
     BEGIN VENDOR_W_ID;
 }
 <VENDOR_W_NAME>0x[0-9a-f]+   {
-    vendor_id = (guint32) strtoul(yytext,NULL,16);
+    yyextra->vendor_id = (guint32) strtoul(yytext,NULL,16);
     BEGIN VENDOR_W_ID;
 }
 <VENDOR_W_ID>format= {
     BEGIN VENDOR_W_FORMAT;
 }
 <VENDOR_W_FORMAT>[124] {
-    vendor_type_octets = (guint) strtoul(yytext,NULL,10);
+    yyextra->vendor_type_octets = (guint) strtoul(yytext,NULL,10);
     BEGIN VENDOR_W_TYPE_OCTETS;
 }
 <VENDOR_W_TYPE_OCTETS>,[012] {
-    vendor_length_octets = (guint) strtoul(yytext+1,NULL,10);
+    yyextra->vendor_length_octets = (guint) strtoul(yytext+1,NULL,10);
     BEGIN VENDOR_W_LENGTH_OCTETS;
 }
 <VENDOR_W_LENGTH_OCTETS>,c {
-    vendor_has_flags = TRUE;
+    yyextra->vendor_has_flags = TRUE;
     BEGIN VENDOR_W_CONTINUATION;
 }
 <VENDOR_W_FORMAT>\n |
@@ -196,98 +235,103 @@
 <VENDOR_W_LENGTH_OCTETS>\n |
 <VENDOR_W_CONTINUATION>\n |
 <VENDOR_W_ID>\n {
-    add_vendor(vendor_name, vendor_id, vendor_type_octets, vendor_length_octets, vendor_has_flags);
-    g_free(vendor_name);
+    add_vendor(yyextra, yyextra->vendor_name, yyextra->vendor_id, yyextra->vendor_type_octets, yyextra->vendor_length_octets, yyextra->vendor_has_flags);
+    g_free(yyextra->vendor_name);
     BEGIN WS_OUT;
 }
 
-<ATTR>[0-9a-z_/.-]+			{ attr_name = g_strdup(yytext); encrypted = 0; has_tag = FALSE; BEGIN ATTR_W_NAME; }
-<ATTR_W_NAME>[0-9]+			{ attr_id = g_strdup(yytext);  BEGIN ATTR_W_ID;}
-<ATTR_W_NAME>0x[0-9a-f]+		{ attr_id = g_strdup_printf("%u",(int)strtoul(yytext,NULL,16)); BEGIN ATTR_W_ID;}
-<ATTR_W_ID>integer			{ attr_type = radius_integer;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>string			{ attr_type = radius_string;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>octets			{ attr_type = radius_octets;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>ipaddr			{ attr_type = radius_ipaddr;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>ipv6addr			{ attr_type = radius_ipv6addr;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>ipv6prefix			{ attr_type = radius_ipv6prefix;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>ipxnet			{ attr_type = radius_ipxnet;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>date				{ attr_type = radius_date;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>abinary			{ attr_type = radius_abinary;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>ether  			{ attr_type = radius_ether;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>ifid				{ attr_type = radius_ifid;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>byte				{ attr_type = radius_integer;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>short			{ attr_type = radius_integer;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>signed			{ attr_type = radius_signed;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>combo-ip			{ attr_type = radius_combo_ip;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>tlv				{ attr_type = radius_tlv;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_ID>[0-9a-z_-]+			{ attr_type = radius_octets;  BEGIN ATTR_W_TYPE; }
-<ATTR_W_TYPE>has_tag[,]?		{ has_tag = TRUE; }
-<ATTR_W_TYPE>encrypt=[123][,]?		{ encrypted = (guint) strtoul(yytext+8,NULL,10); }
+<ATTR>[0-9a-z_/.-]+			{ yyextra->attr_name = g_strdup(yytext); yyextra->encrypted = 0; yyextra->has_tag = FALSE; BEGIN ATTR_W_NAME; }
+<ATTR_W_NAME>[0-9]+			{ yyextra->attr_id = g_strdup(yytext);  BEGIN ATTR_W_ID;}
+<ATTR_W_NAME>0x[0-9a-f]+		{ yyextra->attr_id = g_strdup_printf("%u",(int)strtoul(yytext,NULL,16)); BEGIN ATTR_W_ID;}
+<ATTR_W_ID>integer			{ yyextra->attr_type = radius_integer;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>string			{ yyextra->attr_type = radius_string;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>octets			{ yyextra->attr_type = radius_octets;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>ipaddr			{ yyextra->attr_type = radius_ipaddr;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>ipv6addr			{ yyextra->attr_type = radius_ipv6addr;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>ipv6prefix			{ yyextra->attr_type = radius_ipv6prefix;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>ipxnet			{ yyextra->attr_type = radius_ipxnet;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>date				{ yyextra->attr_type = radius_date;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>abinary			{ yyextra->attr_type = radius_abinary;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>ether  			{ yyextra->attr_type = radius_ether;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>ifid				{ yyextra->attr_type = radius_ifid;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>byte				{ yyextra->attr_type = radius_integer;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>short			{ yyextra->attr_type = radius_integer;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>signed			{ yyextra->attr_type = radius_signed;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>combo-ip			{ yyextra->attr_type = radius_combo_ip;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>tlv				{ yyextra->attr_type = radius_tlv;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_ID>[0-9a-z_-]+			{ yyextra->attr_type = radius_octets;  BEGIN ATTR_W_TYPE; }
+<ATTR_W_TYPE>has_tag[,]?		{ yyextra->has_tag = TRUE; }
+<ATTR_W_TYPE>encrypt=[123][,]?		{ yyextra->encrypted = (guint) strtoul(yytext+8,NULL,10); }
 <ATTR_W_TYPE>[0-9a-z_-]+=([^\n]*)	;
 <ATTR_W_TYPE>[0-9a-z_-]+		{
-    attr_vendor = g_strdup(yytext);
-    add_attribute(attr_name,attr_id,attr_type,attr_vendor,encrypted,has_tag,current_attr);
-    g_free(attr_id);
-    g_free(attr_vendor);
-    g_free(attr_name);
-    attr_id = NULL;
-    attr_vendor = NULL;
-    attr_name = NULL;
-    BEGIN WS_OUT;
+    gboolean attribute_ok;
+
+    yyextra->attr_vendor = g_strdup(yytext);
+    attribute_ok = add_attribute(yyextra, yyextra->attr_name, yyextra->attr_id, yyextra->attr_type, yyextra->attr_vendor, yyextra->encrypted, yyextra->has_tag, yyextra->current_attr);
+    g_free(yyextra->attr_id);
+    g_free(yyextra->attr_vendor);
+    g_free(yyextra->attr_name);
+    yyextra->attr_id = NULL;
+    yyextra->attr_vendor = NULL;
+    yyextra->attr_name = NULL;
+    if (attribute_ok)
+        BEGIN WS_OUT;
+    else
+        BEGIN JUNK;
 }
 <ATTR_W_TYPE>\n						{
-    add_attribute(attr_name,attr_id,attr_type,current_vendor,encrypted,has_tag,current_attr);
-    g_free(attr_id);
-    g_free(attr_name);
-    linenums[include_stack_ptr]++;
-    has_tag = FALSE;
-    encrypted=FALSE;
+    add_attribute(yyextra, yyextra->attr_name, yyextra->attr_id, yyextra->attr_type, yyextra->current_vendor, yyextra->encrypted, yyextra->has_tag, yyextra->current_attr);
+    g_free(yyextra->attr_id);
+    g_free(yyextra->attr_name);
+    yyextra->linenums[yyextra->include_stack_ptr]++;
+    yyextra->has_tag = FALSE;
+    yyextra->encrypted=FALSE;
     BEGIN WS_OUT;
 }
 <ATTR_W_VENDOR>\n					{
-    add_attribute(attr_name,attr_id,attr_type,attr_vendor,encrypted,has_tag,current_attr);
-    g_free(attr_id);
-    g_free(attr_vendor);
-    g_free(attr_name);
-    linenums[include_stack_ptr]++;
+    add_attribute(yyextra, yyextra->attr_name, yyextra->attr_id, yyextra->attr_type, yyextra->attr_vendor, yyextra->encrypted, yyextra->has_tag, yyextra->current_attr);
+    g_free(yyextra->attr_id);
+    g_free(yyextra->attr_vendor);
+    g_free(yyextra->attr_name);
+    yyextra->linenums[yyextra->include_stack_ptr]++;
     BEGIN WS_OUT;
 };
 
-<VALUE>[0-9a-z_/-]+				{ attr_name = g_strdup(yytext); BEGIN VALUE_W_ATTR; }
-<VALUE_W_ATTR>[^[:blank:]]+			{ value_repr = g_strdup(yytext); BEGIN VALUE_W_NAME; }
-<VALUE_W_NAME>[0-9]+				{ add_value(attr_name,value_repr, (guint32) strtoul(yytext,NULL,10));  g_free(attr_name); g_free(value_repr); BEGIN WS_OUT;}
-<VALUE_W_NAME>0x[0-9a-f]+			{ add_value(attr_name,value_repr, (guint32) strtoul(yytext,NULL,16));  g_free(attr_name); g_free(value_repr); BEGIN WS_OUT;}
+<VALUE>[0-9a-z_/-]+				{ yyextra->attr_name = g_strdup(yytext); BEGIN VALUE_W_ATTR; }
+<VALUE_W_ATTR>[^[:blank:]]+			{ yyextra->value_repr = g_strdup(yytext); BEGIN VALUE_W_NAME; }
+<VALUE_W_NAME>[0-9]+				{ add_value(yyextra, yyextra->attr_name,yyextra->value_repr, (guint32) strtoul(yytext,NULL,10));  g_free(yyextra->attr_name); g_free(yyextra->value_repr); BEGIN WS_OUT;}
+<VALUE_W_NAME>0x[0-9a-f]+			{ add_value(yyextra, yyextra->attr_name,yyextra->value_repr, (guint32) strtoul(yytext,NULL,16));  g_free(yyextra->attr_name); g_free(yyextra->value_repr); BEGIN WS_OUT;}
 
 <INCLUDE>[^[:blank:]\n]+   {
-	if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) {
-		g_string_append_printf(error, "$INCLUDE files nested to deeply\n");
+	if ( yyextra->include_stack_ptr >= MAX_INCLUDE_DEPTH ) {
+		g_string_append_printf(yyextra->error, "$INCLUDE files nested too deeply\n");
 		yyterminate();
 	}
 
-	include_stack[include_stack_ptr++] = YY_CURRENT_BUFFER;
+	yyextra->include_stack[yyextra->include_stack_ptr++] = YY_CURRENT_BUFFER;
 
-	fullpaths[include_stack_ptr] = g_strdup_printf("%s" G_DIR_SEPARATOR_S "%s",
-	    directory,yytext);
+	yyextra->fullpaths[yyextra->include_stack_ptr] = g_strdup_printf("%s" G_DIR_SEPARATOR_S "%s",
+	    yyextra->directory,yytext);
 
-	yyin = ws_fopen( fullpaths[include_stack_ptr], "r" );
+	yyin = ws_fopen( yyextra->fullpaths[yyextra->include_stack_ptr], "r" );
 
 	if (!yyin) {
 		if (errno) {
-			g_string_append_printf(error,
+			g_string_append_printf(yyextra->error,
 					"Could not open file: '%s', error: %s\n",
-					fullpaths[include_stack_ptr],
+					yyextra->fullpaths[yyextra->include_stack_ptr],
 					g_strerror(errno) );
 		} else {
-			g_string_append_printf(error,
+			g_string_append_printf(yyextra->error,
 					"Could not open file: '%s', no errno\n",
-					fullpaths[include_stack_ptr]);
+					yyextra->fullpaths[yyextra->include_stack_ptr]);
 		}
-		g_free(fullpaths[include_stack_ptr]);
-		fullpaths[include_stack_ptr] = NULL;
-		include_stack_ptr--;
+		g_free(yyextra->fullpaths[yyextra->include_stack_ptr]);
+		yyextra->fullpaths[yyextra->include_stack_ptr] = NULL;
+		yyextra->include_stack_ptr--;
 	} else {
-		linenums[include_stack_ptr] = 1;
-		yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
+		yyextra->linenums[yyextra->include_stack_ptr] = 1;
+		yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner), yyscanner);
 	}
 
 
@@ -299,28 +343,28 @@
 	fclose(yyin);
 	yyin = NULL;
 
-	if ( --include_stack_ptr < 0 ) {
+	if ( --yyextra->include_stack_ptr < 0 ) {
 		yyterminate();
 	} else {
-		g_free(fullpaths[include_stack_ptr+1]);
-		fullpaths[include_stack_ptr+1] = NULL;
+		g_free(yyextra->fullpaths[yyextra->include_stack_ptr+1]);
+		yyextra->fullpaths[yyextra->include_stack_ptr+1] = NULL;
 
-		yy_delete_buffer( YY_CURRENT_BUFFER );
-		yy_switch_to_buffer(include_stack[include_stack_ptr]);
+		Radius__delete_buffer(YY_CURRENT_BUFFER, yyscanner);
+		Radius__switch_to_buffer(yyextra->include_stack[yyextra->include_stack_ptr], yyscanner);
 	}
 
 	BEGIN WS_OUT;
 }
 
-\n 	{ linenums[include_stack_ptr]++; BEGIN WS_OUT; }
+\n 	{ yyextra->linenums[yyextra->include_stack_ptr]++; BEGIN WS_OUT; }
 
 
 %%
 
-static void add_vendor(const gchar* name, guint32 id, guint type_octets, guint length_octets, gboolean has_flags) {
+static void add_vendor(Radius_scanner_state_t* state, const gchar* name, guint32 id, guint type_octets, guint length_octets, gboolean has_flags) {
 	radius_vendor_info_t* v;
 
-	v = (radius_vendor_info_t *)g_hash_table_lookup(dict->vendors_by_id, GUINT_TO_POINTER(id));
+	v = (radius_vendor_info_t *)g_hash_table_lookup(state->dict->vendors_by_id, GUINT_TO_POINTER(id));
 
 	if (!v) {
 		/*
@@ -337,8 +381,8 @@ static void add_vendor(const gchar* name, guint32 id, guint type_octets, guint l
 		v->length_octets = length_octets;
 		v->has_flags = has_flags;
 
-		g_hash_table_insert(dict->vendors_by_id,GUINT_TO_POINTER(v->code),v);
-		g_hash_table_insert(dict->vendors_by_name, (gpointer) v->name, v);
+		g_hash_table_insert(state->dict->vendors_by_id,GUINT_TO_POINTER(v->code),v);
+		g_hash_table_insert(state->dict->vendors_by_name, (gpointer) v->name, v);
 	} else {
 		/*
 		 * This vendor is already in the table.
@@ -367,38 +411,36 @@ static void add_vendor(const gchar* name, guint32 id, guint type_octets, guint l
 			 * Yes.  Remove the entry from the by-name hash table
 			 * and re-insert it with the new name.
 			 */
-			g_hash_table_remove(dict->vendors_by_name, (gpointer) v->name);
+			g_hash_table_remove(state->dict->vendors_by_name, (gpointer) v->name);
 			g_free((gpointer) v->name);
 			v->name = g_strdup(name);
-			g_hash_table_insert(dict->vendors_by_name, (gpointer) v->name, v);
+			g_hash_table_insert(state->dict->vendors_by_name, (gpointer) v->name, v);
 		}
 	}
 }
 
-static void add_attribute(const gchar* name, const  gchar* codestr, radius_attr_dissector_t type, const  gchar* vendor, guint encrypted_flag, gboolean tagged, const gchar* attr) {
+static gboolean add_attribute(Radius_scanner_state_t* state, const gchar* name, const  gchar* codestr, radius_attr_dissector_t type, const  gchar* vendor, guint encrypted_flag, gboolean tagged, const gchar* attr) {
 	radius_attr_info_t* a;
 	GHashTable* by_id;
 	guint32 code;
 
 	if (attr){
-		add_tlv(name, codestr, type, attr);
-		return;
+		return add_tlv(state, name, codestr, type, attr);
 	}
 
 
 	if (vendor) {
 		radius_vendor_info_t* v;
-		v = (radius_vendor_info_t *)g_hash_table_lookup(dict->vendors_by_name,vendor);
+		v = (radius_vendor_info_t *)g_hash_table_lookup(state->dict->vendors_by_name,vendor);
 
 		if (! v) {
-			g_string_append_printf(error, "Vendor: '%s', does not exist in %s:%i \n", vendor, fullpaths[include_stack_ptr], linenums[include_stack_ptr] );
-			BEGIN JUNK;
-			return;
+			g_string_append_printf(state->error, "Vendor: '%s', does not exist in %s:%i \n", vendor, state->fullpaths[state->include_stack_ptr], state->linenums[state->include_stack_ptr] );
+			return FALSE;
 		} else {
 			by_id = v->attrs_by_id;
 		}
 	} else {
-		by_id = dict->attrs_by_id;
+		by_id = state->dict->attrs_by_id;
 	}
 
 	code= (guint32) strtoul(codestr, NULL, 10);
@@ -426,7 +468,7 @@ static void add_attribute(const gchar* name, const  gchar* codestr, radius_attr_
 		a->ett = -1;
 		a->tlvs_by_id = NULL;
 		g_hash_table_insert(by_id, GUINT_TO_POINTER(code),a);
-		g_hash_table_insert(dict->attrs_by_name,(gpointer) (a->name),a);
+		g_hash_table_insert(state->dict->attrs_by_name,(gpointer) (a->name),a);
 	} else {
 		/*
 		 * This attribute is already in the table.
@@ -454,31 +496,30 @@ static void add_attribute(const gchar* name, const  gchar* codestr, radius_attr_
 			 * Yes.  Remove the entry from the by-name hash table
 			 * and re-insert it with the new name.
 			 */
-			g_hash_table_remove(dict->attrs_by_name, (gpointer) (a->name));
+			g_hash_table_remove(state->dict->attrs_by_name, (gpointer) (a->name));
 			g_free((gpointer) a->name);
 			a->name = g_strdup(name);
-			g_hash_table_insert(dict->attrs_by_name, (gpointer) (a->name),a);
+			g_hash_table_insert(state->dict->attrs_by_name, (gpointer) (a->name),a);
 		}
 	}
+	return TRUE;
 }
 
-static void add_tlv(const gchar* name, const  gchar* codestr, radius_attr_dissector_t type, const gchar* attr) {
+static gboolean add_tlv(Radius_scanner_state_t* state, const gchar* name, const  gchar* codestr, radius_attr_dissector_t type, const gchar* attr) {
 	radius_attr_info_t* a;
 	radius_attr_info_t* s;
 	guint32 code;
 
-	a = (radius_attr_info_t*)g_hash_table_lookup(dict->attrs_by_name, attr);
+	a = (radius_attr_info_t*)g_hash_table_lookup(state->dict->attrs_by_name, attr);
 
 	if (! a) {
-		g_string_append_printf(error, "Attr: '%s', does not exist in %s:%i \n", attr, fullpaths[include_stack_ptr], linenums[include_stack_ptr]);
-		BEGIN JUNK;
-		return;
+		g_string_append_printf(state->error, "Attr: '%s', does not exist in %s:%i \n", attr, state->fullpaths[state->include_stack_ptr], state->linenums[state->include_stack_ptr]);
+		return FALSE;
 	}
 
 	if (type == radius_tlv) {
-		g_string_append_printf(error, "sub-TLV: '%s', sub-TLV's type is specified as tlv in %s:%i \n", name, fullpaths[include_stack_ptr], linenums[include_stack_ptr]);
-		BEGIN JUNK;
-		return;
+		g_string_append_printf(state->error, "sub-TLV: '%s', sub-TLV's type is specified as tlv in %s:%i \n", name, state->fullpaths[state->include_stack_ptr], state->linenums[state->include_stack_ptr]);
+		return FALSE;
 	}
 
 
@@ -512,7 +553,7 @@ static void add_tlv(const gchar* name, const  gchar* codestr, radius_attr_dissec
 		s->tlvs_by_id = NULL;
 
 		g_hash_table_insert(a->tlvs_by_id,GUINT_TO_POINTER(s->code),s);
-		g_hash_table_insert(dict->tlvs_by_name,(gpointer) (s->name),s);
+		g_hash_table_insert(state->dict->tlvs_by_name,(gpointer) (s->name),s);
 	}
 
 	/*
@@ -522,17 +563,18 @@ static void add_tlv(const gchar* name, const  gchar* codestr, radius_attr_dissec
 	 * one adding some TLV values), and we don't directly add entries
 	 * for TLVs in the RADIUS dissector.
 	 *
-	 * XXX - report the duplicate entries? 
+	 * XXX - report the duplicate entries?
 	 */
+	return TRUE;
 }
 
-void add_value(const gchar* attrib_name, const gchar* repr, guint32 value) {
+void add_value(Radius_scanner_state_t* state, const gchar* attrib_name, const gchar* repr, guint32 value) {
 	value_string v;
-	GArray* a = (GArray*)g_hash_table_lookup(value_strings,attrib_name);
+	GArray* a = (GArray*)g_hash_table_lookup(state->value_strings,attrib_name);
 
 	if (! a) {
 		a = g_array_new(TRUE,TRUE,sizeof(value_string));
-		g_hash_table_insert(value_strings,g_strdup(attrib_name),a);
+		g_hash_table_insert(state->value_strings,g_strdup(attrib_name),a);
 	}
 
 	v.value = value;
@@ -541,8 +583,9 @@ void add_value(const gchar* attrib_name, const gchar* repr, guint32 value) {
 	g_array_append_val(a,v);
 }
 
-static void setup_tlvs(gpointer k _U_, gpointer v, gpointer p _U_) {
+static void setup_tlvs(gpointer k _U_, gpointer v, gpointer p) {
 	radius_attr_info_t* s = (radius_attr_info_t*)v;
+	Radius_scanner_state_t* state = (Radius_scanner_state_t*)p;
 	gpointer key;
 
 	union {
@@ -550,16 +593,17 @@ static void setup_tlvs(gpointer k _U_, gpointer v, gpointer p _U_) {
 		gpointer p;
 	} vs;
 
-	if (g_hash_table_lookup_extended(value_strings, s->name, &key, &vs.p)) {
+	if (g_hash_table_lookup_extended(state->value_strings, s->name, &key, &vs.p)) {
 		s->vs = (value_string*)(void *)vs.a->data;
 		g_array_free(vs.a, FALSE);
-		g_hash_table_remove(value_strings, key);
+		g_hash_table_remove(state->value_strings, key);
 		g_free(key);
 	}
 }
 
-static void setup_attrs(gpointer k _U_, gpointer v, gpointer p _U_) {
+static void setup_attrs(gpointer k _U_, gpointer v, gpointer p) {
 	radius_attr_info_t* a = (radius_attr_info_t*)v;
+	Radius_scanner_state_t* state = (Radius_scanner_state_t*)p;
 	gpointer key;
 
 	union {
@@ -567,10 +611,10 @@ static void setup_attrs(gpointer k _U_, gpointer v, gpointer p _U_) {
 		gpointer p;
 	} vs;
 
-	if (g_hash_table_lookup_extended(value_strings,a->name,&key,&vs.p) ) {
+	if (g_hash_table_lookup_extended(state->value_strings,a->name,&key,&vs.p) ) {
 		a->vs = (value_string*)(void *)vs.a->data;
 		g_array_free(vs.a,FALSE);
-		g_hash_table_remove(value_strings,key);
+		g_hash_table_remove(state->value_strings,key);
 		g_free(key);
 	}
 
@@ -599,48 +643,89 @@ static gboolean destroy_value_strings(gpointer k, gpointer v, gpointer p _U_) {
 }
 
 gboolean radius_load_dictionary (radius_dictionary_t* d, gchar* dir, const gchar* filename, gchar** err_str) {
+	FILE *in;
+	yyscan_t scanner;
+	Radius_scanner_state_t state;
 	int i;
 
-	dict = d;
-	directory = dir;
+	state.include_stack_ptr = 0;
+
+	state.dict = d;
+	state.value_strings = NULL;
+
+	state.attr_name = NULL;
+	state.attr_id = NULL;
+	state.attr_type = NULL;
+	state.attr_vendor = NULL;
+	state.vendor_name = NULL;
+	state.vendor_id = 0;
+	state.vendor_type_octets = 1;
+	state.vendor_length_octets = 1;
+	state.vendor_has_flags = FALSE;
+	state.value_repr = NULL;
+	state.encrypted = 0;
+	state.has_tag = FALSE;
+	state.current_vendor = NULL;
+	state.current_attr = NULL;
+
+	state.directory = dir;
+
+	state.fullpaths[0] = g_strdup_printf("%s" G_DIR_SEPARATOR_S "%s",
+	    state.directory,filename);
+	state.linenums[0] = 1;
+	for (i = 1; i < MAX_INCLUDE_DEPTH; i++) {
+		state.fullpaths[i] = NULL;
+		state.linenums[i] = 1;
+	}
 
-	fullpaths[include_stack_ptr] = g_strdup_printf("%s" G_DIR_SEPARATOR_S "%s",
-	    directory,filename);
+	state.error = g_string_new("");
 
-	error = g_string_new("");
+	in = ws_fopen(state.fullpaths[0],"r");
 
-	yyin = ws_fopen(fullpaths[include_stack_ptr],"r");
+	if (!in) {
+		g_string_append_printf(state.error, "Could not open file: '%s', error: %s\n", state.fullpaths[0], g_strerror(errno));
+		g_free(state.fullpaths[0]);
+		*err_str = g_string_free(state.error,FALSE);
+		return FALSE;
+	}
 
-	if (!yyin) {
-		g_string_append_printf(error, "Could not open file: '%s', error: %s\n", fullpaths[include_stack_ptr], g_strerror(errno) );
-		g_free(fullpaths[include_stack_ptr]);
-		*err_str = g_string_free(error,FALSE);
+	state.value_strings = g_hash_table_new(g_str_hash,g_str_equal);
+
+	if (Radius_lex_init(&scanner) != 0) {
+		g_string_append_printf(state.error, "Can't initialize scanner: %s",
+		    strerror(errno));
+		fclose(in);
+		g_free(state.fullpaths[0]);
+		*err_str = g_string_free(state.error,FALSE);
 		return FALSE;
 	}
 
-	value_strings = g_hash_table_new(g_str_hash,g_str_equal);
+	Radius_set_in(in, scanner);
 
-	BEGIN WS_OUT;
+	/* Associate the state with the scanner */
+	Radius_set_extra(&state, scanner);
 
-	yylex();
+	Radius_lex(scanner);
 
-	if (yyin != NULL) fclose(yyin);
-	yyin = NULL;
+	Radius_lex_destroy(scanner);
+
+	fclose(in);
 
-	for (i=0; i < 10; i++) {
-		if (fullpaths[i]) g_free(fullpaths[i]);
+	for (i = 0; i < MAX_INCLUDE_DEPTH; i++) {
+		if (state.fullpaths[i])
+			g_free(state.fullpaths[i]);
 	}
 
-	g_hash_table_foreach(dict->attrs_by_id,setup_attrs,NULL);
-	g_hash_table_foreach(dict->vendors_by_id,setup_vendors,NULL);
-	g_hash_table_foreach_remove(value_strings,destroy_value_strings,NULL);
+	g_hash_table_foreach(state.dict->attrs_by_id,setup_attrs,&state);
+	g_hash_table_foreach(state.dict->vendors_by_id,setup_vendors,&state);
+	g_hash_table_foreach_remove(state.value_strings,destroy_value_strings,NULL);
 
-	if (error->len > 0) {
-		*err_str = g_string_free(error,FALSE);
+	if (state.error->len > 0) {
+		*err_str = g_string_free(state.error,FALSE);
 		return FALSE;
 	} else {
 		*err_str = NULL;
-		g_string_free(error,TRUE);
+		g_string_free(state.error,TRUE);
 		return TRUE;
 	}
 }