Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

2 files changed, 277 insertions, 174 deletions

diff --git a/plugins/wimaxasncp/Makefile.am b/plugins/wimaxasncp/Makefile.am
index e4f4cd7f2f..2e265fbe6f 100644
--- a/plugins/wimaxasncp/Makefile.am
+++ b/plugins/wimaxasncp/Makefile.am
@@ -45,6 +45,10 @@ wimaxasncp_la_CFLAGS = $(GENERATED_CFLAGS) $(PLUGIN_CFLAGS)
 
 wimaxasncp_la_LDFLAGS = $(PLUGIN_LDFLAGS)
 
+BUILT_SOURCES = \
+	$(LEMON_GENERATED_HEADER_FILES) \
+	$(FLEX_GENERATED_HEADER_FILES)
+
 CLEANFILES = \
 	wimaxasncp \
 	*~
diff --git a/plugins/wimaxasncp/wimaxasncp_dict.l b/plugins/wimaxasncp/wimaxasncp_dict.l
index cec21a0eed..c34f413154 100644
--- a/plugins/wimaxasncp/wimaxasncp_dict.l
+++ b/plugins/wimaxasncp/wimaxasncp_dict.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -19,6 +24,24 @@
 %option noyywrap
 
 /*
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="WimaxasncpDict_scanner_state_t *"
+
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
+/*
  * The language we're scanning is case-insensitive.
  */
 %option caseless
@@ -29,10 +52,10 @@
 %option stack
 
 /*
- * Prefix scanner routines with "WimaxasncpDict" rather than "yy", so this
+ * Prefix scanner routines with "WimaxasncpDict_" rather than "yy", so this
  * scanner can coexist with other scanners.
  */
-%option prefix="WimaxasncpDict"
+%option prefix="WimaxasncpDict_"
 
 %option outfile="wimaxasncp_dict.c"
 
@@ -73,7 +96,6 @@
 #include <wsutil/file_util.h>
 
 #include "wimaxasncp_dict.h"
-#include "wimaxasncp_dict_lex.h"
 
 typedef struct entity_t {
 	gchar *name;
@@ -81,44 +103,74 @@ typedef struct entity_t {
 	struct entity_t *next;
 } entity_t;
 
-#define ATTR_UINT(cont) do { D(("attr_uint " #cont "\t" )); attr_uint = &(cont); yy_push_state(GET_UINT_ATTR); } while(0)
-#define ATTR_UINT16(cont) do { D(("attr_uint16 " #cont "\t" )); attr_uint16 = &(cont); yy_push_state(GET_UINT16_ATTR); } while(0)
-#define ATTR_STR(cont) do { D(("attr_str " #cont "\t" )); attr_str = &(cont); yy_push_state(GET_ATTR); } while(0)
-#define ATTR_DECODER(cont) do { D(("attr_decoder " #cont "\t" )); attr_uint = &(cont); yy_push_state(GET_DECODER_ATTR); } while(0)
-#define WIMAXASNCP_IGNORE() do { D(("ignore: %s\t",yytext)); yy_push_state(IGNORE_ATTR); } while(0)
+#define ATTR_UINT(cont) do { D(("attr_uint " #cont "\t" )); yyextra->attr_uint = &(cont); yy_push_state(GET_UINT_ATTR, yyscanner); } while(0)
+#define ATTR_UINT16(cont) do { D(("attr_uint16 " #cont "\t" )); yyextra->attr_uint16 = &(cont); yy_push_state(GET_UINT16_ATTR, yyscanner); } while(0)
+#define ATTR_STR(cont) do { D(("attr_str " #cont "\t" )); yyextra->attr_str = &(cont); yy_push_state(GET_ATTR, yyscanner); } while(0)
+#define ATTR_DECODER(cont) do { D(("attr_decoder " #cont "\t" )); yyextra->attr_uint = &(cont); yy_push_state(GET_DECODER_ATTR, yyscanner); } while(0)
+#define WIMAXASNCP_IGNORE() do { D(("ignore: %s\t",yytext)); yy_push_state(IGNORE_ATTR, yyscanner); } while(0)
 
 #define D(args) wimaxasncp_dict_debug args
 
 #define MAX_INCLUDE_DEPTH 10
-#define YY_INPUT(buf,result,max_size) { result = current_yyinput(buf,max_size); }
+#define YY_INPUT(buf,result,max_size) { result = yyextra->current_yyinput(buf,max_size,yyscanner); }
+#define YY_USER_INIT { \
+	WimaxasncpDict_scanner_state_t *scanner_state = WimaxasncpDict_get_extra(yyscanner); \
+	BEGIN(scanner_state->start_state); \
+}
 #define ECHO
-#define APPEND(txt,len) append_to_buffer(txt,(int)len)
-
-static entity_t ents;
-static YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
-static int include_stack_ptr = 0;
-static size_t (*current_yyinput)(gchar*,size_t);
-static const gchar *sys_dir;
-static wimaxasncp_dict_t *dict;
-static wimaxasncp_dict_tlv_t *tlv;
-static wimaxasncp_dict_enum_t *enumitem;
-static wimaxasncp_dict_xmlpi_t *xmlpi;
-
-static wimaxasncp_dict_tlv_t *last_tlv;
-static wimaxasncp_dict_enum_t *last_enumitem;
-static wimaxasncp_dict_xmlpi_t *last_xmlpi;
-
-static gchar **attr_str;
-static guint *attr_uint;
-static gint16 *attr_uint16;
+#define APPEND(txt,len) append_to_buffer(txt,(int)len,yyextra)
+
+typedef struct {
+	GString *dict_error;
+
+	const gchar *sys_dir;
+
+	gchar *strbuf;
+	guint size_strbuf;
+	guint len_strbuf;
+
+	gchar *write_ptr;
+	gchar *read_ptr;
+
+	wimaxasncp_dict_t *dict;
+	wimaxasncp_dict_tlv_t *tlv;
+	wimaxasncp_dict_enum_t *enumitem;
+	wimaxasncp_dict_xmlpi_t *xmlpi;
+
+	wimaxasncp_dict_tlv_t *last_tlv;
+	wimaxasncp_dict_enum_t *last_enumitem;
+	wimaxasncp_dict_xmlpi_t *last_xmlpi;
+
+	entity_t *ents;
+
+	YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
+	int include_stack_ptr;
+	size_t (*current_yyinput)(gchar*,size_t,yyscan_t);
+
+	gchar **attr_str;
+	guint *attr_uint;
+	gint16 *attr_uint16;
+
+	int start_state;
+} WimaxasncpDict_scanner_state_t;
 
 static guint wimaxasncp_bits(guint bits, char *n);
 static gint wimaxasncp_decode_type(const gchar *name);
 static void wimaxasncp_dict_debug(const gchar *fmt, ...) G_GNUC_PRINTF(1, 2);
-static void append_to_buffer(const gchar *txt, int len);
+static void append_to_buffer(const gchar *txt, int len, WimaxasncpDict_scanner_state_t *state);
 static FILE *wimaxasncp_dict_open(const gchar*, const gchar*);
 
-static GString *dict_error = NULL;
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define WimaxasncpDict_alloc(size, yyscanner)		(void *)malloc(size)
+#define WimaxasncpDict_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define WimaxasncpDict_free(ptr, yyscanner)		free((char *)ptr)
 
 %}
 
@@ -197,23 +249,23 @@ since_attr since=\042
 <LOADING>{xmlpi_start} BEGIN LOADING_XMLPI;
 <LOADING_XMLPI>{whitespace} ;
 <LOADING_XMLPI>{entityname} {
-	xmlpi = g_new(wimaxasncp_dict_xmlpi_t,1);
-	xmlpi->name = g_strdup(yytext);
-	xmlpi->key = NULL;
-	xmlpi->value = NULL;
-	xmlpi->next = NULL;
+	yyextra->xmlpi = g_new(wimaxasncp_dict_xmlpi_t,1);
+	yyextra->xmlpi->name = g_strdup(yytext);
+	yyextra->xmlpi->key = NULL;
+	yyextra->xmlpi->value = NULL;
+	yyextra->xmlpi->next = NULL;
 
-	if (!dict->xmlpis) last_xmlpi = dict->xmlpis = xmlpi;
-	else last_xmlpi = last_xmlpi->next = xmlpi;
+	if (!yyextra->dict->xmlpis) yyextra->last_xmlpi = yyextra->dict->xmlpis = yyextra->xmlpi;
+	else yyextra->last_xmlpi = yyextra->last_xmlpi->next = yyextra->xmlpi;
 
 	BEGIN XMLPI_ATTRS;
 }
 
 <XMLPI_ATTRS>{xmlpi_key_attr} BEGIN XMLPI_GETKEY;
-<XMLPI_GETKEY>{ndquot} { xmlpi->key = g_strdup(yytext); BEGIN XMLPI_ATTRS; }
+<XMLPI_GETKEY>{ndquot} { yyextra->xmlpi->key = g_strdup(yytext); BEGIN XMLPI_ATTRS; }
 
 <XMLPI_ATTRS>{xmlpi_value_attr} BEGIN XMLPI_GETVAL;
-<XMLPI_GETVAL>{ndquot} { xmlpi->value = g_strdup(yytext); BEGIN XMLPI_ATTRS; }
+<XMLPI_GETVAL>{ndquot} { yyextra->xmlpi->value = g_strdup(yytext); BEGIN XMLPI_ATTRS; }
 
 <XMLPI_ATTRS>.
 <XMLPI_ATTRS>{xmlpi_end} BEGIN LOADING;
@@ -224,14 +276,14 @@ since_attr since=\042
 	entity_t *e = g_new(entity_t,1);
 	D(("ENTITY: %s\n",yytext));
 	e->name = g_strdup(yytext);
-	e->next = ents.next;
-	ents.next = e;
+	e->next = yyextra->ents;
+	yyextra->ents = e;
 	BEGIN GET_SYSTEM;
 };
 <GET_SYSTEM>{system} BEGIN GET_FILE;
 <GET_FILE>{ndquot} {
 	D(("GET_FILE: %s\n",yytext));
-	ents.next->file = g_strdup(yytext);
+	yyextra->ents->file = g_strdup(yytext);
 	BEGIN END_ENTITY;
 }
 <END_ENTITY>{end_entity} BEGIN LOADING;
@@ -262,23 +314,23 @@ since_attr since=\042
 
 	D(("looking for entity: %s\n",yytext));
 
-	if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) {
-		dict_error = g_string_append(
-			dict_error, "included files nested too deeply\n");
+	if ( yyextra->include_stack_ptr >= MAX_INCLUDE_DEPTH ) {
+		yyextra->dict_error = g_string_append(
+			yyextra->dict_error, "included files nested too deeply\n");
 		yyterminate();
 	}
 
-	include_stack[include_stack_ptr++] = YY_CURRENT_BUFFER;
+	yyextra->include_stack[yyextra->include_stack_ptr++] = YY_CURRENT_BUFFER;
 
 
-	for (e = ents.next; e; e = e->next) {
+	for (e = yyextra->ents; e; e = e->next) {
 		if (strcmp(e->name,yytext) == 0) {
-			yyin = wimaxasncp_dict_open(sys_dir,e->file);
+			yyin = wimaxasncp_dict_open(yyextra->sys_dir,e->file);
 			D(("entity: %s filename: %s yyin: %p\n",e->name,e->file,(void*)yyin));
 			if (!yyin) {
 				yyterminate();
 			} else {
-				yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
+				WimaxasncpDict__switch_to_buffer(WimaxasncpDict__create_buffer(yyin, YY_BUF_SIZE, yyscanner), yyscanner);
 			}
 			break;
 		}
@@ -287,7 +339,7 @@ since_attr since=\042
 	if (!e) {
 		temp_str = g_strdup_printf(
 			"cannot find entity: '%s'\n", yytext);
-		dict_error = g_string_append(dict_error, temp_str);
+		yyextra->dict_error = g_string_append(yyextra->dict_error, temp_str);
 		g_free(temp_str);
 		yyterminate();
 	}
@@ -298,75 +350,75 @@ since_attr since=\042
 	if (!yyin) yyterminate();
 
 	fclose(yyin);
-	D(("closing: %p %i\n",(void*)yyin,include_stack_ptr));
+	D(("closing: %p %i\n",(void*)yyin,yyextra->include_stack_ptr));
 
-	if ( --include_stack_ptr < 0 ) {
+	if ( --yyextra->include_stack_ptr < 0 ) {
 		D(("DONE READING\n"));
 		yyin = NULL;
 		yyterminate();
 	} else {
-		yy_delete_buffer( YY_CURRENT_BUFFER );
-		yy_switch_to_buffer(include_stack[include_stack_ptr]);
+		WimaxasncpDict__delete_buffer(YY_CURRENT_BUFFER, yyscanner);
+		WimaxasncpDict__switch_to_buffer(yyextra->include_stack[yyextra->include_stack_ptr], yyscanner);
 		BEGIN LOADING;
 	}
 }
 
 
 <GET_ATTR>{ndquot} {
-	*attr_str = g_strdup(yytext);
+	*yyextra->attr_str = g_strdup(yytext);
 	D(("%s\n",yytext));
-	attr_str = NULL;
+	yyextra->attr_str = NULL;
 	BEGIN END_ATTR;
 }
 
 <GET_UINT_ATTR>{number} {
-	*attr_uint = (guint)strtoul(yytext,NULL,0);
+	*yyextra->attr_uint = (guint)strtoul(yytext,NULL,0);
 	D(("%s\n",yytext););
-	attr_uint = NULL;
+	yyextra->attr_uint = NULL;
 	BEGIN END_ATTR;
 }
 
 <GET_UINT16_ATTR>{number} {
-	*attr_uint16 = (gint16) strtol(yytext,NULL,0);
+	*yyextra->attr_uint16 = (gint16) strtol(yytext,NULL,0);
 	D(("%s\n",yytext););
-	attr_uint16 = NULL;
+	yyextra->attr_uint16 = NULL;
 	BEGIN END_ATTR;
 }
 
 <GET_UINT_ATTR>"WIMAXASNCP_BIT32"[ \t]*"("    { BEGIN BIT32; }
 
 <BIT32>[0-9]+ {
-	*attr_uint = wimaxasncp_bits(32, yytext);
+	*yyextra->attr_uint = wimaxasncp_bits(32, yytext);
 	D(("WIMAXASNCP_BIT32(%s)\n",yytext););
-	attr_uint = NULL;
+	yyextra->attr_uint = NULL;
 }
 
 <GET_UINT_ATTR>"WIMAXASNCP_BIT16"[ \t]*"("    { BEGIN BIT16; }
 
 <BIT16>[0-9]+ {
-	*attr_uint = wimaxasncp_bits(16, yytext);
+	*yyextra->attr_uint = wimaxasncp_bits(16, yytext);
 	D(("WIMAXASNCP_BIT16(%s)\n",yytext););
-	attr_uint = NULL;
+	yyextra->attr_uint = NULL;
 }
 
 <GET_UINT_ATTR>"WIMAXASNCP_BIT8"[ \t]*"("     { BEGIN BIT8; }
 
 <BIT8>[0-9]+ {
-	*attr_uint = wimaxasncp_bits(8, yytext);
+	*yyextra->attr_uint = wimaxasncp_bits(8, yytext);
 	D(("WIMAXASNCP_BIT8(%s)\n",yytext););
-	attr_uint = NULL;
+	yyextra->attr_uint = NULL;
 }
 
 <BIT32,BIT16,BIT8>[ \t]*")" { BEGIN END_ATTR; }
 
 <GET_DECODER_ATTR>{ndquot} {
-	*attr_uint = wimaxasncp_decode_type(yytext);
+	*yyextra->attr_uint = wimaxasncp_decode_type(yytext);
 	D(("%s\n",yytext));
-	attr_uint = NULL;
+	yyextra->attr_uint = NULL;
 	BEGIN END_ATTR;
 }
 
-<END_ATTR>{dquot} {	yy_pop_state(); }
+<END_ATTR>{dquot} {	yy_pop_state(yyscanner); }
 
 <IGNORE_ATTR>. {
 	/* XXX: should go?*/
@@ -375,7 +427,7 @@ since_attr since=\042
 
 <IGNORE_ATTR>{ignored_quoted} {
 	D(("=>%s<=\n",yytext));
-	yy_pop_state();
+	yy_pop_state(yyscanner);
 }
 
 <OUTSIDE>{dictionary_start} {
@@ -387,39 +439,41 @@ since_attr since=\042
 <IN_DICT>{tlv_start}	{
 	D(("tlv_start\n"));
 
-	tlv = g_new(wimaxasncp_dict_tlv_t,1);
-	tlv->type = 0;
-	tlv->name = NULL;
-	tlv->description = NULL;
-	tlv->decoder = 0;
-	tlv->since = 0;
-	tlv->hf_root = -1;
-	tlv->hf_value = -1;
-	tlv->hf_ipv4 = -1;
-	tlv->hf_ipv6 = -1;
-	tlv->hf_bsid = -1;
-	tlv->hf_protocol = -1;
-	tlv->hf_port_low = -1;
-	tlv->hf_port_high = -1;
-	tlv->hf_ipv4_mask = -1;
-	tlv->hf_ipv6_mask = -1;
-	tlv->hf_vendor_id = -1;
-	tlv->hf_vendor_rest_of_info = -1;
-	tlv->enum_vs = NULL;
-	tlv->enums = NULL;
-	tlv->next = NULL;
-
-	if (! dict->tlvs ) last_tlv = dict->tlvs = tlv;
-	else last_tlv = last_tlv->next = tlv;
+	yyextra->tlv = g_new(wimaxasncp_dict_tlv_t,1);
+	yyextra->tlv->type = 0;
+	yyextra->tlv->name = NULL;
+	yyextra->tlv->description = NULL;
+	yyextra->tlv->decoder = 0;
+	yyextra->tlv->since = 0;
+	yyextra->tlv->hf_root = -1;
+	yyextra->tlv->hf_value = -1;
+	yyextra->tlv->hf_ipv4 = -1;
+	yyextra->tlv->hf_ipv6 = -1;
+	yyextra->tlv->hf_bsid = -1;
+	yyextra->tlv->hf_protocol = -1;
+	yyextra->tlv->hf_port_low = -1;
+	yyextra->tlv->hf_port_high = -1;
+	yyextra->tlv->hf_ipv4_mask = -1;
+	yyextra->tlv->hf_ipv6_mask = -1;
+	yyextra->tlv->hf_vendor_id = -1;
+	yyextra->tlv->hf_vendor_rest_of_info = -1;
+	yyextra->tlv->enum_vs = NULL;
+	yyextra->tlv->enums = NULL;
+	yyextra->tlv->next = NULL;
+
+	if (! yyextra->dict->tlvs )
+		yyextra->last_tlv = yyextra->dict->tlvs = yyextra->tlv;
+	else
+		yyextra->last_tlv = yyextra->last_tlv->next = yyextra->tlv;
 
 	BEGIN TLV_ATTRS;
 }
 
-<TLV_ATTRS>{name_attr}			{ ATTR_STR(tlv->name); }
-<TLV_ATTRS>{description_attr}		{ ATTR_STR(tlv->description); }
-<TLV_ATTRS>{type_attr}			{ ATTR_UINT16(tlv->type); }
-<TLV_ATTRS>{decoder_attr}		{ ATTR_DECODER(tlv->decoder); }
-<TLV_ATTRS>{since_attr}			{ ATTR_UINT(tlv->since); }
+<TLV_ATTRS>{name_attr}			{ ATTR_STR(yyextra->tlv->name); }
+<TLV_ATTRS>{description_attr}		{ ATTR_STR(yyextra->tlv->description); }
+<TLV_ATTRS>{type_attr}			{ ATTR_UINT16(yyextra->tlv->type); }
+<TLV_ATTRS>{decoder_attr}		{ ATTR_DECODER(yyextra->tlv->decoder); }
+<TLV_ATTRS>{since_attr}			{ ATTR_UINT(yyextra->tlv->since); }
 <TLV_ATTRS>{stop}			{ BEGIN IN_TLV;	 }
 <TLV_ATTRS>{stop_end}			{ BEGIN IN_DICT; }
 
@@ -427,20 +481,22 @@ since_attr since=\042
 <IN_TLV>{enum_start} {
 	D(("enum_start\n"));
 
-	enumitem = g_new(wimaxasncp_dict_enum_t,1);
-	enumitem->name = NULL;
-	enumitem->code = 0;
-	enumitem->next = NULL;
+	yyextra->enumitem = g_new(wimaxasncp_dict_enum_t,1);
+	yyextra->enumitem->name = NULL;
+	yyextra->enumitem->code = 0;
+	yyextra->enumitem->next = NULL;
 
-	if (!tlv->enums) last_enumitem = tlv->enums = enumitem;
-	else last_enumitem = last_enumitem->next = enumitem;
+	if (!yyextra->tlv->enums)
+		yyextra->last_enumitem = yyextra->tlv->enums = yyextra->enumitem;
+	else
+		yyextra->last_enumitem = yyextra->last_enumitem->next = yyextra->enumitem;
 
 	BEGIN ENUM_ATTRS;
 }
 
 
-<ENUM_ATTRS>{name_attr}			{ ATTR_STR(enumitem->name); }
-<ENUM_ATTRS>{code_attr}			{ ATTR_UINT(enumitem->code); }
+<ENUM_ATTRS>{name_attr}			{ ATTR_STR(yyextra->enumitem->name); }
+<ENUM_ATTRS>{code_attr}			{ ATTR_UINT(yyextra->enumitem->code); }
 
 <ENUM_ATTRS>{stop}	     { BEGIN IN_TLV; }
 <ENUM_ATTRS>{stop_end}	     { BEGIN IN_TLV; }
@@ -526,38 +582,32 @@ static gint wimaxasncp_decode_type(const gchar *name)
 	return WIMAXASNCP_TLV_TBD;
 }
 
-static gchar *strbuf = NULL;
-static gchar *write_ptr = NULL;
-static gchar *read_ptr = NULL;
-
-static guint size_strbuf = 8192;
-static guint len_strbuf = 0;
-
-extern void wimaxasncp_dict_unused(void);
-void wimaxasncp_dict_unused(void) {
-	yy_top_state();
+extern void wimaxasncp_dict_unused(yyscan_t yyscanner);
+void wimaxasncp_dict_unused(yyscan_t yyscanner) {
+	yy_top_state(yyscanner);
 }
 
-static void append_to_buffer(const gchar *txt, int len) {
+static void append_to_buffer(const gchar *txt, int len, WimaxasncpDict_scanner_state_t *state) {
 
-	if (strbuf == NULL) {
-		read_ptr = write_ptr = strbuf = (gchar *)g_malloc(size_strbuf);
+	if (state->strbuf == NULL) {
+		state->read_ptr = state->write_ptr = state->strbuf = (gchar *)g_malloc(state->size_strbuf);
 	}
 
-	if ( (len_strbuf + len + 1) >= size_strbuf ) {
-		read_ptr = strbuf = (gchar *)g_realloc(strbuf,size_strbuf *= 2);
+	if ( (state->len_strbuf + len + 1) >= state->size_strbuf ) {
+		state->read_ptr = state->strbuf = (gchar *)g_realloc(state->strbuf,state->size_strbuf *= 2);
 	}
 
-	write_ptr = strbuf + len_strbuf;
-	strncpy(write_ptr,txt,len);
-	len_strbuf += len;
-	strbuf[len_strbuf] = '\0';
+	state->write_ptr = state->strbuf + state->len_strbuf;
+	strncpy(state->write_ptr,txt,len);
+	state->len_strbuf += len;
+	state->strbuf[state->len_strbuf] = '\0';
 }
 
-static size_t file_input(gchar *buf, size_t max) {
+static size_t file_input(gchar *buf, size_t max, yyscan_t scanner) {
+	FILE *in = yyget_in(scanner);
 	size_t read_cnt;
 
-	read_cnt = fread(buf,1,max,yyin);
+	read_cnt = fread(buf,1,max,in);
 
 	if ( read_cnt == max ) {
 		return max;
@@ -569,15 +619,17 @@ static size_t file_input(gchar *buf, size_t max) {
 }
 
 
-static size_t string_input(gchar *buf, size_t max) {
-	if (read_ptr >= write_ptr ) {
+static size_t string_input(gchar *buf, size_t max, yyscan_t scanner) {
+	WimaxasncpDict_scanner_state_t *statep = yyget_extra(scanner);
+
+	if (statep->read_ptr >= statep->write_ptr ) {
 		return YY_NULL;
-	} else if ( read_ptr + max > write_ptr ) {
-		max = write_ptr - read_ptr;
+	} else if ( statep->read_ptr + max > statep->write_ptr ) {
+		max = statep->write_ptr - statep->read_ptr;
 	}
 
-	memcpy(buf,read_ptr,max);
-	read_ptr += max;
+	memcpy(buf,statep->read_ptr,max);
+	statep->read_ptr += max;
 
 	return max;
 }
@@ -612,54 +664,106 @@ wimaxasncp_dict_t *wimaxasncp_dict_scan(
 	const gchar *system_directory, const gchar *filename, int dbg,
 	gchar **error) {
 
+	WimaxasncpDict_scanner_state_t state;
+	FILE *in;
+	yyscan_t scanner;
 	entity_t *e;
 
-	dict_error = g_string_new("");
-
 	debugging = dbg;
 
-	sys_dir = system_directory;
+	state.dict_error = g_string_new("");
 
-	write_ptr = NULL;
-	read_ptr = NULL;
+	state.sys_dir = system_directory;
 
-	if (dict)
-	{
-		wimaxasncp_dict_free(dict);
-	}
+	state.dict = g_new(wimaxasncp_dict_t,1);
+	state.dict->tlvs = NULL;
+	state.dict->xmlpis = NULL;
 
-	dict = g_new(wimaxasncp_dict_t,1);
-	dict->tlvs = NULL;
-	dict->xmlpis = NULL;
+	state.strbuf = NULL;
+	state.size_strbuf = 8192;
+	state.len_strbuf = 0;
 
-	tlv = NULL;
-	enumitem = NULL;
-	xmlpi = NULL;
+	state.write_ptr = NULL;
+	state.read_ptr = NULL;
 
-	last_tlv = NULL;
-	last_enumitem = NULL;
-	last_xmlpi = NULL;
+	state.tlv = NULL;
+	state.enumitem = NULL;
+	state.xmlpi = NULL;
 
-	ents.next = NULL;
+	state.last_tlv = NULL;
+	state.last_enumitem = NULL;
+	state.last_xmlpi = NULL;
 
-	yyin = wimaxasncp_dict_open(sys_dir,filename);
+	state.ents = NULL;
 
-	if (yyin)
-	{
-		current_yyinput = file_input;
-		BEGIN LOADING;
-		yylex();
+	/*
+	 * Pass 1.
+	 *
+	 * Reads the file, does some work, and stores a modified version
+	 * of the file contents in memory.
+	 */
+	state.current_yyinput = file_input;
+	state.include_stack_ptr = 0;
+
+	in = wimaxasncp_dict_open(system_directory,filename);
+
+	if (in == NULL) {
+		/*
+		 * Couldn't open the dictionary.
+		 *
+		 * Treat all failures other then ENOENT as errors?
+		 */
+		*error = NULL;
+		return state.dict;
+	}
+
+	if (WimaxasncpDict_lex_init(&scanner) != 0) {
+		D(("Can't initialize scanner: %s\n", strerror(errno)));
+		fclose(in);
+		g_free(state.dict);
+		return NULL;
+	}
+
+	WimaxasncpDict_set_in(in, scanner);
 
-		D(("\n---------------\n%s\n------- %d -------\n",
-		   strbuf, len_strbuf));
+	/* Associate the state with the scanner */
+	WimaxasncpDict_set_extra(&state, scanner);
 
-		current_yyinput = string_input;
+	state.start_state = LOADING;
+	WimaxasncpDict_lex(scanner);
 
-		BEGIN OUTSIDE;
-		yylex();
+	WimaxasncpDict_lex_destroy(scanner);
+	fclose(in);
+
+	D(("\n---------------\n%s\n------- %d -------\n",
+	   state.strbuf, state.len_strbuf));
+
+	/*
+	 * Pass 2.
+	 *
+	 * Reads the modified version of the file contents and does the
+	 * rest of the work.
+	 */
+	state.current_yyinput = string_input;
+
+	if (WimaxasncpDict_lex_init(&scanner) != 0) {
+		D(("Can't initialize scanner: %s\n", strerror(errno)));
+		fclose(in);
+		g_free(state.dict);
+		g_free(state.strbuf);
+		return NULL;
 	}
 
-	e = ents.next;
+	/* Associate the state with the scanner */
+	WimaxasncpDict_set_extra(&state, scanner);
+
+	state.start_state = OUTSIDE;
+	WimaxasncpDict_lex(scanner);
+
+	WimaxasncpDict_lex_destroy(scanner);
+	g_free(state.strbuf);
+
+	e = state.ents;
 	while (e)
 	{
 		entity_t *next = e->next;
@@ -669,21 +773,16 @@ wimaxasncp_dict_t *wimaxasncp_dict_scan(
 		e = next;
 	}
 
-	g_free(strbuf);
-	strbuf = NULL;
-	size_strbuf = 8192;
-
-	if (dict_error->len > 0)
+	if (state.dict_error->len > 0)
 	{
-		*error = g_string_free(dict_error, FALSE);
+		*error = g_string_free(state.dict_error, FALSE);
 	}
 	else
 	{
 		*error = NULL;
-		g_string_free(dict_error, TRUE);
+		g_string_free(state.dict_error, TRUE);
 	}
-
-	return dict;
+	return state.dict;
 }
 
 void wimaxasncp_dict_free(wimaxasncp_dict_t *d) {