Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

1 files changed, 329 insertions, 208 deletions

diff --git a/epan/diam_dict.l b/epan/diam_dict.l
index e97ca8d821..3c450465dc 100644
--- a/epan/diam_dict.l
+++ b/epan/diam_dict.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -29,13 +34,31 @@
 %option noyywrap
 
 /*
- * Prefix scanner routines with "DiamDict" rather than "yy", so this scanner
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="DiamDict_scanner_state_t *"
+
+/*
+ * Prefix scanner routines with "DiamDict_" rather than "yy", so this scanner
  * can coexist with other scanners.
  */
-%option prefix="DiamDict"
+%option prefix="DiamDict_"
 
 %option outfile="diam_dict.c"
 
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
 %{
 	/*
 	 ** diam_dict.h
@@ -68,7 +91,6 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include "diam_dict.h"
-#include "diam_dict_lex.h"
 #include <epan/to_str.h>
 #include <wsutil/file_util.h>
 
@@ -78,48 +100,80 @@ typedef struct entity_t {
 	struct entity_t* next;
 } entity_t;
 
-#define ATTR_UINT(cont) do { D(("attr_uint " #cont "\t" )); attr_uint = &(cont); yy_push_state(GET_UINT_ATTR); } while(0)
-#define ATTR_STR(cont) do { D(("attr_str " #cont "\t" )); attr_str = &(cont); yy_push_state(GET_ATTR); } while(0)
-#define IGNORE() do { D(("ignore: %s\t",yytext)); yy_push_state(IGNORE_ATTR); } while(0)
+#define ATTR_UINT(cont) do { D(("attr_uint " #cont "\t" )); yyextra->attr_uint = &(cont); yy_push_state(GET_UINT_ATTR, yyscanner); } while(0)
+#define ATTR_STR(cont) do { D(("attr_str " #cont "\t" )); yyextra->attr_str = &(cont); yy_push_state(GET_ATTR, yyscanner); } while(0)
+#define IGNORE() do { D(("ignore: %s\t",yytext)); yy_push_state(IGNORE_ATTR, yyscanner); } while(0)
 
 #define D(args) ddict_debug args
 
 #define MAX_INCLUDE_DEPTH 10
-#define YY_INPUT(buf,result,max_size) { result = current_yyinput(buf,max_size); }
+#define YY_INPUT(buf,result,max_size) { result = yyextra->current_yyinput(buf,max_size,yyscanner); }
+#define YY_USER_INIT { \
+	DiamDict_scanner_state_t *scanner_state = DiamDict_get_extra(yyscanner); \
+	BEGIN(scanner_state->start_state); \
+}
 #define ECHO
-#define APPEND(txt,len) append_to_buffer(txt,len)
-
-static entity_t ents;
-static YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
-static int include_stack_ptr = 0;
-static size_t (*current_yyinput)(char*,size_t);
-static const char* sys_dir;
-static ddict_t* dict;
-static ddict_application_t* appl;
-static ddict_avp_t* avp;
-static ddict_enum_t* enumitem;
-static ddict_gavp_t* gavp;
-static ddict_typedefn_t* typedefn;
-static ddict_cmd_t* cmd;
-static ddict_vendor_t* vnd;
-static ddict_xmlpi_t* xmlpi;
-
-static ddict_application_t* last_appl;
-static ddict_avp_t* last_avp;
-static ddict_enum_t* last_enumitem;
-static ddict_gavp_t* last_gavp;
-static ddict_typedefn_t* last_typedefn;
-static ddict_cmd_t* last_cmd;
-static ddict_vendor_t* last_vnd;
-static ddict_xmlpi_t* last_xmlpi;
-
-static char** attr_str;
-static unsigned* attr_uint;
+#define APPEND(txt,len) append_to_buffer(txt,len,yyextra)
+
+typedef struct {
+	const char* sys_dir;
+
+	char* write_ptr;
+	char* read_ptr;
+
+	char* strbuf;
+	unsigned size_strbuf;
+	unsigned len_strbuf;
+
+	ddict_t* dict;
+
+	ddict_application_t* appl;
+	ddict_avp_t* avp;
+	ddict_enum_t* enumitem;
+	ddict_gavp_t* gavp;
+	ddict_typedefn_t* typedefn;
+	ddict_cmd_t* cmd;
+	ddict_vendor_t* vnd;
+	ddict_xmlpi_t* xmlpi;
+
+	ddict_application_t* last_appl;
+	ddict_avp_t* last_avp;
+	ddict_enum_t* last_enumitem;
+	ddict_gavp_t* last_gavp;
+	ddict_typedefn_t* last_typedefn;
+	ddict_cmd_t* last_cmd;
+	ddict_vendor_t* last_vnd;
+	ddict_xmlpi_t* last_xmlpi;
+
+	entity_t *ents;
+
+	char** attr_str;
+	unsigned* attr_uint;
+
+	size_t (*current_yyinput)(char*,size_t,yyscan_t);
+
+	YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
+	int include_stack_ptr;
+
+	int start_state;
+} DiamDict_scanner_state_t;
 
 static void ddict_debug(const char* fmt, ...) G_GNUC_PRINTF(1, 2);
-static void append_to_buffer(const char* txt, unsigned len);
+static void append_to_buffer(const char* txt, unsigned len, DiamDict_scanner_state_t *statep);
 static FILE* ddict_open(const char*, const char*);
 
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define DiamDict_alloc(size, yyscanner)		(void *)malloc(size)
+#define DiamDict_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define DiamDict_free(ptr, yyscanner)		free((char *)ptr)
+
 %}
 
 
@@ -215,23 +269,25 @@ description_attr description=\042
 <LOADING>{xmlpi_start} BEGIN LOADING_XMLPI;
 <LOADING_XMLPI>{whitespace} ;
 <LOADING_XMLPI>{entityname} {
-	xmlpi = g_new(ddict_xmlpi_t,1);
-	xmlpi->name = g_strdup(yytext);
-	xmlpi->key = NULL;
-	xmlpi->value = NULL;
-	xmlpi->next = NULL;
+	yyextra->xmlpi = g_new(ddict_xmlpi_t,1);
+	yyextra->xmlpi->name = g_strdup(yytext);
+	yyextra->xmlpi->key = NULL;
+	yyextra->xmlpi->value = NULL;
+	yyextra->xmlpi->next = NULL;
 
-	if (!dict->xmlpis) last_xmlpi = dict->xmlpis = xmlpi;
-	else last_xmlpi = last_xmlpi->next = xmlpi;
+	if (!yyextra->dict->xmlpis)
+		yyextra->last_xmlpi = yyextra->dict->xmlpis = yyextra->xmlpi;
+	else
+		yyextra->last_xmlpi = yyextra->last_xmlpi->next = yyextra->xmlpi;
 
 	BEGIN XMLPI_ATTRS;
 }
 
 <XMLPI_ATTRS>{xmlpi_key_attr} BEGIN XMLPI_GETKEY;
-<XMLPI_GETKEY>{ndquot} { xmlpi->key = strdup(yytext); BEGIN XMLPI_ATTRS; }
+<XMLPI_GETKEY>{ndquot} { yyextra->xmlpi->key = strdup(yytext); BEGIN XMLPI_ATTRS; }
 
 <XMLPI_ATTRS>{xmlpi_value_attr} BEGIN XMLPI_GETVAL;
-<XMLPI_GETVAL>{ndquot} { xmlpi->value = strdup(yytext); BEGIN XMLPI_ATTRS; }
+<XMLPI_GETVAL>{ndquot} { yyextra->xmlpi->value = strdup(yytext); BEGIN XMLPI_ATTRS; }
 
 <XMLPI_ATTRS>.
 <XMLPI_ATTRS>{xmlpi_end} BEGIN LOADING;
@@ -241,13 +297,13 @@ description_attr description=\042
 <ENTITY>{entityname} {
 	entity_t* e = g_new(entity_t,1);
 	e->name = strdup(yytext);
-	e->next = ents.next;
-	ents.next = e;
+	e->next = yyextra->ents;
+	yyextra->ents = e;
 	BEGIN GET_SYSTEM;
 	};
 <GET_SYSTEM>{system} BEGIN GET_FILE;
 <GET_FILE>{ndquot} {
-		ents.next->file = strdup(yytext);
+		yyextra->ents->file = strdup(yytext);
 		BEGIN END_ENTITY;
 	}
 <END_ENTITY>{end_entity} BEGIN LOADING;
@@ -278,14 +334,14 @@ description_attr description=\042
 
 	D(("looking for entity: %s\n",yytext));
 
-	if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) {
+	if ( yyextra->include_stack_ptr >= MAX_INCLUDE_DEPTH ) {
 		fprintf(stderr, "included files nested to deeply\n");
 		yyterminate();
 	}
 
-	for (e = ents.next; e; e = e->next) {
+	for (e = yyextra->ents; e; e = e->next) {
 		if (strcmp(e->name,yytext) == 0) {
-			yyin = ddict_open(sys_dir,e->file);
+			yyin = ddict_open(yyextra->sys_dir,e->file);
 			D(("entity: %s filename: %s yyin: %p\n",e->name,e->file,(void*)yyin));
 			if (!yyin) {
 				if (errno)
@@ -294,8 +350,8 @@ description_attr description=\042
 					fprintf(stderr, "Could not open file: '%s', error unknown (errno == 0)\n", e->file );
 				yyterminate();
 			} else {
-				include_stack[include_stack_ptr++] = YY_CURRENT_BUFFER;
-				yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
+				yyextra->include_stack[yyextra->include_stack_ptr++] = YY_CURRENT_BUFFER;
+				yy_switch_to_buffer(yy_create_buffer(yyin, YY_BUF_SIZE, yyscanner), yyscanner);
 				BEGIN LOADING;
 			}
 			break;
@@ -313,35 +369,35 @@ description_attr description=\042
 	if (!yyin) yyterminate();
 
 	fclose(yyin);
-	D(("closing: %p %i\n",(void*)yyin,include_stack_ptr));
+	D(("closing: %p %i\n",(void*)yyin,yyextra->include_stack_ptr));
 
-	if ( --include_stack_ptr < 0 ) {
+	if ( --yyextra->include_stack_ptr < 0 ) {
 		D(("DONE READING\n"));
 		yyin = NULL;
 		yyterminate();
 	} else {
-		yy_delete_buffer( YY_CURRENT_BUFFER );
-		yy_switch_to_buffer(include_stack[include_stack_ptr]);
+		yy_delete_buffer( YY_CURRENT_BUFFER, yyscanner);
+		yy_switch_to_buffer(yyextra->include_stack[yyextra->include_stack_ptr], yyscanner);
 		BEGIN LOADING;
 	}
 }
 
 
 <GET_ATTR>{ndquot} {
-	*attr_str = strdup(yytext);
+	*(yyextra->attr_str) = strdup(yytext);
 	D(("%s\n",yytext));
-	attr_str = NULL;
+	yyextra->attr_str = NULL;
 	BEGIN END_ATTR;
 }
 
 <GET_UINT_ATTR>{number} {
-	*attr_uint = (unsigned) strtoul(yytext,NULL,10);
+	*(yyextra->attr_uint) = (unsigned) strtoul(yytext,NULL,10);
 	D(("%s\n",yytext););
-	attr_uint = NULL;
+	yyextra->attr_uint = NULL;
 	BEGIN END_ATTR;
 }
 
-<END_ATTR>{dquot} {	yy_pop_state(); }
+<END_ATTR>{dquot} {	yy_pop_state(yyscanner); }
 
 <IGNORE_ATTR>. {
 	/* XXX: should go?*/
@@ -350,7 +406,7 @@ description_attr description=\042
 
 <IGNORE_ATTR>{ignored_quoted} {
 	D(("=>%s<=\n",yytext));
-	yy_pop_state();
+	yy_pop_state(yyscanner);
 }
 
 <OUTSIDE>{dictionary_start} {
@@ -367,19 +423,21 @@ description_attr description=\042
 <IN_DICT>{application_start} {
 	D(("application_start\n"));
 
-	appl = g_new(ddict_application_t,1);
-	appl->name = NULL;
-	appl->code = 0;
-	appl->next = NULL;
+	yyextra->appl = g_new(ddict_application_t,1);
+	yyextra->appl->name = NULL;
+	yyextra->appl->code = 0;
+	yyextra->appl->next = NULL;
 
-	if (!dict->applications) last_appl = dict->applications = appl;
-	else last_appl = last_appl->next = appl;
+	if (!yyextra->dict->applications)
+		yyextra->last_appl = yyextra->dict->applications = yyextra->appl;
+	else
+		yyextra->last_appl = yyextra->last_appl->next = yyextra->appl;
 
 	BEGIN APPL_ATTRS;
 }
 
-<APPL_ATTRS>{name_attr} { ATTR_STR(appl->name); }
-<APPL_ATTRS>{id_attr} { ATTR_UINT(appl->code); }
+<APPL_ATTRS>{name_attr} { ATTR_STR(yyextra->appl->name); }
+<APPL_ATTRS>{id_attr} { ATTR_UINT(yyextra->appl->code); }
 
 <APPL_ATTRS>{stop} BEGIN IN_APPL;
 <APPL_ATTRS>{stop_end} BEGIN IN_DICT;
@@ -389,60 +447,66 @@ description_attr description=\042
 <IN_APPL>{command_start}		{
 	D(("command_start\n"));
 
-	cmd = g_new(ddict_cmd_t,1);
-	cmd->name = NULL;
-	cmd->vendor = NULL;
-	cmd->code = 0;
-	cmd->next = NULL;
+	yyextra->cmd = g_new(ddict_cmd_t,1);
+	yyextra->cmd->name = NULL;
+	yyextra->cmd->vendor = NULL;
+	yyextra->cmd->code = 0;
+	yyextra->cmd->next = NULL;
 
-	if (!dict->cmds) last_cmd = dict->cmds = cmd;
-	else last_cmd = last_cmd->next = cmd;
+	if (!yyextra->dict->cmds)
+		yyextra->last_cmd = yyextra->dict->cmds = yyextra->cmd;
+	else
+		yyextra->last_cmd = yyextra->last_cmd->next = yyextra->cmd;
 
 	BEGIN COMMAND_ATTRS;
 }
 
-<COMMAND_ATTRS>{name_attr}		{ ATTR_STR(cmd->name); }
-<COMMAND_ATTRS>{vendor_attr}		{ ATTR_STR(cmd->vendor); }
-<COMMAND_ATTRS>{code_attr}		{ ATTR_UINT(cmd->code); }
+<COMMAND_ATTRS>{name_attr}		{ ATTR_STR(yyextra->cmd->name); }
+<COMMAND_ATTRS>{vendor_attr}		{ ATTR_STR(yyextra->cmd->vendor); }
+<COMMAND_ATTRS>{code_attr}		{ ATTR_UINT(yyextra->cmd->code); }
 <COMMAND_ATTRS>{stop}			|
 <COMMAND_ATTRS>{stop_end}		{ BEGIN IN_APPL; }
 
 <IN_DICT>{vendor_start} {
 	D(("vendor_start\n"));
 
-	vnd = g_new(ddict_vendor_t,1);
-	vnd->name = NULL;
-	vnd->code = 0;
-	vnd->next = NULL;
+	yyextra->vnd = g_new(ddict_vendor_t,1);
+	yyextra->vnd->name = NULL;
+	yyextra->vnd->code = 0;
+	yyextra->vnd->next = NULL;
 
-	if (!dict->vendors) last_vnd = dict->vendors = vnd;
-	else last_vnd = last_vnd->next = vnd;
+	if (!yyextra->dict->vendors)
+		yyextra->last_vnd = yyextra->dict->vendors = yyextra->vnd;
+	else
+		yyextra->last_vnd = yyextra->last_vnd->next = yyextra->vnd;
 
 	BEGIN VENDOR_ATTRS;
 }
 
-<VENDOR_ATTRS>{name_attr}		{ ATTR_STR(vnd->desc); }
-<VENDOR_ATTRS>{vendor_attr}		{ ATTR_STR(vnd->name); }
-<VENDOR_ATTRS>{code_attr}		{ ATTR_UINT(vnd->code); }
+<VENDOR_ATTRS>{name_attr}		{ ATTR_STR(yyextra->vnd->desc); }
+<VENDOR_ATTRS>{vendor_attr}		{ ATTR_STR(yyextra->vnd->name); }
+<VENDOR_ATTRS>{code_attr}		{ ATTR_UINT(yyextra->vnd->code); }
 <VENDOR_ATTRS>{stop}			{ BEGIN IN_APPL; }
 <VENDOR_ATTRS>{stop_end}		{ BEGIN IN_DICT; }
 
 <IN_APPL>{typedefn_start} {
 	D(("typedefn_start\n"));
 
-	typedefn = g_new(ddict_typedefn_t,1);
-	typedefn->name = NULL;
-	typedefn->parent = NULL;
-	typedefn->next = NULL;
+	yyextra->typedefn = g_new(ddict_typedefn_t,1);
+	yyextra->typedefn->name = NULL;
+	yyextra->typedefn->parent = NULL;
+	yyextra->typedefn->next = NULL;
 
-	if (!dict->typedefns) last_typedefn = dict->typedefns = typedefn;
-	else last_typedefn = last_typedefn->next = typedefn;
+	if (!yyextra->dict->typedefns)
+		yyextra->last_typedefn = yyextra->dict->typedefns = yyextra->typedefn;
+	else
+		yyextra->last_typedefn = yyextra->last_typedefn->next = yyextra->typedefn;
 
 	BEGIN TYPEDEFN_ATTRS;
 }
 
-<TYPEDEFN_ATTRS>{typename_attr}		{ ATTR_STR(typedefn->name); }
-<TYPEDEFN_ATTRS>{typeparent_attr}	{ ATTR_STR(typedefn->parent); }
+<TYPEDEFN_ATTRS>{typename_attr}		{ ATTR_STR(yyextra->typedefn->name); }
+<TYPEDEFN_ATTRS>{typeparent_attr}	{ ATTR_STR(yyextra->typedefn->parent); }
 <TYPEDEFN_ATTRS>{stop}			|
 <TYPEDEFN_ATTRS>{stop_end}		{ BEGIN IN_APPL; }
 
@@ -450,71 +514,77 @@ description_attr description=\042
 <IN_APPL>{avp_start}	{
 	D(("avp_start\n"));
 
-	avp = g_new(ddict_avp_t,1);
-	avp->name = NULL;
-	avp->description = NULL;
-	avp->vendor = NULL;
-	avp->code = 0;
-	avp->type = NULL;
-	avp->enums = NULL;
-	avp->gavps = NULL;
-	avp->next = NULL;
-
-	if (! dict->avps ) last_avp = dict->avps = avp;
-	else last_avp = last_avp->next = avp;
+	yyextra->avp = g_new(ddict_avp_t,1);
+	yyextra->avp->name = NULL;
+	yyextra->avp->description = NULL;
+	yyextra->avp->vendor = NULL;
+	yyextra->avp->code = 0;
+	yyextra->avp->type = NULL;
+	yyextra->avp->enums = NULL;
+	yyextra->avp->gavps = NULL;
+	yyextra->avp->next = NULL;
+
+	if (! yyextra->dict->avps )
+		yyextra->last_avp = yyextra->dict->avps = yyextra->avp;
+	else
+		yyextra->last_avp = yyextra->last_avp->next = yyextra->avp;
 
 	BEGIN AVP_ATTRS;
 }
 
-<AVP_ATTRS>{name_attr}			{ ATTR_STR(avp->name); }
-<AVP_ATTRS>{description_attr}		{ ATTR_STR(avp->description); }
-<AVP_ATTRS>{vendor_attr}		{ ATTR_STR(avp->vendor); }
-<AVP_ATTRS>{code_attr}			{ ATTR_UINT(avp->code); }
+<AVP_ATTRS>{name_attr}			{ ATTR_STR(yyextra->avp->name); }
+<AVP_ATTRS>{description_attr}		{ ATTR_STR(yyextra->avp->description); }
+<AVP_ATTRS>{vendor_attr}		{ ATTR_STR(yyextra->avp->vendor); }
+<AVP_ATTRS>{code_attr}			{ ATTR_UINT(yyextra->avp->code); }
 <AVP_ATTRS>{stop}			{ BEGIN IN_AVP;  }
 <AVP_ATTRS>{stop_end}			{ BEGIN IN_APPL; }
 
 
-<IN_AVP>{grouped_start} { avp->type = strdup("Grouped"); };
+<IN_AVP>{grouped_start} { yyextra->avp->type = strdup("Grouped"); };
 <IN_AVP>{grouped_end} ;
 
 <IN_AVP>{type_start} { BEGIN TYPE_ATTRS; }
-<TYPE_ATTRS>{typename_attr}			{ ATTR_STR(avp->type); }
+<TYPE_ATTRS>{typename_attr}			{ ATTR_STR(yyextra->avp->type); }
 
 <IN_AVP>{gavp_start} {
 	D(("gavp_start\n"));
 
-	gavp = g_new(ddict_gavp_t,1);
-	gavp->name = NULL;
-	gavp->code = 0;
-	gavp->next = NULL;
+	yyextra->gavp = g_new(ddict_gavp_t,1);
+	yyextra->gavp->name = NULL;
+	yyextra->gavp->code = 0;
+	yyextra->gavp->next = NULL;
 
-	if (!avp->gavps) last_gavp = avp->gavps = gavp;
-	else last_gavp = last_gavp->next = gavp;
+	if (!yyextra->avp->gavps)
+		yyextra->last_gavp = yyextra->avp->gavps = yyextra->gavp;
+	else
+		yyextra->last_gavp = yyextra->last_gavp->next = yyextra->gavp;
 
 	BEGIN GAVP_ATTRS;
 }
 
 
-<GAVP_ATTRS>{name_attr}			{ ATTR_STR(gavp->name); }
+<GAVP_ATTRS>{name_attr}			{ ATTR_STR(yyextra->gavp->name); }
 
 
 <IN_AVP>{enum_start} {
 	D(("enum_start\n"));
 
-	enumitem = g_new(ddict_enum_t,1);
-	enumitem->name = NULL;
-	enumitem->code = 0;
-	enumitem->next = NULL;
+	yyextra->enumitem = g_new(ddict_enum_t,1);
+	yyextra->enumitem->name = NULL;
+	yyextra->enumitem->code = 0;
+	yyextra->enumitem->next = NULL;
 
-	if (!avp->enums) last_enumitem = avp->enums = enumitem;
-	else last_enumitem = last_enumitem->next = enumitem;
+	if (!yyextra->avp->enums)
+		yyextra->last_enumitem = yyextra->avp->enums = yyextra->enumitem;
+	else
+		yyextra->last_enumitem = yyextra->last_enumitem->next = yyextra->enumitem;
 
 	BEGIN ENUM_ATTRS;
 }
 
 
-<ENUM_ATTRS>{name_attr}			{ ATTR_STR(enumitem->name); }
-<ENUM_ATTRS>{code_attr}			{ ATTR_UINT(enumitem->code); }
+<ENUM_ATTRS>{name_attr}			{ ATTR_STR(yyextra->enumitem->name); }
+<ENUM_ATTRS>{code_attr}			{ ATTR_UINT(yyextra->enumitem->code); }
 
 <TYPE_ATTRS,GAVP_ATTRS,ENUM_ATTRS>{stop}		{ BEGIN IN_AVP; }
 <TYPE_ATTRS,GAVP_ATTRS,ENUM_ATTRS>{stop_end}		{ BEGIN IN_AVP; }
@@ -561,45 +631,44 @@ static void ddict_debug(const char* fmt, ...) {
 	fflush(stderr);
 }
 
+/*
+ * Sleazy hack to avoid unused function warnings for yy_top_state.
+ */
+extern void ddict_unused(yyscan_t yyscanner);
 
-static char* strbuf = NULL;
-static char* write_ptr = NULL;
-static char* read_ptr = NULL;
-
-static unsigned size_strbuf = 8192;
-static unsigned len_strbuf = 0;
-
-extern void ddict_unused(void);
 void
-ddict_unused(void)
+ddict_unused(yyscan_t yyscanner)
 {
-	yy_top_state();
+	yy_top_state(yyscanner);
 }
 
 static void
-append_to_buffer(const char* txt, unsigned len)
+append_to_buffer(const char* txt, unsigned len, DiamDict_scanner_state_t *statep)
 {
 
-	if (strbuf == NULL) {
-		read_ptr = write_ptr = strbuf = (char*)g_malloc(size_strbuf);
+	if (statep->strbuf == NULL) {
+		statep->strbuf = (char*)g_malloc(statep->size_strbuf);
+		statep->read_ptr = statep->strbuf;
+		statep->write_ptr = statep->strbuf;
 	}
 
-	if ( (len_strbuf + len) >= size_strbuf ) {
-		read_ptr = strbuf = (char*)g_realloc(strbuf,size_strbuf *= 2);
+	if ( (statep->len_strbuf + len) >= statep->size_strbuf ) {
+		statep->strbuf = (char*)g_realloc(statep->strbuf,statep->size_strbuf *= 2);
+		statep->read_ptr = statep->strbuf;
 	}
 
-	write_ptr = strbuf + len_strbuf;
-	strncpy(write_ptr,txt,len);
-	len_strbuf += len;
-
+	statep->write_ptr = statep->strbuf + statep->len_strbuf;
+	strncpy(statep->write_ptr,txt,len);
+	statep->len_strbuf += len;
 }
 
 static size_t
-file_input(char* buf, size_t max)
+file_input(char* buf, size_t max, yyscan_t scanner)
 {
+	FILE *in = yyget_in(scanner);
 	size_t read_cnt;
 
-	read_cnt = fread(buf,1,max,yyin);
+	read_cnt = fread(buf,1,max,in);
 
 	if ( read_cnt == max ) {
 		return max;
@@ -612,16 +681,18 @@ file_input(char* buf, size_t max)
 
 
 static size_t
-string_input(char* buf, size_t max)
+string_input(char* buf, size_t max, yyscan_t scanner)
 {
-	if (read_ptr >= write_ptr ) {
+	DiamDict_scanner_state_t *statep = yyget_extra(scanner);
+
+	if (statep->read_ptr >= statep->write_ptr ) {
 		return YY_NULL;
-	} else if ( read_ptr + max > write_ptr ) {
-		max = write_ptr - read_ptr;
+	} else if ( statep->read_ptr + max > statep->write_ptr ) {
+		max = statep->write_ptr - statep->read_ptr;
 	}
 
-	memcpy(buf,read_ptr,max);
-	read_ptr += max;
+	memcpy(buf,statep->read_ptr,max);
+	statep->read_ptr += max;
 
 	return max;
 }
@@ -651,64 +722,114 @@ ddict_open(const char* system_directory, const char* filename)
 ddict_t *
 ddict_scan(const char* system_directory, const char* filename, int dbg)
 {
+	DiamDict_scanner_state_t state;
+	FILE *in;
+	yyscan_t scanner;
 
 	debugging = dbg;
 
-	sys_dir = system_directory;
+	state.sys_dir = system_directory;
+
+	state.write_ptr = NULL;
+	state.read_ptr = NULL;
+
+	state.strbuf = NULL;
+	state.size_strbuf = 8192;
+	state.len_strbuf = 0;
+
+	state.dict = g_new(ddict_t,1);
+	state.dict->applications = NULL;
+	state.dict->cmds = NULL;
+	state.dict->vendors = NULL;
+	state.dict->typedefns = NULL;
+	state.dict->avps = NULL;
+	state.dict->xmlpis = NULL;
+
+	state.appl = NULL;
+	state.avp = NULL;
+	state.enumitem = NULL;
+	state.gavp = NULL;
+	state.typedefn = NULL;
+	state.cmd = NULL;
+	state.vnd = NULL;
+	state.xmlpi = NULL;
+
+	state.last_appl = NULL;
+	state.last_avp = NULL;
+	state.last_enumitem = NULL;
+	state.last_gavp = NULL;
+	state.last_typedefn = NULL;
+	state.last_cmd = NULL;
+	state.last_vnd = NULL;
+	state.last_xmlpi = NULL;
+
+	state.ents = NULL;
+
+	state.attr_str = NULL;
+	state.attr_uint = NULL;
 
-	yyin = ddict_open(sys_dir,filename);
+	/*
+	 * Pass 1.
+	 *
+	 * Reads the file, does some work, and stores a modified version
+	 * of the file contents in memory.
+	 */
+	state.current_yyinput = file_input;
+	state.include_stack_ptr = 0;
 
-	if (yyin == NULL) {
-		D(("unable to open %s\n", filename));
+	in = ddict_open(system_directory,filename);
+
+	if (in == NULL) {
+		D(("unable to open %s: %s\n", filename, g_strerror(errno)));
+		g_free(state.dict);
 		return NULL;
 	}
 
-	write_ptr = NULL;
-	read_ptr = NULL;
-
-	dict = g_new(ddict_t,1);
-	dict->applications = NULL;
-	dict->cmds = NULL;
-	dict->vendors = NULL;
-	dict->typedefns = NULL;
-	dict->avps = NULL;
-	dict->xmlpis = NULL;
-
-	appl = NULL;
-	avp = NULL;
-	enumitem = NULL;
-	gavp = NULL;
-	typedefn = NULL;
-	cmd = NULL;
-	vnd = NULL;
-	xmlpi = NULL;
-
-	last_appl = NULL;
-	last_avp = NULL;
-	last_enumitem = NULL;
-	last_gavp = NULL;
-	last_typedefn = NULL;
-	last_cmd = NULL;
-	last_vnd = NULL;
-	last_xmlpi = NULL;
-
-	ents.next = NULL;
-	current_yyinput = file_input;
-	BEGIN LOADING;
-	yylex();
-
-	D(("\n---------------\n%s\n------- %u -------\n",strbuf,len_strbuf));
-
-	current_yyinput = string_input;
-
-	BEGIN OUTSIDE;
-	yylex();
-
-	g_free(strbuf);
-	strbuf = NULL;
-	size_strbuf = 8192;
-
-	return dict;
+	if (DiamDict_lex_init(&scanner) != 0) {
+		D(("Can't initialize scanner: %s\n", g_strerror(errno)));
+		fclose(in);
+		g_free(state.dict);
+		return NULL;
+	}
+
+	DiamDict_set_in(in, scanner);
+
+	/* Associate the state with the scanner */
+	DiamDict_set_extra(&state, scanner);
+
+	state.start_state = LOADING;
+	DiamDict_lex(scanner);
+
+	DiamDict_lex_destroy(scanner);
+	fclose(in);
+
+	D(("\n---------------\n%s\n------- %u -------\n",state.strbuf,state.len_strbuf));
+
+	/*
+	 * Pass 2.
+	 *
+	 * Reads the modified version of the file contents and does the
+	 * rest of the work.
+	 */
+	state.current_yyinput = string_input;
+
+	if (DiamDict_lex_init(&scanner) != 0) {
+		D(("Can't initialize scanner: %s\n", g_strerror(errno)));
+		g_free(state.dict);
+		g_free(state.strbuf);
+		return NULL;
+	}
+
+	/* Associate the state with the scanner */
+	DiamDict_set_extra(&state, scanner);
+
+	state.start_state = OUTSIDE;
+	DiamDict_lex(scanner);
+
+	DiamDict_lex_destroy(scanner);
+	g_free(state.strbuf);
+
+	return state.dict;
 }
 
 void