Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

1 files changed, 110 insertions, 55 deletions

diff --git a/epan/dtd_preparse.l b/epan/dtd_preparse.l
index 5188fbae02..f51f1c7f7a 100644
--- a/epan/dtd_preparse.l
+++ b/epan/dtd_preparse.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -19,6 +24,11 @@
 %option noyywrap
 
 /*
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="Dtd_PreParse_scanner_state_t *"
+
+/*
  * The language we're scanning is case-insensitive.
  */
 %option caseless
@@ -31,14 +41,27 @@
 
 %option outfile="dtd_preparse.c"
 
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
 %{
 	/*
-	 * dtd_preparser.l
+	 * dtd_preparse.l
 	 *
 	 * an XML dissector for wireshark
 	 *
 	 * DTD Preparser -  import a dtd file into a GString
-	 *					including files, removing comments
+	 *		    including files, removing comments
 	 *                  and resolving %entities;
 	 *
 	 * Copyright 2004, Luis E. Garcia Ontanon <luis@ontanon.org>
@@ -69,23 +92,29 @@
 #include <errno.h>
 #include <stdio.h>
 #include "dtd.h"
-#include "dtd_preparse_lex.h"
 #include <wsutil/file_util.h>
 
-#define ECHO g_string_append(current,yytext);
+#define ECHO g_string_append(yyextra->current,yytext);
 
-static GString* current;
-static GString* output;
-static GHashTable* entities;
-static gchar* entity_name;
-static GString* error;
+typedef struct {
+	const gchar* dtd_dirname;
+	const gchar* filename;
+	guint linenum;
 
-static const gchar* dtd_dirname;
-static const gchar* filename;
-static guint linenum;
+	GString* error;
 
-static const gchar* replace_entity(gchar* s);
-static const gchar* location(void);
+	GHashTable* entities;
+	GString* current;
+	GString* output;
+	gchar* entity_name;
+} Dtd_PreParse_scanner_state_t;
+
+static const gchar* replace_entity(Dtd_PreParse_scanner_state_t* state, gchar* s);
+static const gchar* location(Dtd_PreParse_scanner_state_t* state);
+
+#define YY_USER_INIT { \
+	BEGIN OUTSIDE; \
+}
 
 /*
  * Flex (v 2.5.35) uses this symbol to "exclude" unistd.h
@@ -100,6 +129,18 @@ static const gchar* location(void);
 #pragma warning (disable:4018)
 #endif
 
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define Dtd_PreParse_alloc(size, yyscanner)		(void *)malloc(size)
+#define Dtd_PreParse_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define Dtd_PreParse_free(ptr, yyscanner)		free((char *)ptr)
+
 %}
 xmlpi_start "<?"
 xmlpi_stop  "?>"
@@ -132,63 +173,63 @@ newline    \n
 %%
 
 
-{entity}						if (current) g_string_append_printf(current,"%s\n%s\n",replace_entity(yytext),location());
+{entity}				if (yyextra->current) g_string_append_printf(yyextra->current,"%s\n%s\n",replace_entity(yyextra, yytext),location(yyextra));
 
-{whitespace}					if (current) g_string_append(current," ");
+{whitespace}				if (yyextra->current) g_string_append(yyextra->current," ");
 
-<OUTSIDE>{xmlpi_start}			{ g_string_append(current,yytext); BEGIN XMLPI; }
-<XMLPI>{xmlpi_chars}			{ g_string_append(current,yytext); }
-<XMLPI>{newline}				{ g_string_append(current,yytext); }
-<XMLPI>{xmlpi_stop}				{ g_string_append(current,yytext); BEGIN OUTSIDE; }
+<OUTSIDE>{xmlpi_start}			{ g_string_append(yyextra->current,yytext); BEGIN XMLPI; }
+<XMLPI>{xmlpi_chars}			{ g_string_append(yyextra->current,yytext); }
+<XMLPI>{newline}			{ g_string_append(yyextra->current,yytext); }
+<XMLPI>{xmlpi_stop}			{ g_string_append(yyextra->current,yytext); BEGIN OUTSIDE; }
 
-<OUTSIDE>{comment_start}		{ current = NULL; BEGIN IN_COMMENT; }
-<IN_COMMENT>[^-]?				|
-<IN_COMMENT>[-]					;
-<IN_COMMENT>{comment_stop}		{ current = output; BEGIN OUTSIDE; }
+<OUTSIDE>{comment_start}		{ yyextra->current = NULL; BEGIN IN_COMMENT; }
+<IN_COMMENT>[^-]?			|
+<IN_COMMENT>[-]				;
+<IN_COMMENT>{comment_stop}		{ yyextra->current = yyextra->output; BEGIN OUTSIDE; }
 
-{newline}						{
-	linenum++;
-	if (current) g_string_append_printf(current,"%s\n",location());
+{newline}				{
+	yyextra->linenum++;
+	if (yyextra->current) g_string_append_printf(yyextra->current,"%s\n",location(yyextra));
 }
 
 
 <OUTSIDE>{entity_start}			{ BEGIN IN_ENTITY; }
-<IN_ENTITY>{name}				{ entity_name = g_strdup_printf("%%%s;",yytext); BEGIN NAMED_ENTITY; }
-<NAMED_ENTITY>{quote}			{ current = g_string_new(location()); BEGIN IN_QUOTE; }
-<IN_QUOTE>{quote}				{ g_hash_table_insert(entities,entity_name,current);  BEGIN ENTITY_DONE; }
-<IN_QUOTE>{percent}				|
+<IN_ENTITY>{name}			{ yyextra->entity_name = g_strdup_printf("%%%s;",yytext); BEGIN NAMED_ENTITY; }
+<NAMED_ENTITY>{quote}			{ yyextra->current = g_string_new(location(yyextra)); BEGIN IN_QUOTE; }
+<IN_QUOTE>{quote}			{ g_hash_table_insert(yyextra->entities,yyextra->entity_name,yyextra->current);  BEGIN ENTITY_DONE; }
+<IN_QUOTE>{percent}			|
 <IN_QUOTE>{non_quote}			|
-<IN_QUOTE>{escaped_quote}		g_string_append(current,yytext);
+<IN_QUOTE>{escaped_quote}		g_string_append(yyextra->current,yytext);
 <NAMED_ENTITY>{system}			{
-	g_string_append_printf(error,"at %s:%u: file inclusion is not supported!", filename, linenum);
+	g_string_append_printf(yyextra->error,"at %s:%u: file inclusion is not supported!", yyextra->filename, yyextra->linenum);
 	yyterminate();
 }
-<ENTITY_DONE>{special_stop}		{ current = output; g_string_append(current,"\n"); BEGIN OUTSIDE; }
+<ENTITY_DONE>{special_stop}		{ yyextra->current = yyextra->output; g_string_append(yyextra->current,"\n"); BEGIN OUTSIDE; }
 
 %%
 
-static const gchar* replace_entity(gchar* entity) {
+static const gchar* replace_entity(Dtd_PreParse_scanner_state_t* state, gchar* entity) {
 	GString* replacement;
 
 	*entity = '%';
 
-	replacement = (GString*)g_hash_table_lookup(entities,entity);
+	replacement = (GString*)g_hash_table_lookup(state->entities,entity);
 
 	if (replacement) {
 		return replacement->str;
 	} else {
-		g_string_append_printf(error,"dtd_preparse: in file '%s': entity %s does not exists\n", filename, entity);
+		g_string_append_printf(state->error,"dtd_preparse: in file '%s': entity %s does not exists\n", state->filename, entity);
 		return "";
 	}
 
 }
 
-static const gchar* location(void) {
+static const gchar* location(Dtd_PreParse_scanner_state_t* state) {
 	static gchar* loc = NULL;
 
 	if (loc) g_free(loc);
 
-	loc = g_strdup_printf("<? wireshark:location %s:%u ?>", filename, linenum);
+	loc = g_strdup_printf("<? wireshark:location %s:%u ?>", state->filename, state->linenum);
 
 	return loc;
 }
@@ -201,37 +242,51 @@ static gboolean free_gstring_hash_items(gpointer k,gpointer v,gpointer p _U_) {
 
 extern GString* dtd_preparse(const gchar* dname,const  gchar* fname, GString* err) {
 	gchar* fullname = g_strdup_printf("%s%c%s",dname,G_DIR_SEPARATOR,fname);
+	FILE *in;
+	yyscan_t scanner;
+	Dtd_PreParse_scanner_state_t state;
 
-	dtd_dirname = dname;
-	filename = fname;
-	linenum = 1;
+	in = ws_fopen(fullname,"r");
 
-	yyin = ws_fopen(fullname,"r");
-
-	if (!yyin) {
+	if (!in) {
 		if (err)
 			g_string_append_printf(err, "Could not open file: '%s', error: %s",fullname,g_strerror(errno));
 
 		return NULL;
 	}
 
-	error = err;
+	if (Dtd_PreParse_lex_init(&scanner) != 0) {
+		if (err)
+			g_string_append_printf(err, "Can't initialize scanner: %s",
+			    strerror(errno));
+		fclose(in);
+		return FALSE;
+	}
+
+	Dtd_PreParse_set_in(in, scanner);
+
+	state.dtd_dirname = dname;
+	state.filename = fname;
+	state.linenum = 1;
 
-	entities = g_hash_table_new(g_str_hash,g_str_equal);
-	current = output = g_string_new(location());
+	state.error = err;
 
-	BEGIN OUTSIDE;
+	state.entities = g_hash_table_new(g_str_hash,g_str_equal);
+	state.current = state.output = g_string_new(location(&state));
+	state.entity_name = NULL;
 
-	yylex();
+	/* Associate the state with the scanner */
+	Dtd_PreParse_set_extra(&state, scanner);
 
-	fclose(yyin);
+	Dtd_PreParse_lex(scanner);
 
-	yyrestart(NULL);
+	Dtd_PreParse_lex_destroy(scanner);
+	fclose(in);
 
-	g_hash_table_foreach_remove(entities,free_gstring_hash_items,NULL);
-	g_hash_table_destroy(entities);
+	g_hash_table_foreach_remove(state.entities,free_gstring_hash_items,NULL);
+	g_hash_table_destroy(state.entities);
 
 	g_free(fullname);
 
-	return output;
+	return state.output;
 }