Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

2 files changed, 167 insertions, 111 deletions

diff --git a/plugins/mate/Makefile.am b/plugins/mate/Makefile.am
index fe1cc4d6ad..26647b445c 100644
--- a/plugins/mate/Makefile.am
+++ b/plugins/mate/Makefile.am
@@ -50,7 +50,8 @@ mate_la_CFLAGS = $(GENERATED_CFLAGS) $(PLUGIN_CFLAGS)
 mate_la_LDFLAGS = $(PLUGIN_LDFLAGS)
 
 BUILT_SOURCES = \
-	$(LEMON_GENERATED_HEADER_FILES)
+	$(LEMON_GENERATED_HEADER_FILES) \
+	$(FLEX_GENERATED_HEADER_FILES)
 
 CLEANFILES = \
 	mate \
diff --git a/plugins/mate/mate_parser.l b/plugins/mate/mate_parser.l
index 865b8d138c..706900b839 100644
--- a/plugins/mate/mate_parser.l
+++ b/plugins/mate/mate_parser.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -19,10 +24,28 @@
 %option noyywrap
 
 /*
- * Prefix scanner routines with "Mate" rather than "yy", so this scanner
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="Mate_scanner_state_t *"
+
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
+/*
+ * Prefix scanner routines with "Mate_" rather than "yy", so this scanner
  * can coexist with other scanners.
  */
-%option prefix="Mate"
+%option prefix="Mate_"
 
 %{
 
@@ -52,7 +75,6 @@
 
 #include "mate.h"
 #include "mate_grammar.h"
-#include "mate_parser_lex.h"
 
 #include <wsutil/file_util.h>
 
@@ -72,15 +94,18 @@
 	void MateParseTrace(FILE*,char*);
 
 #define MAX_INCLUDE_DEPTH 10
-	static YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
-	static int include_stack_ptr = 0;
+typedef struct {
+	mate_config* mc;
 
-	static void* pParser;
-	static mate_config_frame* current_frame;
+	mate_config_frame* current_frame;
 
-	static mate_config* mc;
+	void* pParser;
 
-#define MATE_PARSE(token_type) MateParser(pParser, (token_type), g_strdup(yytext), mc );
+	YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH];
+	int include_stack_ptr;
+} Mate_scanner_state_t;
+
+#define MATE_PARSE(token_type) MateParser(yyextra->pParser, (token_type), g_strdup(yytext), yyextra->mc);
 
 /*
  * Flex (v 2.5.35) uses this symbol to "exclude" unistd.h
@@ -94,6 +119,20 @@ static void free_config_frame(mate_config_frame *frame) {
 	g_free(frame);
 }
 
+#define YY_USER_INIT BEGIN OUTSIDE;
+
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define Mate_alloc(size, yyscanner)		(void *)malloc(size)
+#define Mate_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define Mate_free(ptr, yyscanner)		free((char *)ptr)
+
 %}
 
 pdu_kw				Pdu
@@ -176,36 +215,36 @@ blk_cmnt_stop  "*/"
 %START OUTSIDE QUOTED INCLUDING COMMENT
 %%
 
-{newline}						current_frame->linenum++;
-{whitespace}					;
+{newline}				yyextra->current_frame->linenum++;
+{whitespace}				;
 
-<OUTSIDE>{include}					BEGIN INCLUDING;
+<OUTSIDE>{include}			BEGIN INCLUDING;
 
 <INCLUDING>{filename}			{
-	if ( include_stack_ptr >= MAX_INCLUDE_DEPTH )
+	if ( yyextra->include_stack_ptr >= MAX_INCLUDE_DEPTH )
 		g_error("dtd_preparse: include files nested too deeply");
 
-	include_stack[include_stack_ptr++] = YY_CURRENT_BUFFER;
+	yyextra->include_stack[yyextra->include_stack_ptr++] = YY_CURRENT_BUFFER;
 	yyin = ws_fopen( yytext, "r" );
 
 	if (!yyin) {
-		yy_delete_buffer( YY_CURRENT_BUFFER );
+		Mate__delete_buffer(YY_CURRENT_BUFFER, yyscanner);
 
 		/* coverity[negative_sink] */
-		yy_switch_to_buffer(include_stack[--include_stack_ptr] );
+		Mate__switch_to_buffer(yyextra->include_stack[--yyextra->include_stack_ptr], yyscanner);
 
 		if (errno)
-			g_string_append_printf(mc->config_error, "Mate parser: Could not open file: '%s': %s", yytext, g_strerror(errno) );
+			g_string_append_printf(yyextra->mc->config_error, "Mate parser: Could not open file: '%s': %s", yytext, g_strerror(errno) );
 
 	} else {
 
-		current_frame = (mate_config_frame *)g_malloc(sizeof(mate_config_frame));
-		current_frame->filename = g_strdup(yytext);
-		current_frame->linenum = 1;
+		yyextra->current_frame = (mate_config_frame *)g_malloc(sizeof(mate_config_frame));
+		yyextra->current_frame->filename = g_strdup(yytext);
+		yyextra->current_frame->linenum = 1;
 
-		g_ptr_array_add(mc->config_stack,current_frame);
+		g_ptr_array_add(yyextra->mc->config_stack,yyextra->current_frame);
 
-		yy_switch_to_buffer(yy_create_buffer( yyin, YY_BUF_SIZE ) );
+		Mate__switch_to_buffer(Mate__create_buffer(yyin, YY_BUF_SIZE, yyscanner), yyscanner);
 	}
 
 	BEGIN OUTSIDE;
@@ -213,134 +252,150 @@ blk_cmnt_stop  "*/"
 
 <<EOF>>	{
 	/* coverity[check_after_sink] */
-	if ( --include_stack_ptr < 0 ) {
+	if ( --yyextra->include_stack_ptr < 0 ) {
 		yyterminate();
 	} else {
-		yy_delete_buffer( YY_CURRENT_BUFFER );
-		yy_switch_to_buffer( include_stack[include_stack_ptr] );
+		Mate__delete_buffer(YY_CURRENT_BUFFER, yyscanner);
+		Mate__switch_to_buffer(yyextra->include_stack[yyextra->include_stack_ptr], yyscanner);
 
-		free_config_frame(current_frame);
-		current_frame = (mate_config_frame *)g_ptr_array_remove_index(mc->config_stack,mc->config_stack->len-1);
+		free_config_frame(yyextra->current_frame);
+		yyextra->current_frame = (mate_config_frame *)g_ptr_array_remove_index(yyextra->mc->config_stack,yyextra->mc->config_stack->len-1);
 	}
 }
 
-<OUTSIDE>{comment}					;
+<OUTSIDE>{comment}			;
 
-<OUTSIDE>{blk_cmnt_start}			BEGIN COMMENT;
+<OUTSIDE>{blk_cmnt_start}		BEGIN COMMENT;
 <COMMENT>{cmnt_char}			;
 <COMMENT>{blk_cmnt_stop}		BEGIN OUTSIDE;
 
-<OUTSIDE>{pdu_kw}					MATE_PARSE(TOKEN_PDU_KW);
-<OUTSIDE>{gop_kw}					MATE_PARSE(TOKEN_GOP_KW);
-<OUTSIDE>{gog_kw}					MATE_PARSE(TOKEN_GOG_KW);
-<OUTSIDE>{transform_kw}				MATE_PARSE(TOKEN_TRANSFORM_KW);
-<OUTSIDE>{match_kw}					MATE_PARSE(TOKEN_MATCH_KW);
-<OUTSIDE>{strict_kw}				MATE_PARSE(TOKEN_STRICT_KW);
-<OUTSIDE>{every_kw}					MATE_PARSE(TOKEN_EVERY_KW);
-<OUTSIDE>{loose_kw}					MATE_PARSE(TOKEN_LOOSE_KW);
-<OUTSIDE>{replace_kw}				MATE_PARSE(TOKEN_REPLACE_KW);
-<OUTSIDE>{insert_kw}				MATE_PARSE(TOKEN_INSERT_KW);
-<OUTSIDE>{gop_tree_kw}				MATE_PARSE(TOKEN_GOP_TREE_KW);
-<OUTSIDE>{member_kw}				MATE_PARSE(TOKEN_MEMBER_KW);
-<OUTSIDE>{on_kw}					MATE_PARSE(TOKEN_ON_KW);
-<OUTSIDE>{start_kw}					MATE_PARSE(TOKEN_START_KW);
-<OUTSIDE>{stop_kw}					MATE_PARSE(TOKEN_STOP_KW);
-<OUTSIDE>{extra_kw}					MATE_PARSE(TOKEN_EXTRA_KW);
-<OUTSIDE>{show_tree_kw}				MATE_PARSE(TOKEN_SHOW_TREE_KW);
-<OUTSIDE>{show_times_kw}			MATE_PARSE(TOKEN_SHOW_TIMES_KW);
-<OUTSIDE>{expiration_kw}			MATE_PARSE(TOKEN_EXPIRATION_KW);
-<OUTSIDE>{idle_timeout_kw}			MATE_PARSE(TOKEN_IDLE_TIMEOUT_KW);
-<OUTSIDE>{lifetime_kw}				MATE_PARSE(TOKEN_LIFETIME_KW);
-<OUTSIDE>{no_tree_kw}				MATE_PARSE(TOKEN_NO_TREE_KW);
-<OUTSIDE>{pdu_tree_kw}				MATE_PARSE(TOKEN_PDU_TREE_KW);
-<OUTSIDE>{frame_tree_kw}			MATE_PARSE(TOKEN_FRAME_TREE_KW);
-<OUTSIDE>{basic_tree_kw}			MATE_PARSE(TOKEN_BASIC_TREE_KW);
-<OUTSIDE>{true_kw}					MATE_PARSE(TOKEN_TRUE_KW);
-<OUTSIDE>{false_kw}					MATE_PARSE(TOKEN_FALSE_KW);
-<OUTSIDE>{proto_kw}					MATE_PARSE(TOKEN_PROTO_KW);
-<OUTSIDE>{payload_kw}				MATE_PARSE(TOKEN_PAYLOAD_KW);
-<OUTSIDE>{transport_kw}				MATE_PARSE(TOKEN_TRANSPORT_KW);
-<OUTSIDE>{criteria_kw}				MATE_PARSE(TOKEN_CRITERIA_KW);
-<OUTSIDE>{accept_kw}				MATE_PARSE(TOKEN_ACCEPT_KW);
-<OUTSIDE>{reject_kw}				MATE_PARSE(TOKEN_REJECT_KW);
-<OUTSIDE>{extract_kw}				MATE_PARSE(TOKEN_EXTRACT_KW);
-<OUTSIDE>{from_kw}					MATE_PARSE(TOKEN_FROM_KW);
+<OUTSIDE>{pdu_kw}			MATE_PARSE(TOKEN_PDU_KW);
+<OUTSIDE>{gop_kw}			MATE_PARSE(TOKEN_GOP_KW);
+<OUTSIDE>{gog_kw}			MATE_PARSE(TOKEN_GOG_KW);
+<OUTSIDE>{transform_kw}			MATE_PARSE(TOKEN_TRANSFORM_KW);
+<OUTSIDE>{match_kw}			MATE_PARSE(TOKEN_MATCH_KW);
+<OUTSIDE>{strict_kw}			MATE_PARSE(TOKEN_STRICT_KW);
+<OUTSIDE>{every_kw}			MATE_PARSE(TOKEN_EVERY_KW);
+<OUTSIDE>{loose_kw}			MATE_PARSE(TOKEN_LOOSE_KW);
+<OUTSIDE>{replace_kw}			MATE_PARSE(TOKEN_REPLACE_KW);
+<OUTSIDE>{insert_kw}			MATE_PARSE(TOKEN_INSERT_KW);
+<OUTSIDE>{gop_tree_kw}			MATE_PARSE(TOKEN_GOP_TREE_KW);
+<OUTSIDE>{member_kw}			MATE_PARSE(TOKEN_MEMBER_KW);
+<OUTSIDE>{on_kw}			MATE_PARSE(TOKEN_ON_KW);
+<OUTSIDE>{start_kw}			MATE_PARSE(TOKEN_START_KW);
+<OUTSIDE>{stop_kw}			MATE_PARSE(TOKEN_STOP_KW);
+<OUTSIDE>{extra_kw}			MATE_PARSE(TOKEN_EXTRA_KW);
+<OUTSIDE>{show_tree_kw}			MATE_PARSE(TOKEN_SHOW_TREE_KW);
+<OUTSIDE>{show_times_kw}		MATE_PARSE(TOKEN_SHOW_TIMES_KW);
+<OUTSIDE>{expiration_kw}		MATE_PARSE(TOKEN_EXPIRATION_KW);
+<OUTSIDE>{idle_timeout_kw}		MATE_PARSE(TOKEN_IDLE_TIMEOUT_KW);
+<OUTSIDE>{lifetime_kw}			MATE_PARSE(TOKEN_LIFETIME_KW);
+<OUTSIDE>{no_tree_kw}			MATE_PARSE(TOKEN_NO_TREE_KW);
+<OUTSIDE>{pdu_tree_kw}			MATE_PARSE(TOKEN_PDU_TREE_KW);
+<OUTSIDE>{frame_tree_kw}		MATE_PARSE(TOKEN_FRAME_TREE_KW);
+<OUTSIDE>{basic_tree_kw}		MATE_PARSE(TOKEN_BASIC_TREE_KW);
+<OUTSIDE>{true_kw}			MATE_PARSE(TOKEN_TRUE_KW);
+<OUTSIDE>{false_kw}			MATE_PARSE(TOKEN_FALSE_KW);
+<OUTSIDE>{proto_kw}			MATE_PARSE(TOKEN_PROTO_KW);
+<OUTSIDE>{payload_kw}			MATE_PARSE(TOKEN_PAYLOAD_KW);
+<OUTSIDE>{transport_kw}			MATE_PARSE(TOKEN_TRANSPORT_KW);
+<OUTSIDE>{criteria_kw}			MATE_PARSE(TOKEN_CRITERIA_KW);
+<OUTSIDE>{accept_kw}			MATE_PARSE(TOKEN_ACCEPT_KW);
+<OUTSIDE>{reject_kw}			MATE_PARSE(TOKEN_REJECT_KW);
+<OUTSIDE>{extract_kw}			MATE_PARSE(TOKEN_EXTRACT_KW);
+<OUTSIDE>{from_kw}			MATE_PARSE(TOKEN_FROM_KW);
 <OUTSIDE>{drop_unassigned_kw}		MATE_PARSE(TOKEN_DROP_UNASSIGNED_KW);
 <OUTSIDE>{discard_pdu_data_kw}		MATE_PARSE(TOKEN_DISCARD_PDU_DATA_KW);
-<OUTSIDE>{last_pdu_kw}				MATE_PARSE(TOKEN_LAST_PDU_KW);
-<OUTSIDE>{done_kw}					MATE_PARSE(TOKEN_DONE_KW);
-<OUTSIDE>{filename_kw}				MATE_PARSE(TOKEN_FILENAME_KW);
-<OUTSIDE>{debug_kw}					MATE_PARSE(TOKEN_DEBUG_KW);
-<OUTSIDE>{level_kw}					MATE_PARSE(TOKEN_LEVEL_KW);
-<OUTSIDE>{default_kw}				MATE_PARSE(TOKEN_DEFAULT_KW);
-
-<OUTSIDE>{open_parens}				MATE_PARSE(TOKEN_OPEN_PARENS);
-<OUTSIDE>{close_parens}				MATE_PARSE(TOKEN_CLOSE_PARENS);
-<OUTSIDE>{open_brace}				MATE_PARSE(TOKEN_OPEN_BRACE);
-<OUTSIDE>{close_brace}				MATE_PARSE(TOKEN_CLOSE_BRACE);
-<OUTSIDE>{comma}					MATE_PARSE(TOKEN_COMMA);
-<OUTSIDE>{semicolon}				MATE_PARSE(TOKEN_SEMICOLON);
-<OUTSIDE>{slash}					MATE_PARSE(TOKEN_SLASH);
-<OUTSIDE>{pipe}						MATE_PARSE(TOKEN_PIPE);
-
-<OUTSIDE>{integer}					MATE_PARSE(TOKEN_INTEGER);
-<OUTSIDE>{floating}					MATE_PARSE(TOKEN_FLOATING);
-<OUTSIDE>{doted_ip}					MATE_PARSE(TOKEN_DOTED_IP);
-<OUTSIDE>{colonized}				MATE_PARSE(TOKEN_COLONIZED);
-<OUTSIDE>{name}						MATE_PARSE(TOKEN_NAME);
-<OUTSIDE>{avp_operator}				MATE_PARSE(TOKEN_AVP_OPERATOR);
-
-
-<OUTSIDE>{quote}					BEGIN QUOTED;
+<OUTSIDE>{last_pdu_kw}			MATE_PARSE(TOKEN_LAST_PDU_KW);
+<OUTSIDE>{done_kw}			MATE_PARSE(TOKEN_DONE_KW);
+<OUTSIDE>{filename_kw}			MATE_PARSE(TOKEN_FILENAME_KW);
+<OUTSIDE>{debug_kw}			MATE_PARSE(TOKEN_DEBUG_KW);
+<OUTSIDE>{level_kw}			MATE_PARSE(TOKEN_LEVEL_KW);
+<OUTSIDE>{default_kw}			MATE_PARSE(TOKEN_DEFAULT_KW);
+
+<OUTSIDE>{open_parens}			MATE_PARSE(TOKEN_OPEN_PARENS);
+<OUTSIDE>{close_parens}			MATE_PARSE(TOKEN_CLOSE_PARENS);
+<OUTSIDE>{open_brace}			MATE_PARSE(TOKEN_OPEN_BRACE);
+<OUTSIDE>{close_brace}			MATE_PARSE(TOKEN_CLOSE_BRACE);
+<OUTSIDE>{comma}			MATE_PARSE(TOKEN_COMMA);
+<OUTSIDE>{semicolon}			MATE_PARSE(TOKEN_SEMICOLON);
+<OUTSIDE>{slash}			MATE_PARSE(TOKEN_SLASH);
+<OUTSIDE>{pipe}				MATE_PARSE(TOKEN_PIPE);
+
+<OUTSIDE>{integer}			MATE_PARSE(TOKEN_INTEGER);
+<OUTSIDE>{floating}			MATE_PARSE(TOKEN_FLOATING);
+<OUTSIDE>{doted_ip}			MATE_PARSE(TOKEN_DOTED_IP);
+<OUTSIDE>{colonized}			MATE_PARSE(TOKEN_COLONIZED);
+<OUTSIDE>{name}				MATE_PARSE(TOKEN_NAME);
+<OUTSIDE>{avp_operator}			MATE_PARSE(TOKEN_AVP_OPERATOR);
+
+
+<OUTSIDE>{quote}			BEGIN QUOTED;
 <QUOTED>{not_quoted}			MATE_PARSE(TOKEN_QUOTED);
-<QUOTED>{quote}					BEGIN OUTSIDE;
+<QUOTED>{quote}				BEGIN OUTSIDE;
 
 %%
 
-extern gboolean mate_load_config(const gchar* filename, mate_config* matecfg) {
-	volatile gboolean state = TRUE;
-	mc = matecfg;
+extern gboolean mate_load_config(const gchar* filename, mate_config* mc) {
+	FILE *in;
+	yyscan_t scanner;
+	Mate_scanner_state_t state;
+	volatile gboolean status = TRUE;
 
-	yyin = ws_fopen(filename,"r");
+	in = ws_fopen(filename,"r");
 
-	if (!yyin) {
+	if (!in) {
 		g_string_append_printf(mc->config_error,"Mate parser: Could not open file: '%s', error: %s", filename, g_strerror(errno) );
 		return FALSE;
 	}
 
+	if (Mate_lex_init(&scanner) != 0) {
+		g_string_append_printf(mc->config_error, "Mate parse: Could not initialize scanner: %s", g_strerror(errno));
+		fclose(in);
+		return FALSE;
+	}
+
+	Mate_set_in(in, scanner);
+
 	mc->config_stack = g_ptr_array_new();
 
-	current_frame = (mate_config_frame *)g_malloc(sizeof(mate_config_frame));
-	current_frame->filename = g_strdup(filename);
-	current_frame->linenum = 1;
+	state.mc = mc;
+
+	state.current_frame = (mate_config_frame *)g_malloc(sizeof(mate_config_frame));
+	state.current_frame->filename = g_strdup(filename);
+	state.current_frame->linenum = 1;
+
+	g_ptr_array_add(mc->config_stack,state.current_frame);
+
+	state.pParser = MateParserAlloc(g_malloc);
 
-	g_ptr_array_add(mc->config_stack,current_frame);
+	state.include_stack_ptr = 0;
 
-	pParser = MateParserAlloc(g_malloc);
+	/* Associate the state with the scanner */
+	Mate_set_extra(&state, scanner);
 
 	/* MateParserTrace(stdout,""); */
 
 	TRY {
-		BEGIN OUTSIDE;
-
-		yylex();
+		Mate_lex(scanner);
 
 		/* Inform parser that end of input has reached. */
-		MateParser(pParser, 0, NULL,mc);
-
-		yyrestart(NULL);
+		MateParser(state.pParser, 0, NULL, mc);
 
-		MateParserFree(pParser, g_free );
+		MateParserFree(state.pParser, g_free);
 	} CATCH(MateConfigError) {
-		state = FALSE;
+		status = FALSE;
 	} CATCH_ALL {
-		state = FALSE;
+		status = FALSE;
 		g_string_append_printf(mc->config_error,"An unexpected error occurred");
 	}
 	ENDTRY;
 
+	Mate_lex_destroy(scanner);
+	fclose(in);
+
 	g_ptr_array_foreach(mc->config_stack, (GFunc)free_config_frame, NULL);
 	g_ptr_array_free(mc->config_stack, FALSE);
 
-	return state;
+	return status;
 }