Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

1 files changed, 176 insertions, 92 deletions

diff --git a/wiretap/k12text.l b/wiretap/k12text.l
index 3ff7ca8610..4cdea9c951 100644
--- a/wiretap/k12text.l
+++ b/wiretap/k12text.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -19,10 +24,15 @@
 %option noyywrap
 
 /*
- * Prefix scanner routines with "K12Text_" rather than "yy", so this scanner
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="k12text_state_t *"
+
+/*
+ * Prefix scanner routines with "k12text_" rather than "yy", so this scanner
  * can coexist with other scanners.
  */
-%option prefix="K12Text_"
+%option prefix="k12text_"
 
 %option outfile="k12text.c"
 
@@ -45,6 +55,19 @@
 /* %option noline  */
 /* %option debug   */
 
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
 %{
 /* k12text.l
  *
@@ -82,41 +105,64 @@
 #include "file_wrappers.h"
 #include <wsutil/buffer.h>
 #include "k12.h"
-#include "k12text_lex.h"
 
 #ifndef HAVE_UNISTD_H
 #define YY_NO_UNISTD_H
 #endif
 
-static guint g_h;
-static guint g_m;
-static guint g_s;
-static guint g_ms;
-static guint g_ns;
-static gint g_encap;
-static guint8 bb[WTAP_MAX_PACKET_SIZE];
-static guint ii;
-static gboolean is_k12text;
-static gboolean at_eof;
-static guint junk_chars;
-static void finalize_frame(void);
-static gchar* error_str;
-static guint64 file_bytes_read;
-static gboolean ok_frame;
-static FILE_T yy_fh;
-
-#define KERROR(text) do { error_str = g_strdup(text); yyterminate(); } while(0)
-#define SET_HOURS(text) g_h = (guint) strtoul(text,NULL,10)
-#define SET_MINUTES(text) g_m = (guint) strtoul(text,NULL,10)
-#define SET_SECONDS(text) g_s = (guint) strtoul(text,NULL,10)
-#define SET_MS(text) g_ms = (guint) strtoul(text,NULL,10)
-#define SET_NS(text) g_ns = (guint) strtoul(text,NULL,10)
-#define ADD_BYTE(text) do {if (ii >= WTAP_MAX_PACKET_SIZE) {KERROR("frame too large");} bb[ii++] = (guint8)strtoul(text,NULL,16); } while(0)
-#define FINALIZE_FRAME() finalize_frame()
+/*
+ * State kept by the scanner.
+ */
+typedef struct {
+	FILE_T fh;
+	int *err;
+	gchar **err_info;
+	int start_state;
+
+	guint g_h;
+	guint g_m;
+	guint g_s;
+	guint g_ms;
+	guint g_ns;
+	gint g_encap;
+	guint8 bb[WTAP_MAX_PACKET_SIZE];
+	guint ii;
+	gboolean is_k12text;
+	gboolean at_eof;
+	guint junk_chars;
+	gchar* error_str;
+	guint64 file_bytes_read;
+	gboolean ok_frame;
+} k12text_state_t;
+
+#define KERROR(text) do { yyextra->error_str = g_strdup(text); yyterminate(); } while(0)
+#define SET_HOURS(text) yyextra->g_h = (guint) strtoul(text,NULL,10)
+#define SET_MINUTES(text) yyextra->g_m = (guint) strtoul(text,NULL,10)
+#define SET_SECONDS(text) yyextra->g_s = (guint) strtoul(text,NULL,10)
+#define SET_MS(text) yyextra->g_ms = (guint) strtoul(text,NULL,10)
+#define SET_NS(text) yyextra->g_ns = (guint) strtoul(text,NULL,10)
+#define ADD_BYTE(text) do {if (yyextra->ii >= WTAP_MAX_PACKET_SIZE) {KERROR("frame too large");} yyextra->bb[yyextra->ii++] = (guint8)strtoul(text,NULL,16); } while(0)
+#define FINALIZE_FRAME() do { yyextra->ok_frame = TRUE; } while (0)
 /*~ #define ECHO*/
-#define YY_USER_ACTION file_bytes_read += yyleng;
-#define YY_INPUT(buf,result,max_size) { int c = file_getc(yy_fh);  result = (c==EOF) ? YY_NULL : (buf[0] = c, 1); }
-
+#define YY_USER_ACTION yyextra->file_bytes_read += yyleng;
+#define YY_USER_INIT { \
+	k12text_state_t *scanner_state = k12text_get_extra(yyscanner); \
+	BEGIN(scanner_state->start_state); \
+}
+#define YY_INPUT(buf,result,max_size) { \
+	k12text_state_t *scanner_state = k12text_get_extra(yyscanner); \
+	int c = file_getc(scanner_state->fh); \
+	if (c == EOF) { \
+		*(scanner_state->err) = file_error(scanner_state->fh, \
+		    scanner_state->err_info); \
+		if (*(scanner_state->err) == 0) \
+			*(scanner_state->err) = WTAP_ERR_SHORT_READ; \
+		result = YY_NULL; \
+	} else { \
+		buf[0] = c; \
+		result = 1; \
+	} \
+}
 #define MAX_JUNK 400000
 #define ECHO
 
@@ -134,6 +180,19 @@ typedef struct {
 	 */
 	gint64	next_frame_offset;
 } k12text_t;
+
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define k12text_alloc(size, yyscanner)		(void *)malloc(size)
+#define k12text_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define k12text_free(ptr, yyscanner)		free((char *)ptr)
+
 %}
 start_timestamp \053[\055]{9}\053[\055]{15,100}\053[\055]{10,100}\053
 oneormoredigits [0-9]+:
@@ -153,9 +212,9 @@ hdlc HDLC
 
 %START MAGIC NEXT_FRAME HOURS MINUTES M2S SECONDS S2M MS M2N NS ENCAP STARTBYTES BYTE
 %%
-<MAGIC>{start_timestamp}  { is_k12text = TRUE; yyterminate(); }
+<MAGIC>{start_timestamp}  { yyextra->is_k12text = TRUE; yyterminate(); }
 
-<MAGIC>. { if (++ junk_chars > MAX_JUNK) { is_k12text = FALSE;  yyterminate(); } }
+<MAGIC>. { if (++ yyextra->junk_chars > MAX_JUNK) { yyextra->is_k12text = FALSE;  yyterminate(); } }
 
 <NEXT_FRAME>{start_timestamp} {BEGIN(HOURS); }
 <HOURS>{oneormoredigits} { SET_HOURS(yytext); BEGIN(MINUTES); }
@@ -166,42 +225,39 @@ hdlc HDLC
 <MS>{threedigits} { SET_MS(yytext); BEGIN(M2N);  }
 <M2N>{comma}  { BEGIN(NS); }
 <NS>{threedigits} { SET_NS(yytext); BEGIN(ENCAP);}
-<ENCAP>{eth} {g_encap = WTAP_ENCAP_ETHERNET; BEGIN(STARTBYTES); }
-<ENCAP>{mtp2} {g_encap = WTAP_ENCAP_MTP2; BEGIN(STARTBYTES); }
-<ENCAP>{sscop} {g_encap = WTAP_ENCAP_ATM_PDUS; BEGIN(STARTBYTES); }
-<ENCAP>{sscfnni} {g_encap = WTAP_ENCAP_MTP3; BEGIN(STARTBYTES); }
-<ENCAP>{hdlc} {g_encap = WTAP_ENCAP_CHDLC; BEGIN(STARTBYTES); }
+<ENCAP>{eth} {yyextra->g_encap = WTAP_ENCAP_ETHERNET; BEGIN(STARTBYTES); }
+<ENCAP>{mtp2} {yyextra->g_encap = WTAP_ENCAP_MTP2; BEGIN(STARTBYTES); }
+<ENCAP>{sscop} {yyextra->g_encap = WTAP_ENCAP_ATM_PDUS; BEGIN(STARTBYTES); }
+<ENCAP>{sscfnni} {yyextra->g_encap = WTAP_ENCAP_MTP3; BEGIN(STARTBYTES); }
+<ENCAP>{hdlc} {yyextra->g_encap = WTAP_ENCAP_CHDLC; BEGIN(STARTBYTES); }
 <ENCAP,STARTBYTES>{start_bytes} { BEGIN(BYTE); }
 <BYTE>{byte} { ADD_BYTE(yytext); }
 <BYTE>{bytes_junk} ;
 <BYTE>{end_bytes} { FINALIZE_FRAME(); yyterminate(); }
 
-. {  if (++junk_chars > MAX_JUNK) { KERROR("too much junk");  } }
-<<EOF>> { at_eof = TRUE; yyterminate(); }
+. {  if (++yyextra->junk_chars > MAX_JUNK) { KERROR("too much junk");  } }
+<<EOF>> { yyextra->at_eof = TRUE; yyterminate(); }
 
 %%
 
-static void finalize_frame(void) {
-	ok_frame = TRUE;
-}
-
 /* Fill in pkthdr */
 
 static gboolean
-k12text_set_headers(struct wtap_pkthdr *phdr, int *err, gchar **err_info)
+k12text_set_headers(struct wtap_pkthdr *phdr, k12text_state_t *state,
+    int *err, gchar **err_info)
 {
 	phdr->rec_type = REC_TYPE_PACKET;
 	phdr->presence_flags = WTAP_HAS_TS|WTAP_HAS_CAP_LEN;
 
-	phdr->ts.secs = 946681200 + (3600*g_h) + (60*g_m) + g_s;
-	phdr->ts.nsecs = 1000000*g_ms + 1000*g_ns;
+	phdr->ts.secs = 946681200 + (3600*state->g_h) + (60*state->g_m) + state->g_s;
+	phdr->ts.nsecs = 1000000*state->g_ms + 1000*state->g_ns;
 
-	phdr->caplen = phdr->len = ii;
+	phdr->caplen = phdr->len = state->ii;
 
-	phdr->pkt_encap = g_encap;
+	phdr->pkt_encap = state->g_encap;
 
 	/* The file-encap is WTAP_ENCAP_PER_PACKET */
-	switch(g_encap) {
+	switch(state->g_encap) {
 	    case WTAP_ENCAP_ETHERNET:
 		    phdr->pseudo_header.eth.fcs_len = 0;
 		    break;
@@ -236,30 +292,54 @@ k12text_set_headers(struct wtap_pkthdr *phdr, int *err, gchar **err_info)
 /*       used even when multiple files are open simultaneously (as for	*/
 /*       a file merge).							*/
 
-static void
-k12text_reset(FILE_T fh)
+static gboolean
+k12text_run_scanner(k12text_state_t *state, FILE_T fh, int start_state,
+    int *err, gchar **err_info)
 {
-	yy_fh = fh;
-	yyrestart(0);
-	g_encap = WTAP_ENCAP_UNKNOWN;
-	ok_frame = FALSE;
-	is_k12text = FALSE;
-	at_eof = FALSE;
-	junk_chars = 0;
-	error_str = NULL;
-	file_bytes_read=0;
-	g_h=0;
-	g_m=0;
-	g_s=0;
-	g_ns=0;
-	g_ms=0;
-	ii=0;
+	yyscan_t scanner = NULL;
+
+	if (yylex_init(&scanner) != 0) {
+		/* errno is set if this fails */
+		*err = errno;
+		*err_info = NULL;
+		return FALSE;
+	}
+	state->fh = fh;
+	state->err = err;
+	state->err_info = err_info;
+	state->start_state = start_state;
+
+	state->g_encap = WTAP_ENCAP_UNKNOWN;
+	state->ok_frame = FALSE;
+	state->is_k12text = FALSE;
+	state->at_eof = FALSE;
+	state->junk_chars = 0;
+	state->error_str = NULL;
+	state->file_bytes_read=0;
+	state->g_h=0;
+	state->g_m=0;
+	state->g_s=0;
+	state->g_ns=0;
+	state->g_ms=0;
+	state->ii=0;
+
+	/* Associate the state with the scanner */
+	k12text_set_extra(state, scanner);
+
+	yylex(scanner);
+	yylex_destroy(scanner);
+	if (*err != 0 && *err != WTAP_ERR_SHORT_READ) {
+		/* I/O error. */
+		return FALSE;
+	}
+	return TRUE;
 }
 
 static gboolean
 k12text_read(wtap *wth, int *err, char ** err_info, gint64 *data_offset)
 {
 	k12text_t *k12text = (k12text_t *)wth->priv;
+	k12text_state_t state;
 
 	/*
 	 * We seek to the file position after the end of the previous frame
@@ -273,30 +353,29 @@ k12text_read(wtap *wth, int *err, char ** err_info, gint64 *data_offset)
 	if ( file_seek(wth->fh, k12text->next_frame_offset, SEEK_SET, err) == -1) {
 		return FALSE;
 	}
-	k12text_reset(wth->fh);		/* init lexer buffer and vars set by lexer */
-
-	BEGIN(NEXT_FRAME);
-	yylex();
+	if (!k12text_run_scanner(&state, wth->fh, NEXT_FRAME, err, err_info)) {
+		return FALSE;
+	}
 
-	if (ok_frame == FALSE) {
-		if (at_eof) {
+	if (state.ok_frame == FALSE) {
+		if (state.at_eof) {
 			*err = 0;
 			*err_info = NULL;
 		} else {
 			*err = WTAP_ERR_BAD_FILE;
-			*err_info = error_str;
+			*err_info = state.error_str;
 		}
 		return FALSE;
 	}
 
-	*data_offset = k12text->next_frame_offset;       /* file position for beginning of this frame   */
-	k12text->next_frame_offset += file_bytes_read;   /* file position after end of this frame       */
+	*data_offset = k12text->next_frame_offset;           /* file position for beginning of this frame   */
+	k12text->next_frame_offset += state.file_bytes_read; /* file position after end of this frame       */
 
-	if (!k12text_set_headers(&wth->phdr, err, err_info))
+	if (!k12text_set_headers(&wth->phdr, &state, err, err_info))
 		return FALSE;
 
 	ws_buffer_assure_space(wth->frame_buffer, wth->phdr.caplen);
-	memcpy(ws_buffer_start_ptr(wth->frame_buffer), bb, wth->phdr.caplen);
+	memcpy(ws_buffer_start_ptr(wth->frame_buffer), state.bb, wth->phdr.caplen);
 
 	return TRUE;
 }
@@ -304,30 +383,31 @@ k12text_read(wtap *wth, int *err, char ** err_info, gint64 *data_offset)
 static gboolean
 k12text_seek_read(wtap *wth, gint64 seek_off, struct wtap_pkthdr *phdr, Buffer *buf, int *err, char **err_info)
 {
+	k12text_state_t state;
+
 	if ( file_seek(wth->random_fh, seek_off, SEEK_SET, err) == -1) {
 		return FALSE;
 	}
-	k12text_reset(wth->random_fh);		/* init lexer buffer and vars set by lexer */
-
-	BEGIN(NEXT_FRAME);
-	yylex();
+	if (!k12text_run_scanner(&state, wth->random_fh, NEXT_FRAME, err, err_info)) {
+		return FALSE;
+	}
 
-	if (ok_frame == FALSE) {
+	if (state.ok_frame == FALSE) {
 		*err = WTAP_ERR_BAD_FILE;
-		if (at_eof) {
+		if (state.at_eof) {
 			/* What happened ? The desired frame was previously read without a problem */
 			*err_info = g_strdup("Unexpected EOF (program error ?)");
 		} else {
-			*err_info = error_str;
+			*err_info = state.error_str;
 		}
 		return FALSE;
 	}
 
-	if (!k12text_set_headers(phdr, err, err_info))
+	if (!k12text_set_headers(phdr, &state, err, err_info))
 		return FALSE;
 
 	ws_buffer_assure_space(buf, phdr->caplen);
-	memcpy(ws_buffer_start_ptr(buf), bb, phdr->caplen);
+	memcpy(ws_buffer_start_ptr(buf), state.bb, phdr->caplen);
 
 	return TRUE;
 }
@@ -336,13 +416,17 @@ wtap_open_return_val
 k12text_open(wtap *wth, int *err, gchar **err_info _U_)
 {
 	k12text_t *k12text;
+	k12text_state_t state;
 
-	k12text_reset(wth->fh);       /* init lexer buffer and vars set by lexer */
-
-	BEGIN(MAGIC);
-	yylex();
+	if (!k12text_run_scanner(&state, wth->fh, MAGIC, err, err_info)) {
+		return WTAP_OPEN_ERROR;
+	}
 
-	if (! is_k12text) return WTAP_OPEN_NOT_MINE;
+	if (!state.is_k12text) {
+		/* *err might have been set to WTAP_ERR_SHORT_READ */
+		*err = 0;
+		return WTAP_OPEN_NOT_MINE;
+	}
 
 	if ( file_seek(wth->fh, 0, SEEK_SET, err) == -1) {
 		return WTAP_OPEN_ERROR;