Because there's more than just text in XML...

DTDs are imported to create fields


svn path=/trunk/; revision=15851

5 files changed, 646 insertions, 190 deletions

diff --git a/epan/dissectors/packet-xml.c b/epan/dissectors/packet-xml.c
index 5c3ac3238a..e98598198c 100644
--- a/epan/dissectors/packet-xml.c
+++ b/epan/dissectors/packet-xml.c
@@ -31,10 +31,15 @@
 #include "config.h"
 #endif
 
+#ifdef HAVE_DIRENT_H
+#include <dirent.h>
+#endif
+
 #include <ctype.h>
 #include <stdlib.h>
 #include <string.h>
 #include <stdarg.h>
+#include <errno.h>
 
 #include <stdio.h>
 
@@ -44,58 +49,109 @@
 #include <epan/strutil.h>
 #include <epan/tvbparse.h>
 #include <epan/dtd.h>
+#include <epan/report_err.h>
+#include <epan/filesystem.h>
 
-typedef struct _xml_names_t {
+typedef struct _xml_ns_t {
+    /* the name of this namespace */ 
 	gchar* name;
-	gchar* longname;
-	gchar* blurb;
+    
+    /* its fully qualified name */ 
+	gchar* fqn;
+
+	/* the contents of the whole element from <> to </> */
 	int hf_tag;
+    
+	/* chunks of cdata from <> to </> excluding sub tags */
 	int hf_cdata;
+    
+    /* the subtree for its sub items  */
 	gint ett;
-	
-	gboolean is_root;
 
 	GHashTable* attributes;
-	GHashTable* elements;
-} xml_names_t;
+    /*  key:   the attribute name
+        value: hf_id of what's between quotes */
+	
+    /* the namespace's namespaces */
+    GHashTable* elements;
+    /*  key:   the element name
+        value: the child namespace */
+	
+	GPtrArray* element_names;
+    /* imported directly from the parser and used while building the namespace */
+    
+} xml_ns_t;
 
 typedef struct {
 	proto_tree* tree;
 	proto_item* item;
 	proto_item* last_item;
-	xml_names_t* ns;
+	xml_ns_t* ns;
 	int start_offset;
 } xml_frame_t;
 
+struct _attr_reg_data {
+	GArray* hf;
+	gchar* basename;
+};
+
+
 static gint ett_dtd = -1;
 static gint ett_xmpli = -1;
 
-static int hf_junk = -1;
 static int hf_unknowwn_attrib = -1;
 static int hf_comment = -1;
 static int hf_xmlpi = -1;
 static int hf_dtd_tag = -1;
 static int hf_doctype = -1;
 
-/* Dissector handles */
+/* dissector handles */
 static dissector_handle_t xml_handle;
 
-/* tokenizer defs */
+/* parser definitions */
 static tvbparse_wanted_t* want;
 static tvbparse_wanted_t* want_ignore;
 
 static GHashTable* xmpli_names;
 static GHashTable* media_types;
 
-static xml_names_t xml_ns = {"xml","eXtesible Markup Language","XML",-1,-1,-1,TRUE,NULL,NULL};
-static xml_names_t unknown_ns = {"","","",-1,-1,-1,TRUE,NULL,NULL};
-static xml_names_t* root_ns;
+static xml_ns_t xml_ns = {"xml","/",-1,-1,-1,NULL,NULL,NULL};
+static xml_ns_t unknown_ns = {"unknown","?",-1,-1,-1,NULL,NULL,NULL};
+static xml_ns_t* root_ns;
 
 #define XML_CDATA -1000
+#define XML_SCOPED_NAME -1001
 
-GArray* hf;
+
+GArray* hf_arr;
 GArray* ett_arr;
 
+static const gchar* default_media_types[] = {
+	"text/xml", 
+	"application/xml", 
+	"application/soap+xml", 
+	"application/xml-dtd", 
+	"text/vnd.wap.wml", 
+	"text/vnd.wap.si", 
+	"text/vnd.wap.sl", 
+	"text/vnd.wap.co", 
+	"text/vnd.wap.emn", 
+	"application/vnd.wv.csp+xml", 
+	"application/xcap-el+xml", 
+	"application/xcap-att+xml", 
+	"application/xcap-error+xml", 
+	"application/xcap-caps+xml", 
+	"application/auth-policy+xml", 
+	"application/smil", 
+	"application/cpim-pidf+xml", 
+	"application/rdf+xml", 
+	"application/xslt+xml", 
+	"application/mathml+xml", 
+	"image/svg+xml", 
+	"application/vnd.wv.csp.xml",
+};
+
+
 static void
 dissect_xml(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
 {
@@ -119,13 +175,13 @@ dissect_xml(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree)
 	root_ns = g_hash_table_lookup(media_types,pinfo->match_string);
 	
 	if (! root_ns ) {
-		root_ns = &unknown_ns;
+		root_ns = &xml_ns;
 	}
 	
 	current_frame->ns = root_ns;
 	
-	current_frame->item = proto_tree_add_item(tree,xml_ns.hf_tag,tvb,0,-1,FALSE);
-	current_frame->tree = proto_item_add_subtree(current_frame->item,xml_ns.ett);
+	current_frame->item = proto_tree_add_item(tree,current_frame->ns->hf_tag,tvb,0,-1,FALSE);
+	current_frame->tree = proto_item_add_subtree(current_frame->item,current_frame->ns->ett);
 	current_frame->last_item = current_frame->item;
 	
 	while(( tok = tvbparse_get(tt, want) )) ;
@@ -143,7 +199,7 @@ static void after_token(void* tvbparse_data, const void* wanted_data _U_, tvbpar
 	} else if ( tok->id > 0) {
 		hfid = tok->id;
 	} else {
-		hfid = hf_junk;
+		hfid = xml_ns.hf_cdata;
 	}
 	
 	pi = proto_tree_add_item(current_frame->tree, hfid, tok->tvb, tok->offset, tok->len, FALSE);
@@ -159,7 +215,8 @@ static void before_xmpli(void* tvbparse_data, const void* wanted_data _U_, tvbpa
 	proto_tree* pt;
 	tvbparse_elem_t* name_tok = tok->sub->next;
 	gchar* name = tvb_get_ephemeral_string(name_tok->tvb,name_tok->offset,name_tok->len);
-	xml_names_t* ns = g_hash_table_lookup(xmpli_names,name);
+	xml_ns_t* ns = g_hash_table_lookup(xmpli_names,name);
+
 	int hf_tag;
 	gint ett;
 	
@@ -208,20 +265,46 @@ static void before_tag(void* tvbparse_data, const void* wanted_data _U_, tvbpars
 	GPtrArray* stack = tvbparse_data;
 	xml_frame_t* current_frame = g_ptr_array_index(stack,stack->len - 1);
 	tvbparse_elem_t* name_tok = tok->sub->next;
-	gchar* name =tvb_get_ephemeral_string(name_tok->tvb,name_tok->offset,name_tok->len);
-	xml_names_t* ns = g_hash_table_lookup(current_frame->ns->elements,name);
+    gchar* root_name;
+	gchar* name;
+	xml_ns_t* ns;
 	xml_frame_t* new_frame;
 	proto_item* pi;
 	proto_tree* pt;
-	
-	g_strdown(name);
-	if (!ns) {
-		if (! ( ns = g_hash_table_lookup(root_ns->elements,name) ) ) {
-			ns = &unknown_ns;
-		}
-	}
-	
-	pi = proto_tree_add_item(current_frame->tree,ns->hf_tag,tok->tvb,tok->offset,tok->len,FALSE);
+
+	if (name_tok->sub->id == XML_SCOPED_NAME) {
+        tvbparse_elem_t* root_tok = name_tok->sub->sub;
+        tvbparse_elem_t* leaf_tok = name_tok->sub->sub->next->next;
+        xml_ns_t* nameroot_ns;
+        
+        root_name = tvb_get_ephemeral_string(root_tok->tvb,root_tok->offset,root_tok->len);
+        name = tvb_get_ephemeral_string(leaf_tok->tvb,leaf_tok->offset,leaf_tok->len);
+        
+        nameroot_ns = g_hash_table_lookup(xml_ns.elements,root_name);
+        
+        if(nameroot_ns) {
+            ns = g_hash_table_lookup(nameroot_ns->elements,name);
+            if (!ns) {
+                ns = &unknown_ns;                
+            }
+        } else {
+            ns = &unknown_ns;
+        }
+        
+    } else {
+        name = tvb_get_ephemeral_string(name_tok->tvb,name_tok->offset,name_tok->len);
+        g_strdown(name);
+        
+        ns = g_hash_table_lookup(current_frame->ns->elements,name);
+        
+        if (!ns) {
+            if (! ( ns = g_hash_table_lookup(root_ns->elements,name) ) ) {
+                ns = &unknown_ns;
+            }
+        }
+    }
+
+    pi = proto_tree_add_item(current_frame->tree,ns->hf_tag,tok->tvb,tok->offset,tok->len,FALSE);
 	proto_item_set_text(pi,tvb_format_text(tok->tvb,tok->offset,(name_tok->offset - tok->offset) + name_tok->len));
 	
 	pt = proto_item_add_subtree(pi,ns->ett);
@@ -354,11 +437,20 @@ static void unrecognized_token(void* tvbparse_data, const void* wanted_data _U_,
 
 
 void init_xml_parser(void) {	
-	tvbparse_wanted_t* want_name = tvbparse_chars(-1,0,0,"abcdefghijklmnopqrstuvwxyz-_:ABCDEFGHIJKLMNOPQRSTUVWXYZ",NULL,NULL,NULL);
-
+	tvbparse_wanted_t* want_name = tvbparse_chars(-1,0,0,"abcdefghijklmnopqrstuvwxyz-_ABCDEFGHIJKLMNOPQRSTUVWXYZ",NULL,NULL,NULL);
+    
+    tvbparse_wanted_t* want_tag_name = tvbparse_set_oneof(0, NULL, NULL, NULL,
+                                                          tvbparse_set_seq(XML_SCOPED_NAME, NULL, NULL, NULL,
+                                                                           want_name,
+                                                                           tvbparse_char(-1,":",NULL,NULL,NULL),
+                                                                           want_name,
+                                                                           NULL),
+                                                          want_name,
+                                                          NULL);
+        
 	tvbparse_wanted_t* want_attributes = tvbparse_one_or_more(-1, NULL, NULL, NULL,
 															  tvbparse_set_seq(-1, NULL, NULL, after_attrib,
-																			   want_name,
+                                                                               tvbparse_chars(-1,0,0,"abcdefghijklmnopqrstuvwxyz-_ABCDEFGHIJKLMNOPQRSTUVWXYZ:",NULL,NULL,NULL),
 																			   tvbparse_char(-1,"=",NULL,NULL,NULL),
 																			   tvbparse_set_oneof(0, NULL, NULL, get_attrib_value,
 																								  tvbparse_quoted(-1, NULL, NULL, tvbparse_shrink_token_cb,'\"','\\'),
@@ -398,7 +490,7 @@ void init_xml_parser(void) {
 							  tvbparse_set_seq(0,NULL,NULL,after_untag,
 											   tvbparse_char(-1, "<", NULL, NULL, NULL),
 											   tvbparse_char(-1, "/", NULL, NULL, NULL),
-											   want_name,
+											   want_tag_name,
 											   tvbparse_char(-1, ">", NULL, NULL, NULL),
 											   NULL),
 							  tvbparse_set_seq(-1,NULL,before_dtd_doctype,NULL,
@@ -435,7 +527,7 @@ void init_xml_parser(void) {
 											   NULL),
 							  tvbparse_set_seq(-1, NULL, before_tag, NULL,
 											   tvbparse_char(-1,"<",NULL,NULL,NULL),
-											   want_name,
+											   want_tag_name,
 											   tvbparse_set_oneof(-1,NULL,NULL,NULL,
 																  tvbparse_set_seq(-1,NULL,NULL,NULL,
 																				   want_attributes,
@@ -452,21 +544,19 @@ void init_xml_parser(void) {
 }
 
 
-xml_names_t* xml_new_namespace(GHashTable* hash, gchar* name, gchar* longname, gchar* blurb, ...) {
-	xml_names_t* ns = g_malloc(sizeof(xml_names_t));
+xml_ns_t* xml_new_namespace(GHashTable* hash, gchar* name, ...) {
+	xml_ns_t* ns = g_malloc(sizeof(xml_ns_t));
 	va_list ap;
 	gchar* attr_name;
 	
 	ns->name = g_strdup(name);
-	ns->longname = g_strdup(longname);
-	ns->blurb = g_strdup(blurb);
 	ns->hf_tag = -1;
 	ns->hf_cdata = -1;
 	ns->ett = -1;
 	ns->attributes = g_hash_table_new(g_str_hash,g_str_equal);
 	ns->elements = g_hash_table_new(g_str_hash,g_str_equal);
 	
-	va_start(ap,blurb);
+	va_start(ap,name);
 	
 	while(( attr_name = va_arg(ap,gchar*) )) {
 		int* hfp = g_malloc(sizeof(int));
@@ -481,18 +571,18 @@ xml_names_t* xml_new_namespace(GHashTable* hash, gchar* name, gchar* longname, g
 	return ns;
 }
 
-void add_xml_attribute_names(gpointer k, gpointer v, gpointer p) {
-	gchar* basename = g_strdup_printf("%s.%s",(gchar*)p,(gchar*)k);
+
+void add_xml_field(GArray* hfs, int* p_id, gchar* name, gchar* fqn) {
 	hf_register_info hfri;
 	
-	hfri.p_id = (int*)v;
-	hfri.hfinfo.name = basename;
-	hfri.hfinfo.abbrev = basename;
+	hfri.p_id = p_id;
+	hfri.hfinfo.name = name;
+	hfri.hfinfo.abbrev = fqn;
 	hfri.hfinfo.type = FT_STRING;
 	hfri.hfinfo.display = BASE_NONE;
 	hfri.hfinfo.strings = NULL;
 	hfri.hfinfo.bitmask = 0x0;
-	hfri.hfinfo.blurb = basename;
+	hfri.hfinfo.blurb = "";
 	hfri.hfinfo.id = 0;
 	hfri.hfinfo.parent = 0;
 	hfri.hfinfo.ref_count = 0;
@@ -500,55 +590,469 @@ void add_xml_attribute_names(gpointer k, gpointer v, gpointer p) {
 	hfri.hfinfo.same_name_next = NULL;
 	hfri.hfinfo.same_name_prev = NULL;
 	
-	g_array_append_val(hf,hfri);
+	g_array_append_val(hfs,hfri);	
+}
+
+void add_xml_attribute_names(gpointer k, gpointer v, gpointer p) {
+	struct _attr_reg_data* d = p;
+	gchar* basename = g_strdup_printf("%s.%s",d->basename,(gchar*)k);
+	add_xml_field(d->hf, (int*) v, (gchar*)k, basename);
 }
 
+
 void add_xmlpi_namespace(gpointer k _U_, gpointer v, gpointer p) {
-	xml_names_t* ns = v;
-	hf_register_info hfri;
+	xml_ns_t* ns = v;
 	gchar* basename = g_strdup_printf("%s.%s",(gchar*)p,ns->name);
 	gint* ett_p = &(ns->ett);
-	
-	hfri.p_id = &(ns->hf_tag);
-	hfri.hfinfo.name = basename;
-	hfri.hfinfo.abbrev = basename;
-	hfri.hfinfo.type = FT_STRING;
-	hfri.hfinfo.display = BASE_NONE;
-	hfri.hfinfo.strings = NULL;
-	hfri.hfinfo.bitmask = 0x0;
-	hfri.hfinfo.blurb = basename;
-	hfri.hfinfo.id = 0;
-	hfri.hfinfo.parent = 0;
-	hfri.hfinfo.ref_count = 0;
-	hfri.hfinfo.bitshift = 0;
-	hfri.hfinfo.same_name_next = NULL;
-	hfri.hfinfo.same_name_prev = NULL;
+	struct _attr_reg_data d;
+
+	add_xml_field(hf_arr, &(ns->hf_tag), basename, basename);
 
-	g_array_append_val(hf,hfri);
 	g_array_append_val(ett_arr,ett_p);
 	
-	g_hash_table_foreach(ns->attributes,add_xml_attribute_names,basename);
+	d.basename = basename;
+	d.hf = hf_arr;
+	
+	g_hash_table_foreach(ns->attributes,add_xml_attribute_names,&d);
 
 }
 
-void init_xml_names(void) {
-	xml_names_t* xmlpi_xml_ns;
+static void destroy_dtd_data(dtd_build_data_t* dtd_data) {
 
+	if(dtd_data->proto_name) g_free(dtd_data->proto_name);
+	if(dtd_data->media_type) g_free(dtd_data->media_type);
+	if(dtd_data->description) g_free(dtd_data->description);
+	if(dtd_data->proto_root) g_free(dtd_data->proto_root);
+	
+	if(dtd_data->location) g_free(dtd_data->location);
+	g_string_free(dtd_data->error,TRUE);
+	
+
+	while(dtd_data->elements->len) {
+		dtd_named_list_t* nl = g_ptr_array_remove_index_fast(dtd_data->elements,0);
+		g_ptr_array_free(nl->list,TRUE);
+		g_free(nl);
+	}
+
+	g_ptr_array_free(dtd_data->elements,FALSE);
+	
+	while(dtd_data->attributes->len) {
+		dtd_named_list_t* nl = g_ptr_array_remove_index_fast(dtd_data->elements,0);
+		g_ptr_array_free(nl->list,TRUE);
+		g_free(nl);
+	}
+
+	g_ptr_array_free(dtd_data->attributes,FALSE);
+
+	g_free(dtd_data);
+
+}
+
+
+static void copy_attrib_item(gpointer k, gpointer v _U_, gpointer p) {
+	gchar* key = g_strdup(k);
+	int* value = g_malloc(sizeof(int));
+	GHashTable* dst = p;
+	
+	*value = -1;
+	g_hash_table_insert(dst,key,value);
+	
+}
+
+static GHashTable* copy_attributes_hash(GHashTable* src) {
+	GHashTable* dst = g_hash_table_new(g_str_hash,g_str_equal);
+	
+	g_hash_table_foreach(src,copy_attrib_item,dst);
+	
+	return dst;
+}
+
+static xml_ns_t* duplicate_element(xml_ns_t* orig) {
+	xml_ns_t* new_item  = g_malloc(sizeof(xml_ns_t));
+	guint i;
+	
+	new_item->name = g_strdup(orig->name);
+	new_item->hf_tag = -1;
+	new_item->hf_cdata = -1;
+	new_item->ett = -1;
+	new_item->attributes = copy_attributes_hash(orig->attributes);
+	new_item->elements =  g_hash_table_new(g_str_hash,g_str_equal);
+	new_item->element_names = g_ptr_array_new();
+	
+	for(i=0; i < orig->element_names->len; i++) {
+		g_ptr_array_add(new_item->element_names,
+						   g_ptr_array_index(orig->element_names,i));
+	}
+	
+	return new_item;
+}
+
+static gchar* fully_qualified_name(GPtrArray* hier, gchar* name) {
+	guint i;
+	GString* s = g_string_new("");
+	gchar* str;
+	
+	for (i = 0; i < hier->len; i++) {
+		g_string_sprintfa(s, "%s.",(gchar*)g_ptr_array_index(hier,i));
+	}
+	
+	g_string_append(s,name);
+	str = s->str;
+	g_string_free(s,FALSE);
+	
+	return str;
+}
+
+
+static xml_ns_t* make_xml_hier(gchar* elem_name,
+								  xml_ns_t* root,
+								  GHashTable* elements,
+								  GPtrArray* hier,
+								  GString* error,
+								  GArray* hfs,
+								  GArray* etts) {
+	xml_ns_t* new;
+	xml_ns_t* orig = g_hash_table_lookup(elements,elem_name);
+	gchar* fqn;
+	gint* ett_p;
+	struct _attr_reg_data d;
+    gboolean recurred = FALSE;
+    
+    if ( g_str_equal(elem_name,root->name) ) {
+        return NULL;
+    }
+
+	if (! orig) {
+		g_string_sprintfa(error,"element '%s' is not defined\n", elem_name);
+		return NULL;
+	}
+        
+	fqn = fully_qualified_name(hier,elem_name);
+    
+	new = duplicate_element(orig);
+	new->fqn = fqn;
+    
+    add_xml_field(hfs, &(new->hf_tag), elem_name, fqn);
+	add_xml_field(hfs, &(new->hf_cdata), elem_name, fqn);
+
+	ett_p = &new->ett;
+	g_array_append_val(etts,ett_p);
+
+	d.basename = fqn;
+	d.hf = hfs;
+	
+	g_hash_table_foreach(new->attributes,add_xml_attribute_names,&d);
+
+	while(new->element_names->len) {
+		gchar* child_name = g_ptr_array_remove_index(new->element_names,0);
+		xml_ns_t* child_element = NULL;
+		guint i;
+        
+        for (i = 0; i < hier->len; i++) {
+            if( strcmp(child_name,(gchar*) g_ptr_array_index(hier,i) ) == 0 ) {
+                recurred = TRUE;
+            }
+        }
+                
+		if( ! recurred ) {
+			g_ptr_array_add(hier,elem_name);
+			child_element = make_xml_hier(child_name, root, elements, hier,error,hfs,etts);
+			g_ptr_array_remove_index_fast(hier,hier->len - 1);
+		}
+		
+		if (child_element) {
+			g_hash_table_insert(new->elements,child_element->name,child_element);
+		}
+    }
+	
+	g_ptr_array_free(new->element_names,FALSE);
+	new->element_names = NULL;
+	return new;
+}
+
+static void register_dtd(dtd_build_data_t* dtd_data, GString* errors) {
+	GHashTable* elements = g_hash_table_new(g_str_hash,g_str_equal);
+	gchar* root_name = NULL;
+	xml_ns_t* root_element = NULL;
+	GArray* hfs;
+	GArray* etts;
+	GPtrArray* hier;
+	gchar* curr_name;
+    GPtrArray* element_names = g_ptr_array_new();
+    
+    /* we first populate elements with the those coming from the parser */
+	while(dtd_data->elements->len) {
+		dtd_named_list_t* nl = g_ptr_array_remove_index(dtd_data->elements,0);
+		xml_ns_t* element = g_malloc(sizeof(xml_ns_t));
+		
+		/* we will use the first element found as root in case no other one was given. */
+		if (root_name == NULL) root_name = g_strdup(nl->name);
+		
+		element->name = nl->name;
+		element->element_names = nl->list;
+		element->hf_tag = -1;
+		element->hf_cdata = -1;
+		element->ett = -1;
+		element->attributes = g_hash_table_new(g_str_hash,g_str_equal);
+		element->elements = g_hash_table_new(g_str_hash,g_str_equal);
+		
+		g_hash_table_insert(elements,element->name,element);
+		g_ptr_array_add(element_names,element->name);
+		
+		g_free(nl);
+	}
+
+	/* then we add the attributes to its relative elements */
+	while(dtd_data->attributes->len) {
+		dtd_named_list_t* nl = g_ptr_array_remove_index(dtd_data->attributes,0);
+		xml_ns_t* element = g_hash_table_lookup(elements,nl->name);
+		
+		if (!element) {
+			g_string_sprintfa(errors,"no element %s is been defined\n", nl->name);
+
+            goto next_attribute;
+		}
+		
+		while(nl->list->len) {
+			gchar* name = g_ptr_array_remove_index(nl->list,0);
+			int* id_p = g_malloc(sizeof(int));
+            
+			*id_p = -1;
+			g_hash_table_insert(element->attributes,name,id_p);
+		}
+        
+next_attribute:
+		g_free(nl->name);
+		g_ptr_array_free(nl->list,FALSE);
+		g_free(nl);
+	}
+
+    
+    
+	if( dtd_data->proto_root ) {
+        if(root_name) g_free(root_name);
+        root_name = g_strdup(dtd_data->proto_root);
+	}
+
+	hier = g_ptr_array_new();
+
+	if( ! dtd_data->proto_name ) {
+		hfs = hf_arr;
+		etts = ett_arr;
+		g_ptr_array_add(hier,g_strdup("xml")); 
+		root_element = &xml_ns;
+	} else {
+	    hfs = g_array_new(FALSE,FALSE,sizeof(hf_register_info));
+	    etts = g_array_new(FALSE,FALSE,sizeof(gint*));
+	}
+    
+	root_element = g_malloc(sizeof(xml_ns_t));
+    root_element->name = g_strdup(root_name);
+    root_element->fqn = root_element->name;
+    root_element->hf_tag = -1;
+    root_element->hf_cdata = -1;
+    root_element->ett = -1;
+    root_element->elements = g_hash_table_new(g_str_hash,g_str_equal);
+    root_element->element_names = element_names;
+    
+    if (dtd_data->recursion) {
+        xml_ns_t* orig_root;
+        
+        make_xml_hier(root_name, root_element, elements,hier,errors,hfs,etts);
+        
+        g_hash_table_insert(root_element->elements,root_element->name,root_element);
+        
+        orig_root = g_hash_table_lookup(elements,root_name);
+        
+        if(orig_root) { 
+            struct _attr_reg_data d;
+            
+            d.basename = root_name;
+            d.hf = hfs;
+
+            root_element->attributes = copy_attributes_hash(orig_root->attributes);
+            g_hash_table_foreach(root_element->attributes,add_xml_attribute_names,&d);
+        } else {
+            root_element->attributes = g_hash_table_new(g_str_hash,g_str_equal);
+        }
+
+        g_ptr_array_add(hier,root_name);
+        
+        while(root_element->element_names->len) {
+            curr_name = g_ptr_array_remove_index(root_element->element_names,0);
+            
+            if( ! g_hash_table_lookup(root_element->elements,curr_name) ) {
+                xml_ns_t* new = make_xml_hier(curr_name, root_element, elements,hier,errors,hfs,etts);
+                g_hash_table_insert(root_element->elements,new->name,new);
+            }
+        }
+    } else {
+        g_ptr_array_add(hier,root_name);
+        
+        root_element->attributes = g_hash_table_new(g_str_hash,g_str_equal);
+
+        while(root_element->element_names->len) {
+            xml_ns_t* new;
+            gint* ett_p;
+            struct _attr_reg_data d;
+            
+            curr_name = g_ptr_array_remove_index(root_element->element_names,0);
+            new = duplicate_element(g_hash_table_lookup(elements,curr_name));
+            new->fqn = fully_qualified_name(hier, curr_name);
+
+            add_xml_field(hfs, &(new->hf_tag), curr_name, new->fqn);
+            add_xml_field(hfs, &(new->hf_cdata), curr_name, new->fqn);
+
+            d.basename = new->fqn;
+            d.hf = hfs;
+
+            g_hash_table_foreach(new->attributes,add_xml_attribute_names,&d);
+
+            ett_p = &new->ett;
+            g_array_append_val(etts,ett_p);
+            
+            g_hash_table_insert(root_element->elements,new->name,new);
+            
+        }
+    }
+
+    g_ptr_array_free(element_names,FALSE);
+    
+	g_ptr_array_free(hier,FALSE);
+	
+	if( dtd_data->proto_name ) {
+        gint* ett_p;
+
+		if ( ! dtd_data->description) {
+			dtd_data->description = g_strdup(root_name);
+		}
+		
+        ett_p = &root_element->ett;
+        g_array_append_val(etts,ett_p);
+
+        add_xml_field(hfs, &root_element->hf_cdata, root_element->name, root_element->fqn);
+            
+		root_element->hf_tag = proto_register_protocol(dtd_data->description, dtd_data->proto_name, root_element->name);
+		proto_register_field_array(root_element->hf_tag, (hf_register_info*)hfs->data, hfs->len);
+		proto_register_subtree_array((gint**)etts->data, etts->len);
+		
+		if (dtd_data->media_type) {
+			g_hash_table_insert(media_types,dtd_data->media_type,root_element);
+			dtd_data->media_type = NULL;
+		}
+        
+		dtd_data->description = NULL;
+		dtd_data->proto_name = NULL;
+		g_array_free(hfs,FALSE);
+		g_array_free(etts,FALSE);
+	}
+    
+    g_hash_table_insert(xml_ns.elements,root_element->name,root_element);
+    
+	destroy_dtd_data(dtd_data);
+}
+
+#if GLIB_MAJOR_VERSION < 2
+#  define DIRECTORY_T DIR
+#  define FILE_T struct dirent
+#  define OPENDIR_OP(name) opendir(name)
+#  define DIRGETNEXT_OP(dir) readdir(dir)
+#  define GETFNAME_OP(file) (gchar *)file->d_name
+#  define CLOSEDIR_OP(dir) closedir(dir)
+#else /* GLIB 2 */
+#  define DIRECTORY_T GDir
+#  define FILE_T gchar
+#  define OPENDIR_OP(name) g_dir_open(name, 0, dummy)
+#  define DIRGETNEXT_OP(dir) g_dir_read_name(dir)
+#  define GETFNAME_OP(file) (file);
+#  define CLOSEDIR_OP(dir) g_dir_close(dir)
+#endif
+
+void init_xml_names(void) {
+	xml_ns_t* xmlpi_xml_ns;
+	guint i;
+    DIRECTORY_T* dir;
+    const FILE_T* file;
+    const gchar* filename;
+    gchar* dirname;
+    
+#if GLIB_MAJOR_VERSION >= 2
+    GError** dummy = g_malloc(sizeof(GError *));
+    *dummy = NULL;
+#endif
+    
 	xmpli_names = g_hash_table_new(g_str_hash,g_str_equal);
 	media_types = g_hash_table_new(g_str_hash,g_str_equal);
 	
-	unknown_ns.elements = g_hash_table_new(g_str_hash,g_str_equal);
-	unknown_ns.attributes = g_hash_table_new(g_str_hash,g_str_equal);
+	unknown_ns.elements = xml_ns.elements = g_hash_table_new(g_str_hash,g_str_equal);
+	unknown_ns.attributes = xml_ns.attributes = g_hash_table_new(g_str_hash,g_str_equal);
 	
-	xmlpi_xml_ns = xml_new_namespace(xmpli_names,"xml","XML XMLPI","XML XMLPI",
-									 "version","encoding","standalone",NULL);
+	xmlpi_xml_ns = xml_new_namespace(xmpli_names,"xml","version","encoding","standalone",NULL);
 	
 	g_hash_table_destroy(xmlpi_xml_ns->elements);
 	xmlpi_xml_ns->elements = NULL;
+
+    
+    dirname = get_persconffile_path("dtds", FALSE);
+	
+	if (test_for_directory(dirname) != EISDIR) {
+		dirname = get_datafile_path("dtds");
+    }
+    
+	if (test_for_directory(dirname) == EISDIR) {
+        
+        if ((dir = OPENDIR_OP(dirname)) != NULL) {
+            while ((file = DIRGETNEXT_OP(dir)) != NULL) {
+                guint namelen;
+                filename = GETFNAME_OP(file);
+                
+                namelen = strlen(filename);
+                if ( namelen > 4 && ( g_strcasecmp(filename+(namelen-4),".dtd")  == 0 ) ) {
+                    GString* errors = g_string_new("");
+                    GString* preparsed = dtd_preparse(dirname, filename, errors);
+                    dtd_build_data_t* dtd_data;
+                    
+                    if (errors->len) {
+                        report_failure("Dtd Preparser in file %s%c%s: %s",dirname,G_DIR_SEPARATOR,filename,errors->str);
+                        continue;
+                    }
+                    
+                    dtd_data = dtd_parse(preparsed);
+                    
+                    g_string_free(preparsed,TRUE);
+                    
+                    if (dtd_data->error->len) {
+                        report_failure("Dtd Parser in file %s%c%s: %s",dirname,G_DIR_SEPARATOR,filename,dtd_data->error->str);
+                        destroy_dtd_data(dtd_data);
+                        continue;
+                    }
+                    
+                    register_dtd(dtd_data,errors);
+                    
+                    if (errors->len) {
+                        report_failure("Dtd Registration in file: %s%c%s: %s",dirname,G_DIR_SEPARATOR,filename,errors->str);
+                        g_string_free(errors,TRUE);
+                        continue;
+                    }
+                }
+            }
+            
+            CLOSEDIR_OP(dir);
+        }
+    }
+    
+	for(i=0;i<array_length(default_media_types);i++) {
+		if( ! g_hash_table_lookup(media_types,default_media_types[i]) ) {
+			g_hash_table_insert(media_types,(gpointer)default_media_types[i],&xml_ns);
+		}
+	}
 	
 	g_hash_table_foreach(xmpli_names,add_xmlpi_namespace,"xml.xmlpi");
-}
 
+#if GLIB_MAJOR_VERSION >= 2
+    g_free(dummy);
+#endif    
+}
 
 void
 proto_register_xml(void) {
@@ -568,25 +1072,30 @@ proto_register_xml(void) {
 		{ &hf_dtd_tag, {"DTD Tag", "xml.dtdtag", FT_STRING, BASE_NONE, NULL, 0, "", HFILL }},
 		{ &unknown_ns.hf_cdata, {"CDATA", "xml.cdata", FT_STRING, BASE_NONE, NULL, 0, "", HFILL }},
 		{ &unknown_ns.hf_tag, {"Tag", "xml.tag", FT_STRING, BASE_NONE, NULL, 0, "", HFILL }},
-		{ &hf_junk, {"Unknown", "xml.unknown", FT_STRING, BASE_NONE, NULL, 0, "", HFILL }}
+		{ &xml_ns.hf_cdata, {"Unknown", "xml.unknown", FT_STRING, BASE_NONE, NULL, 0, "", HFILL }}
 	};
 
-	hf = g_array_new(FALSE,FALSE,sizeof(hf_register_info));
+	hf_arr = g_array_new(FALSE,FALSE,sizeof(hf_register_info));
 	ett_arr = g_array_new(FALSE,FALSE,sizeof(gint*));
 
-	g_array_append_vals(hf,hf_base,array_length(hf_base));
+	g_array_append_vals(hf_arr,hf_base,array_length(hf_base));
 	g_array_append_vals(ett_arr,ett_base,array_length(ett_base));
 	
 	init_xml_names();
 
-	xml_ns.hf_tag = proto_register_protocol(xml_ns.blurb, xml_ns.longname, xml_ns.name);
+	xml_ns.hf_tag = proto_register_protocol("eXtensible Markup Language", "XML", xml_ns.name);
 
-	proto_register_field_array(xml_ns.hf_tag, (hf_register_info*)hf->data, hf->len);
+	proto_register_field_array(xml_ns.hf_tag, (hf_register_info*)hf_arr->data, hf_arr->len);
 	proto_register_subtree_array((gint**)ett_arr->data, ett_arr->len);
 	
 	register_dissector("xml", dissect_xml, xml_ns.hf_tag);
 	
 	init_xml_parser();
+    
+}
+
+void add_dissector_media(gpointer k, gpointer v _U_, gpointer p _U_) {
+	dissector_add_string("media_type", (gchar*)k, xml_handle);
 }
 
 void
@@ -595,33 +1104,6 @@ proto_reg_handoff_xml(void)
 	
 	xml_handle = find_dissector("xml");
 	
-	dissector_add_string("media_type", "text/xml", xml_handle);
-	dissector_add_string("media_type", "application/xml", xml_handle);
-	dissector_add_string("media_type", "application/soap+xml", xml_handle);
-	dissector_add_string("media_type", "application/xml-dtd", xml_handle);
-	/* WAP and OMA XML media */
-	dissector_add_string("media_type", "text/vnd.wap.wml", xml_handle);
-	dissector_add_string("media_type", "text/vnd.wap.si", xml_handle);
-	dissector_add_string("media_type", "text/vnd.wap.sl", xml_handle);
-	dissector_add_string("media_type", "text/vnd.wap.co", xml_handle);
-	dissector_add_string("media_type", "text/vnd.wap.emn", xml_handle);
-	dissector_add_string("media_type", "application/vnd.wv.csp+xml", xml_handle);
-	/* The Extensible Markup Language (XML) Configuration Access Protocol (XCAP)
-		* draft-ietf-simple-xcap-06
-		*/
-	dissector_add_string("media_type", "application/xcap-el+xml", xml_handle);
-	dissector_add_string("media_type", "application/xcap-att+xml", xml_handle);
-	dissector_add_string("media_type", "application/xcap-error+xml", xml_handle);
-	dissector_add_string("media_type", "application/xcap-caps+xml", xml_handle);
-	/* draft-ietf-simple-presence-rules-02 */
-	dissector_add_string("media_type", "application/auth-policy+xml", xml_handle);
-	/* Other */
-	dissector_add_string("media_type", "application/smil", xml_handle);
-	dissector_add_string("media_type", "application/cpim-pidf+xml", xml_handle);
-	dissector_add_string("media_type", "application/rdf+xml", xml_handle);
-	dissector_add_string("media_type", "application/xslt+xml", xml_handle);
-	dissector_add_string("media_type", "application/mathml+xml", xml_handle);
-	dissector_add_string("media_type", "image/svg+xml", xml_handle);
-	dissector_add_string("media_type", "application/vnd.wv.csp.xml", xml_handle);
+	g_hash_table_foreach(media_types,add_dissector_media,NULL);
 	
 }
diff --git a/epan/dtd.h b/epan/dtd.h
index 1db10fd371..cd2503024a 100644
--- a/epan/dtd.h
+++ b/epan/dtd.h
@@ -37,6 +37,7 @@ typedef struct _dtd_build_data_t {
 	gchar* media_type;
 	gchar* description;
 	gchar* proto_root;
+    gboolean recursion;
 
 	GPtrArray* elements;
 	GPtrArray* attributes;
@@ -55,7 +56,7 @@ typedef struct _dtd_named_list_t {
 	GPtrArray* list;
 } dtd_named_list_t;
 
-extern GString* dtd_preparse(gchar* dname, gchar* fname, GString* err);
+extern GString* dtd_preparse(const gchar* dname, const gchar* fname, GString* err);
 extern dtd_build_data_t* dtd_parse(GString* s);
 
 #endif
diff --git a/epan/dtd_grammar.lemon b/epan/dtd_grammar.lemon
index 153f320efb..b451749150 100644
--- a/epan/dtd_grammar.lemon
+++ b/epan/dtd_grammar.lemon
@@ -85,8 +85,31 @@ dtd ::= doctype.
 dtd ::= dtd_parts.
 
 doctype ::= TAG_START DOCTYPE_KW NAME(Name) OPEN_BRACKET dtd_parts CLOSE_BRACKET TAG_STOP. {
-	bd->proto_name = g_strdup(Name->text);
+    dtd_named_list_t* root;
+    GPtrArray* root_elems = g_ptr_array_new();
+    guint i;
+
+    if(! bd->proto_name) {
+        bd->proto_name = g_strdup(Name->text);
+    }
+
+    if(bd->proto_root)
+        g_free(bd->proto_root);
+
+	bd->proto_root = g_strdup(Name->text);
+    
 	g_strdown(bd->proto_name);
+    
+    for( i = 0; i< bd->elements->len; i++) {
+        dtd_named_list_t* el = g_ptr_array_index(bd->elements,i);
+        
+        g_ptr_array_add(root_elems,g_strdup(el->name));
+    }
+    
+    root = dtd_named_list_new(g_strdup(Name->text),root_elems);
+    
+    g_ptr_array_add(bd->elements,root);
+    
 }
 
 dtd_parts ::= dtd_parts element(Element). { g_ptr_array_add(bd->elements,Element); }
@@ -95,10 +118,10 @@ dtd_parts ::= element(Element). { g_ptr_array_add(bd->elements,Element); }
 dtd_parts ::= attlist(Attlist). { g_ptr_array_add(bd->attributes,Attlist); }
 
 %type   attlist				{ dtd_named_list_t* }
-attlist(A) ::= TAG_START ATTLIST_KW NAME(B) attrib_list(TheList) TAG_STOP. { A = dtd_named_list_new(B->text,TheList); }
+attlist(A) ::= TAG_START ATTLIST_KW NAME(B) attrib_list(TheList) TAG_STOP. { g_strdown(B->text); A = dtd_named_list_new(B->text,TheList); }
 
 %type element { dtd_named_list_t* }
-element(A) ::= TAG_START ELEMENT_KW NAME(B) sub_elements(C) TAG_STOP. { A = dtd_named_list_new(B->text,C); }
+element(A) ::= TAG_START ELEMENT_KW NAME(B) sub_elements(C) TAG_STOP. { g_strdown(B->text); A = dtd_named_list_new(B->text,C); }
 
 %type   attrib_list			{ GPtrArray* }
 attrib_list(A) ::= attrib_list(B) attrib(C). { g_ptr_array_add(B,C); A = B; }
diff --git a/epan/dtd_parse.l b/epan/dtd_parse.l
index 249681c9b2..b37b8d6944 100644
--- a/epan/dtd_parse.l
+++ b/epan/dtd_parse.l
@@ -62,13 +62,15 @@
 	static void set_media_type (gchar* val) { if(build_data->media_type) g_free(build_data->media_type); build_data->media_type = g_strdup(val); }
 	static void set_proto_root (gchar* val) { if(build_data->proto_root) g_free(build_data->proto_root); build_data->proto_root = g_strdup(val); }
 	static void set_description (gchar* val) { if(build_data->description) g_free(build_data->description); build_data->description = g_strdup(val); }
+	static void set_recursive (gchar* val) { build_data->recursion = ( g_strcasecmp(val,"yes") == 0 ) ? TRUE : FALSE; }
 
 	struct _proto_xmlpi_attr proto_attrs[] =
 	{
-		{ "name", set_proto_name },
+		{ "proto_name", set_proto_name },
 		{ "media", set_media_type },
 		{ "root", set_proto_root },
 		{ "description", set_description },
+        { "hierarchy", set_recursive },
 		{NULL,NULL}
 	};
 	
@@ -82,6 +84,9 @@
 
 %}
 
+comment_start "<!--"
+comment_stop "-->"
+
 start_xmlpi "<?"
 
 location_xmlpi "ethereal:location"
@@ -133,15 +138,21 @@ comma          ","
 pipe           "|"
 dquote         ["]
 
-name           [a-z][-a-z0-9_]*
+name           [a-z][-a-zA-Z0-9_]*
 dquoted        ["][^\"]*["]
 squoted        ['][^\']*[']
 
-%START DTD XMLPI LOCATION DONE PROTOCOL GET_ATTR_QUOTE GET_ATTR_VAL GET_ATTR_CLOSE_QUOTE
+%START DTD XMLPI LOCATION DONE PROTOCOL GET_ATTR_QUOTE GET_ATTR_VAL GET_ATTR_CLOSE_QUOTE IN_COMMENT
 %%
 
 {whitespace}            ;
 
+
+<DTD>{comment_start}		{ BEGIN IN_COMMENT; }
+<IN_COMMENT>[^-]?				|
+<IN_COMMENT>[-]					;
+<IN_COMMENT>{comment_stop}		{ BEGIN DTD; }
+
 <DTD>{start_xmlpi}		{
 	BEGIN XMLPI;
 }
@@ -167,6 +178,7 @@ squoted        ['][^\']*[']
 
 <PROTOCOL>{name} {
 	attr_name = g_strdup(yytext);
+    g_strdown(attr_name);
 	BEGIN GET_ATTR_QUOTE;
 }
 
@@ -292,13 +304,16 @@ extern dtd_build_data_t* dtd_parse(GString* s) {
 	build_data->media_type = NULL;
 	build_data->description = NULL;
 	build_data->proto_root = NULL;
-	
+	build_data->recursion = FALSE;
+    
 	build_data->elements = g_ptr_array_new();
 	build_data->attributes = g_ptr_array_new();
 
 	build_data->location = NULL;
 	build_data->error = g_string_new("");
 	
+    location = NULL;
+    
 	BEGIN DTD;
 	
 	yylex();
diff --git a/epan/dtd_preparse.l b/epan/dtd_preparse.l
index 660b9b00ca..f6b21a66ea 100644
--- a/epan/dtd_preparse.l
+++ b/epan/dtd_preparse.l
@@ -56,16 +56,14 @@ GHashTable* entities;
 gchar* entity_name;
 GString* error;
 
-gchar* dirname;
-gchar* filename;
+const gchar* dirname;
+const gchar* filename;
 guint linenum;
 
-gchar* textstr;
+GString* textstr;
 
 static gchar* replace_entity(gchar* s);
 static const gchar* location(void);
-static gchar* load_entity_file(gchar* filename);
-		/* [:blank:]+file[:blank:]*=[:blank:]*["] */
 
 %}
 xmlpi_start "<?"
@@ -95,7 +93,7 @@ entity        [%&][A-Za-z][-A-Za-z0-9_]*;
 
 whitespace [[blank:]]+
 newline    \n
-%START OUTSIDE IN_COMMENT IN_ENTITY NAMED_ENTITY IN_QUOTE ENTITY_DONE GET_FNAME_OPEN_QUOTE GET_FNAME GET_FNAME_CLOSE_QUOTE XMLPI
+%START OUTSIDE IN_COMMENT IN_ENTITY NAMED_ENTITY IN_QUOTE ENTITY_DONE XMLPI
 %%
 
 
@@ -126,77 +124,14 @@ newline    \n
 <IN_QUOTE>{percent}				|
 <IN_QUOTE>{non_quote}			|
 <IN_QUOTE>{escaped_quote}		g_string_append(current,yytext);
-<NAMED_ENTITY>{system}			{ BEGIN GET_FNAME_OPEN_QUOTE; }
-<GET_FNAME_OPEN_QUOTE>{quote}	{ BEGIN GET_FNAME; }
-<GET_FNAME>{filename}		    {
-	if ((textstr = load_entity_file(yytext))) {
-		g_hash_table_insert(entities,entity_name,textstr); 
-	} else {
-		yyterminate();
-	}
-	
-	BEGIN GET_FNAME_CLOSE_QUOTE;
+<NAMED_ENTITY>{system}			{
+    g_string_sprintfa(error,"at %s:%u: file inclusion is not supported!", filename, linenum);
+    yyterminate();
 }
-<GET_FNAME_CLOSE_QUOTE>{quote}  { BEGIN ENTITY_DONE; }
 <ENTITY_DONE>{special_stop}		{ current = output; g_string_append(current,"\n"); BEGIN OUTSIDE; }
 
 %%
 
-static gchar* load_entity_file(gchar* fname) {
-	gchar* fullname = g_strdup_printf("%s%s",dirname,fname);
-	gchar* save_filename = filename;
-	guint save_linenum = linenum; 
-	FILE* fp = fopen(fullname,"r");
-	GString* filetext;
-	gchar* retstr;
-	gchar c;
-	
-	g_free(fullname);
-	
-	if (!fp) {
-		g_string_sprintfa(error,"at %s:%u: could not load file %s: %s", filename, linenum, fname, strerror(errno));
-		return NULL;
-	}
-	
-	filename = fname;
-	linenum = 1;
-	
-	filetext = g_string_new(location());
-	
-	while( ! feof(fp)) {
-		
-		c = fgetc(fp);
-
-		if ( ferror(fp) ) {
-			g_string_sprintfa(error,"at %s:%u: problem reading file %s: %s", filename, linenum, fname, strerror(errno));
-
-			g_string_free(filetext,TRUE);
-			
-			filename = save_filename;
-			save_linenum = linenum;
-			
-			return NULL;
-		}
-		
-		
-		g_string_append_c(filetext,c);
-		
-		if(c == '\n') {
-			g_string_append(filetext,location());
-			linenum++;
-		}
-		
-	}
-
-	retstr = filetext->str;
-	g_string_free(filetext,FALSE);
-
-	filename = save_filename;
-	save_linenum = linenum;
-	
-	return retstr;
-}
-
 static gchar* replace_entity(gchar* entity) {
 	GString* replacement;
 	
@@ -207,7 +142,7 @@ static gchar* replace_entity(gchar* entity) {
 	if (replacement) {
 		return replacement->str;
 	} else {
-		g_string_sprintfa(error,"dtd_preparse: in file '%s': %s does not exists\n", filename, entity);
+		g_string_sprintfa(error,"dtd_preparse: in file '%s': entity %s does not exists\n", filename, entity);
 		return "";
 	}
 	
@@ -244,8 +179,8 @@ static gboolean free_gstring_hash_items(gpointer k,gpointer v,gpointer p _U_) {
 	return TRUE;
 }
 
-extern GString* dtd_preparse(gchar* dname, gchar* fname, GString* err) {
-	gchar* fullname = g_strdup_printf("%s%s",dname,fname);
+extern GString* dtd_preparse(const gchar* dname,const  gchar* fname, GString* err) {
+	gchar* fullname = g_strdup_printf("%s%c%s",dname,G_DIR_SEPARATOR,fname);
 
 	dirname = dname;
 	filename = fname;