From 8fcdcb9c95e7436c4da02d39df048f4a1fbeca9c Mon Sep 17 00:00:00 2001 From: Hadriel Kaplan Date: Sun, 16 Aug 2015 12:37:11 -0400 Subject: Improve file merging for mergecap and wireshark Refactor the file merging code by removing the duplicate logic from mergecap.c and file.c's cf_merge_files(), into a new merge_files() function in merge.c. Also the following user-visible changes: * Removed the '-T' encap type option from mergecap, as it's illogical for mergecap and would complicate common merge code. * Input files with IDBs of different name, speed, tsprecision, etc., will produce an output PCAPNG file with separate IDBs, even if their encap types are the same. * Added a '-I' IDB merge mode option for mergecap, to control how IDBs are merged. * Changed Wireshark's drag-and-drop merging to use PCAPNG instead of PCAP. Bug: 8795 Bug: 7381 Change-Id: Icc30d217e093d6f40114422204afd2e332834f71 Reviewed-on: https://code.wireshark.org/review/10058 Petri-Dish: Hadriel Kaplan Tested-by: Petri Dish Buildbot Reviewed-by: Anders Broman --- file.c | 515 ++++++++++++++++++----------------------------------------------- 1 file changed, 144 insertions(+), 371 deletions(-) (limited to 'file.c') diff --git a/file.c b/file.c index 207ed6d4fb..77b59f89a5 100644 --- a/file.c +++ b/file.c @@ -1230,413 +1230,186 @@ read_packet(capture_file *cf, dfilter_t *dfcode, epan_dissect_t *edt, return row; } -cf_status_t -cf_merge_files(char **out_filenamep, int in_file_count, - char *const *in_filenames, int file_type, gboolean do_append) -{ - merge_in_file_t *in_files, *in_file; - char *out_filename; - char *tmpname; - int out_fd; - wtap_dumper *pdh; - int open_err, read_err, write_err, close_err; - gchar *err_info, *write_err_info = NULL; - int err_fileno; - int i; - gboolean got_read_error = FALSE, got_write_error = FALSE; - gint64 data_offset; - progdlg_t *progbar = NULL; - gboolean stop_flag = FALSE; - gint64 f_len, file_pos; - float progbar_val; - GTimeVal start_time; - gchar status_str[100]; + +typedef struct _callback_data_t { + gint64 f_len; gint64 progbar_nextstep; gint64 progbar_quantum; - gchar *display_basename; - int selected_frame_type; - gboolean fake_interface_ids = FALSE; - - /* open the input files */ - if (!merge_open_in_files(in_file_count, in_filenames, &in_files, - &open_err, &err_info, &err_fileno)) { - g_free(in_files); - cf_open_failure_alert_box(in_filenames[err_fileno], open_err, err_info, - FALSE, 0); - return CF_ERROR; - } - - if (*out_filenamep != NULL) { - out_filename = *out_filenamep; - out_fd = ws_open(out_filename, O_CREAT|O_TRUNC|O_BINARY, 0600); - if (out_fd == -1) - open_err = errno; - } else { - out_fd = create_tempfile(&tmpname, "wireshark"); - if (out_fd == -1) - open_err = errno; - out_filename = g_strdup(tmpname); - *out_filenamep = out_filename; - } - if (out_fd == -1) { - err_info = NULL; - merge_close_in_files(in_file_count, in_files); - g_free(in_files); - cf_open_failure_alert_box(out_filename, open_err, NULL, TRUE, file_type); - return CF_ERROR; - } - - selected_frame_type = merge_select_frame_type(in_file_count, in_files); - - /* If we are trying to merge a number of libpcap files with different encapsulation types - * change the output file type to pcapng and create SHB and IDB:s for the new file use the - * interface index stored in in_files per file to change the phdr before writing the datablock. - * XXX should it be an option to convert to pcapng? - * - * We need something similar when merging pcapng files possibly with an option to say - * the same interface(s) used in all in files. SHBs comments should be merged together. - */ - if ((selected_frame_type == WTAP_ENCAP_PER_PACKET)&&(file_type == WTAP_FILE_TYPE_SUBTYPE_PCAP)) { - /* Write output in pcapng format */ - wtapng_section_t *shb_hdr; - wtapng_iface_descriptions_t *idb_inf, *idb_inf_merge_file; - wtapng_if_descr_t int_data, *file_int_data; - GString *comment_gstr; - guint itf_count, itf_id = 0; - - fake_interface_ids = TRUE; - /* Create SHB info */ - shb_hdr = wtap_file_get_shb_for_new_file(in_files[0].wth); - comment_gstr = g_string_new(""); - g_string_append_printf(comment_gstr, "%s \n",shb_hdr->opt_comment); - g_string_append_printf(comment_gstr, "File created by merging: \n"); - file_type = WTAP_FILE_TYPE_SUBTYPE_PCAPNG; - - for (i = 0; i < in_file_count; i++) { - g_string_append_printf(comment_gstr, "File%d: %s \n",i+1,in_files[i].filename); - } - /* TODO: handle comments from each file being merged */ - if (shb_hdr->opt_comment) - g_free(shb_hdr->opt_comment); - shb_hdr->opt_comment = g_string_free(comment_gstr, FALSE); /* NULL if not available */ - shb_hdr->shb_user_appl = g_strdup("Wireshark"); /* NULL if not available, UTF-8 string containing the name */ - /* of the application used to create this section. */ - - /* TODO: handle name resolution info from each file being merged */ - - /* create fake IDB info */ - idb_inf = g_new(wtapng_iface_descriptions_t,1); - /* TODO make this the number of DIFFERENT encapsulation types - * check that snaplength is the same too? - */ - idb_inf->interface_data = g_array_new(FALSE, FALSE, sizeof(wtapng_if_descr_t)); - - for (i = 0; i < in_file_count; i++) { - idb_inf_merge_file = wtap_file_get_idb_info(in_files[i].wth); - for (itf_count = 0; itf_count < idb_inf_merge_file->interface_data->len; itf_count++) { - /* read the interface data from the in file to our combined interface data */ - file_int_data = &g_array_index (idb_inf_merge_file->interface_data, wtapng_if_descr_t, itf_count); - int_data.wtap_encap = file_int_data->wtap_encap; - int_data.time_units_per_second = file_int_data->time_units_per_second; - int_data.link_type = file_int_data->link_type; - int_data.snap_len = file_int_data->snap_len; - int_data.if_name = g_strdup(file_int_data->if_name); - int_data.opt_comment = NULL; - int_data.if_description = NULL; - int_data.if_speed = 0; - int_data.if_tsresol = file_int_data->if_tsresol; - int_data.if_filter_str = NULL; - int_data.bpf_filter_len = 0; - int_data.if_filter_bpf_bytes = NULL; - int_data.if_os = NULL; - int_data.if_fcslen = -1; - int_data.num_stat_entries = 0; /* Number of ISB:s */ - int_data.interface_statistics = NULL; - - g_array_append_val(idb_inf->interface_data, int_data); - } - g_free(idb_inf_merge_file); - - /* Set fake interface Id in per file data */ - in_files[i].interface_id = itf_id; - itf_id += itf_count; - } + GTimeVal start_time; + progdlg_t *progbar; + gboolean stop_flag; +} callback_data_t; - pdh = wtap_dump_fdopen_ng(out_fd, file_type, - selected_frame_type, - merge_max_snapshot_length(in_file_count, in_files), - FALSE /* compressed */, shb_hdr, idb_inf /* wtapng_iface_descriptions_t *idb_inf */, - NULL, &open_err); - if (pdh == NULL) { - ws_close(out_fd); - merge_close_in_files(in_file_count, in_files); - g_free(in_files); - cf_open_failure_alert_box(out_filename, open_err, err_info, TRUE, - file_type); - return CF_ERROR; - } +static gboolean +merge_callback(merge_event event, int num _U_, + const merge_in_file_t in_files[], const guint in_file_count, + void *data) +{ + guint i; + callback_data_t *cb_data = (callback_data_t*) data; - } else { + g_assert(cb_data != NULL); - pdh = wtap_dump_fdopen(out_fd, file_type, - selected_frame_type, - merge_max_snapshot_length(in_file_count, in_files), - FALSE /* compressed */, &open_err); - if (pdh == NULL) { - ws_close(out_fd); - merge_close_in_files(in_file_count, in_files); - g_free(in_files); - cf_open_failure_alert_box(out_filename, open_err, err_info, TRUE, - file_type); - return CF_ERROR; - } - } + switch (event) { - /* Get the sum of the sizes of all the files. */ - f_len = 0; - for (i = 0; i < in_file_count; i++) - f_len += in_files[i].size; + case MERGE_EVENT_INPUT_FILES_OPENED: + /* do nothing */ + break; - /* Update the progress bar when it gets to this value. */ - progbar_nextstep = 0; - /* When we reach the value that triggers a progress bar update, - bump that value by this amount. */ - progbar_quantum = f_len/N_PROGBAR_UPDATES; - /* Progress so far. */ - progbar_val = 0.0f; + case MERGE_EVENT_FRAME_TYPE_SELECTED: + /* do nothing */ + break; - g_get_current_time(&start_time); + case MERGE_EVENT_READY_TO_MERGE: + /* Get the sum of the sizes of all the files. */ + for (i = 0; i < in_file_count; i++) + cb_data->f_len += in_files[i].size; - /* do the merge (or append) */ - for (;;) { - if (do_append) - in_file = merge_append_read_packet(in_file_count, in_files, &read_err, - &err_info); - else - in_file = merge_read_packet(in_file_count, in_files, &read_err, - &err_info); - if (in_file == NULL) { - /* EOF */ - break; - } + /* When we reach the value that triggers a progress bar update, + bump that value by this amount. */ + cb_data->progbar_quantum = cb_data->f_len / N_PROGBAR_UPDATES; - if (read_err != 0) { - /* I/O error reading from in_file */ - got_read_error = TRUE; + g_get_current_time(&cb_data->start_time); break; - } - - /* Get the sum of the data offsets in all of the files. */ - data_offset = 0; - for (i = 0; i < in_file_count; i++) - data_offset += in_files[i].data_offset; - /* Create the progress bar if necessary. - We check on every iteration of the loop, so that it takes no - longer than the standard time to create it (otherwise, for a - large file, we might take considerably longer than that standard - time in order to get to the next progress bar step). */ - if (progbar == NULL) { - progbar = delayed_create_progress_dlg(NULL, "Merging", "files", - FALSE, &stop_flag, &start_time, progbar_val); - } + case MERGE_EVENT_PACKET_WAS_READ: + { + gint64 data_offset = 0; - /* Update the progress bar, but do it only N_PROGBAR_UPDATES times; - when we update it, we have to run the GTK+ main loop to get it - to repaint what's pending, and doing so may involve an "ioctl()" - to see if there's any pending input from an X server, and doing - that for every packet can be costly, especially on a big file. */ - if (data_offset >= progbar_nextstep) { - /* Get the sum of the seek positions in all of the files. */ - file_pos = 0; + /* Get the sum of the data offsets in all of the files. */ + data_offset = 0; for (i = 0; i < in_file_count; i++) - file_pos += wtap_read_so_far(in_files[i].wth); - progbar_val = (gfloat) file_pos / (gfloat) f_len; - if (progbar_val > 1.0f) { - /* Some file probably grew while we were reading it. - That "shouldn't happen", so we'll just clip the progress - value at 1.0. */ - progbar_val = 1.0f; - } - if (progbar != NULL) { - g_snprintf(status_str, sizeof(status_str), - "%" G_GINT64_MODIFIER "dKB of %" G_GINT64_MODIFIER "dKB", - file_pos / 1024, f_len / 1024); - update_progress_dlg(progbar, progbar_val, status_str); - } - progbar_nextstep += progbar_quantum; - } + data_offset += in_files[i].data_offset; - if (stop_flag) { - /* Well, the user decided to abort the merge. */ - break; - } + /* Create the progress bar if necessary. + We check on every iteration of the loop, so that it takes no + longer than the standard time to create it (otherwise, for a + large file, we might take considerably longer than that standard + time in order to get to the next progress bar step). */ + if (cb_data->progbar == NULL) { + cb_data->progbar = delayed_create_progress_dlg(NULL, "Merging", "files", + FALSE, &cb_data->stop_flag, &cb_data->start_time, 0.0f); + } - /* If we have WTAP_ENCAP_PER_PACKET and the infiles are of type - * WTAP_FILE_TYPE_SUBTYPE_PCAP, we need to set the interface id - * in the paket header = the interface index we used in the IDBs - * interface description for this file(encapsulation type). - */ - if (fake_interface_ids) { - struct wtap_pkthdr *phdr; + /* Update the progress bar, but do it only N_PROGBAR_UPDATES times; + when we update it, we have to run the GTK+ main loop to get it + to repaint what's pending, and doing so may involve an "ioctl()" + to see if there's any pending input from an X server, and doing + that for every packet can be costly, especially on a big file. */ + if (data_offset >= cb_data->progbar_nextstep) { + float progbar_val; + gint64 file_pos = 0; + /* Get the sum of the seek positions in all of the files. */ + for (i = 0; i < in_file_count; i++) + file_pos += wtap_read_so_far(in_files[i].wth); + + progbar_val = (gfloat) file_pos / (gfloat) cb_data->f_len; + if (progbar_val > 1.0f) { + /* Some file probably grew while we were reading it. + That "shouldn't happen", so we'll just clip the progress + value at 1.0. */ + progbar_val = 1.0f; + } - phdr = wtap_phdr(in_file->wth); - if (phdr->presence_flags & WTAP_HAS_INTERFACE_ID) { - phdr->interface_id += in_file->interface_id; - } else { - phdr->interface_id = in_file->interface_id; - phdr->presence_flags = phdr->presence_flags | WTAP_HAS_INTERFACE_ID; + if (cb_data->progbar != NULL) { + gchar status_str[100]; + g_snprintf(status_str, sizeof(status_str), + "%" G_GINT64_MODIFIER "dKB of %" G_GINT64_MODIFIER "dKB", + file_pos / 1024, cb_data->f_len / 1024); + update_progress_dlg(cb_data->progbar, progbar_val, status_str); + } + cb_data->progbar_nextstep += cb_data->progbar_quantum; + } } - } - if (!wtap_dump(pdh, wtap_phdr(in_file->wth), - wtap_buf_ptr(in_file->wth), &write_err, &write_err_info)) { - got_write_error = TRUE; break; - } + + case MERGE_EVENT_DONE: + /* We're done merging the files; destroy the progress bar if it was created. */ + if (cb_data->progbar != NULL) + destroy_progress_dlg(cb_data->progbar); + break; } - /* We're done merging the files; destroy the progress bar if it was created. */ - if (progbar != NULL) - destroy_progress_dlg(progbar); + return cb_data->stop_flag; +} - merge_close_in_files(in_file_count, in_files); - if (!got_write_error) { - if (!wtap_dump_close(pdh, &write_err)) - got_write_error = TRUE; - } else { - /* - * We already got a write error; no need to report another - * write error on close. - * - * Don't overwrite the earlier write error. - */ - (void)wtap_dump_close(pdh, &close_err); - } - if (got_read_error) { - /* - * Find the file on which we got the error, and report the error. - */ - for (i = 0; i < in_file_count; i++) { - if (in_files[i].state == GOT_ERROR) { - /* Put up a message box noting that a read failed somewhere along - the line. */ - display_basename = g_filename_display_basename(in_files[i].filename); - switch (read_err) { - - case WTAP_ERR_SHORT_READ: - simple_error_message_box( - "The capture file %s appears to have been cut short" - " in the middle of a packet.", display_basename); - break; - case WTAP_ERR_BAD_FILE: - simple_error_message_box( - "The capture file %s appears to be damaged or corrupt.\n(%s)", - display_basename, err_info); - g_free(err_info); - break; +cf_status_t +cf_merge_files(char **out_filenamep, int in_file_count, + char *const *in_filenames, int file_type, gboolean do_append) +{ + char *out_filename; + char *tmpname; + int out_fd; + int err = 0; + gchar *err_info = NULL; + int err_fileno; + merge_result status; + merge_progress_callback_t cb; - case WTAP_ERR_DECOMPRESS: - simple_error_message_box( - "The compressed capture file %s appears to be damaged or corrupt.\n" - "(%s)", display_basename, - err_info != NULL ? err_info : "no information supplied"); - g_free(err_info); - break; - default: - simple_error_message_box( - "An error occurred while reading the" - " capture file %s: %s.", - display_basename, wtap_strerror(read_err)); - break; - } - g_free(display_basename); - } - } + if (*out_filenamep != NULL) { + out_filename = *out_filenamep; + out_fd = ws_open(out_filename, O_CREAT|O_TRUNC|O_BINARY, 0600); + if (out_fd == -1) + err = errno; + } else { + out_fd = create_tempfile(&tmpname, "wireshark"); + if (out_fd == -1) + err = errno; + out_filename = g_strdup(tmpname); + *out_filenamep = out_filename; + } + if (out_fd == -1) { + cf_open_failure_alert_box(out_filename, err, NULL, TRUE, file_type); + return CF_ERROR; } - if (got_write_error) { - /* Put up an alert box for the write error. */ - if (write_err < 0) { - /* Wiretap error. */ - switch (write_err) { + /* prepare our callback routine */ + cb.callback_func = merge_callback; + cb.data = g_malloc0(sizeof(callback_data_t)); - case WTAP_ERR_UNWRITABLE_ENCAP: - /* - * This is a problem with the particular frame we're writing and - * the file type and subtype we're writing; note that, and report - * the frame number and file type/subtype. - */ - display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN"); - simple_error_message_box( - "Frame %u of \"%s\" has a network type that can't be saved in a \"%s\" file.", - in_file ? in_file->packet_num : 0, display_basename, - wtap_file_type_subtype_string(file_type)); - g_free(display_basename); - break; + /* merge the files */ + status = merge_files(out_fd, out_filename, file_type, + (const char *const *) in_filenames, in_file_count, + do_append, IDB_MERGE_MODE_ALL_SAME, 0 /* snaplen */, + "Wireshark", &cb, &err, &err_info, &err_fileno); - case WTAP_ERR_PACKET_TOO_LARGE: - /* - * This is a problem with the particular frame we're writing and - * the file type and subtype we're writing; note that, and report - * the frame number and file type/subtype. - */ - display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN"); - simple_error_message_box( - "Frame %u of \"%s\" is too large for a \"%s\" file.", - in_file ? in_file->packet_num : 0, display_basename, - wtap_file_type_subtype_string(file_type)); - g_free(display_basename); - break; + g_free(cb.data); - case WTAP_ERR_UNWRITABLE_REC_TYPE: - /* - * This is a problem with the particular record we're writing and - * the file type and subtype we're writing; note that, and report - * the record number and file type/subtype. - */ - display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN"); - simple_error_message_box( - "Record %u of \"%s\" has a record type that can't be saved in a \"%s\" file.", - in_file ? in_file->packet_num : 0, display_basename, - wtap_file_type_subtype_string(file_type)); - g_free(display_basename); - break; + switch (status) { + case MERGE_OK: + break; - case WTAP_ERR_UNWRITABLE_REC_DATA: - /* - * This is a problem with the particular record we're writing and - * the file type and subtype we're writing; note that, and report - * the frame number and file type/subtype. - */ - display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN"); - simple_error_message_box( - "Record %u of \"%s\" has data that can't be saved in a \"%s\" file.\n(%s)", - in_file ? in_file->packet_num : 0, display_basename, - wtap_file_type_subtype_string(file_type), - write_err_info != NULL ? write_err_info : "no information supplied"); - g_free(write_err_info); - g_free(display_basename); - break; + case MERGE_USER_ABORTED: + /* this isn't really an error, though we will return CF_ERROR later */ + break; - default: - display_basename = g_filename_display_basename(out_filename); - simple_error_message_box( - "An error occurred while writing to the file \"%s\": %s.", - out_filename, wtap_strerror(write_err)); - g_free(display_basename); - break; - } - } else { - /* OS error. */ - write_failure_alert_box(out_filename, write_err); - } + case MERGE_ERR_CANT_OPEN_INFILE: + cf_open_failure_alert_box(in_filenames[err_fileno], err, err_info, + FALSE, 0); + break; + + case MERGE_ERR_CANT_OPEN_OUTFILE: + cf_open_failure_alert_box(out_filename, err, err_info, TRUE, + file_type); + ws_close(out_fd); + break; + + case MERGE_ERR_CANT_READ_INFILE: /* fall through */ + case MERGE_ERR_BAD_PHDR_INTERFACE_ID: + case MERGE_ERR_CANT_WRITE_OUTFILE: + case MERGE_ERR_CANT_CLOSE_OUTFILE: + default: + simple_error_message_box("%s", err_info ? err_info : "unknown error"); + break; } - if (got_read_error || got_write_error || stop_flag) { + g_free(err_info); + + if (status != MERGE_OK) { /* Callers aren't expected to treat an error or an explicit abort differently - we put up error dialogs ourselves, so they don't have to. */ -- cgit v1.2.1