From 7d1049d32694dba11d4e7558b269e8c6db5e2bce Mon Sep 17 00:00:00 2001 From: Nitzan Carmi Date: Sun, 4 Jun 2017 09:45:04 +0000 Subject: nvme: add NVMe Data responses (via RDMA) The commit contains a general framework for parsing NVMe Fabrics data responses, which contain only "pure" data. These packets are received as a response for Data requests inside the SGLs in NVMe commands. Change-Id: I05f8130df6eef37795d258be680f673930ab6e34 Signed-off-by: Nitzan Carmi Reviewed-by: Parav Pandit Tested-by: Nitzan Carmi Reviewed-on: https://code.wireshark.org/review/22207 Petri-Dish: Alexis La Goutte Tested-by: Petri Dish Buildbot Reviewed-by: Anders Broman --- epan/dissectors/packet-nvme-rdma.c | 30 ++++++++- epan/dissectors/packet-nvme.c | 128 +++++++++++++++++++++++++++++++++---- epan/dissectors/packet-nvme.h | 28 +++++++- 3 files changed, 172 insertions(+), 14 deletions(-) (limited to 'epan') diff --git a/epan/dissectors/packet-nvme-rdma.c b/epan/dissectors/packet-nvme-rdma.c index 6d94505893..3d34b9aa54 100644 --- a/epan/dissectors/packet-nvme-rdma.c +++ b/epan/dissectors/packet-nvme-rdma.c @@ -312,6 +312,8 @@ find_add_q_ctx(packet_info *pinfo, conversation_t *conv) q_ctx = wmem_new(wmem_file_scope(), struct nvme_rdma_q_ctx); q_ctx->n_q_ctx.pending_cmds = wmem_tree_new(wmem_file_scope()); q_ctx->n_q_ctx.done_cmds = wmem_tree_new(wmem_file_scope()); + q_ctx->n_q_ctx.data_requests = wmem_tree_new(wmem_file_scope()); + q_ctx->n_q_ctx.data_responses = wmem_tree_new(wmem_file_scope()); q_ctx->n_q_ctx.qid = qid; conversation_add_proto_data(conv, proto_nvme_rdma, q_ctx); } @@ -431,7 +433,7 @@ static void dissect_nvme_fabric_connect_cmd(proto_tree *cmd_tree, tvbuff_t *cmd_ { proto_tree_add_item(cmd_tree, hf_nvme_rdma_cmd_connect_rsvd1, cmd_tvb, 5, 19, ENC_NA); - dissect_nvme_cmd_sgl(cmd_tvb, cmd_tree, hf_nvme_rdma_cmd_connect_sgl1); + dissect_nvme_cmd_sgl(cmd_tvb, cmd_tree, hf_nvme_rdma_cmd_connect_sgl1, NULL); proto_tree_add_item(cmd_tree, hf_nvme_rdma_cmd_connect_recfmt, cmd_tvb, 40, 2, ENC_LITTLE_ENDIAN); proto_tree_add_item(cmd_tree, hf_nvme_rdma_cmd_connect_qid, cmd_tvb, @@ -645,6 +647,10 @@ dissect_nvme_rdma_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_t cmd_ctx->n_cmd_ctx.fabric = FALSE; dissect_nvme_cmd(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx); + if (cmd_ctx->n_cmd_ctx.remote_key) { + nvme_add_data_request(pinfo, &q_ctx->n_q_ctx, + &cmd_ctx->n_cmd_ctx, (void*)cmd_ctx); + } } } @@ -783,6 +789,8 @@ dissect_nvme_to_host(tvbuff_t *nvme_tvb, packet_info *pinfo, struct infinibandinfo *info, struct nvme_rdma_q_ctx *q_ctx, guint len) { + struct nvme_rdma_cmd_ctx *cmd_ctx; + switch (info->opCode) { case RC_SEND_ONLY: case RC_SEND_ONLY_INVAL: @@ -792,6 +800,26 @@ dissect_nvme_to_host(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree_add_item(nvme_tree, hf_nvme_rdma_to_host_unknown_data, nvme_tvb, 0, len, ENC_NA); break; + case RC_RDMA_WRITE_ONLY: + case RC_RDMA_WRITE_FIRST: + if (!PINFO_FD_VISITED(pinfo)) { + cmd_ctx = (struct nvme_rdma_cmd_ctx*) + nvme_lookup_data_request(&q_ctx->n_q_ctx, + info->reth_remote_key); + if (cmd_ctx) { + cmd_ctx->n_cmd_ctx.data_resp_pkt_num = pinfo->num; + nvme_add_data_response(&q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx, + info->reth_remote_key); + } + } else { + cmd_ctx = (struct nvme_rdma_cmd_ctx*) + nvme_lookup_data_response(pinfo, &q_ctx->n_q_ctx, + info->reth_remote_key); + } + if (cmd_ctx) + dissect_nvme_data_response(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx, + &cmd_ctx->n_cmd_ctx, len); + break; default: proto_tree_add_item(nvme_tree, hf_nvme_rdma_to_host_unknown_data, nvme_tvb, 0, len, ENC_NA); diff --git a/epan/dissectors/packet-nvme.c b/epan/dissectors/packet-nvme.c index f846368f3f..7d192bca38 100644 --- a/epan/dissectors/packet-nvme.c +++ b/epan/dissectors/packet-nvme.c @@ -95,6 +95,9 @@ static int hf_nvme_cmd_pkt = -1; static int hf_nvme_cqe_pkt = -1; static int hf_nvme_cmd_latency = -1; +/* Data response fields */ +static int hf_nvme_gen_data = -1; + /* Initialize the subtree pointers */ static gint ett_data = -1; @@ -287,6 +290,7 @@ nvme_add_cmd_to_pending_list(packet_info *pinfo, struct nvme_q_ctx *q_ctx, cmd_ctx->cqe_pkt_num = 0; cmd_ctx->cmd_start_time = pinfo->abs_ts; nstime_set_zero(&cmd_ctx->cmd_end_time); + cmd_ctx->remote_key = 0; /* this is a new cmd, create a new command context and map it to the unmatched table @@ -304,6 +308,59 @@ void* nvme_lookup_cmd_in_pending_list(struct nvme_q_ctx *q_ctx, guint16 cmd_id) return wmem_tree_lookup32_array(q_ctx->pending_cmds, cmd_key); } +void nvme_add_data_request(packet_info *pinfo, struct nvme_q_ctx *q_ctx, + struct nvme_cmd_ctx *cmd_ctx, void *ctx) +{ + wmem_tree_key_t cmd_key[3]; + guint32 key = cmd_ctx->remote_key; + + cmd_ctx->data_req_pkt_num = pinfo->num; + cmd_ctx->data_resp_pkt_num = 0; + nvme_build_pending_cmd_key(cmd_key, &key); + wmem_tree_insert32_array(q_ctx->data_requests, cmd_key, (void *)ctx); +} + +void* nvme_lookup_data_request(struct nvme_q_ctx *q_ctx, guint32 key) +{ + wmem_tree_key_t cmd_key[3]; + + nvme_build_pending_cmd_key(cmd_key, &key); + return wmem_tree_lookup32_array(q_ctx->data_requests, cmd_key); +} + +void +nvme_add_data_response(struct nvme_q_ctx *q_ctx, + struct nvme_cmd_ctx *cmd_ctx, guint32 rkey) +{ + wmem_tree_key_t cmd_key[3]; + guint32 key = rkey; + guint32 frame_num; + + nvme_build_done_cmd_key(cmd_key, &key, &frame_num); + + /* Found matching data response packet. Add entries to the matched table + * for cmd and response packets + */ + frame_num = cmd_ctx->data_req_pkt_num; + wmem_tree_insert32_array(q_ctx->data_responses, cmd_key, (void*)cmd_ctx); + + frame_num = cmd_ctx->data_resp_pkt_num; + wmem_tree_insert32_array(q_ctx->data_responses, cmd_key, (void*)cmd_ctx); +} + +void* +nvme_lookup_data_response(packet_info *pinfo, struct nvme_q_ctx *q_ctx, + guint32 rkey) +{ + wmem_tree_key_t cmd_key[3]; + guint32 key = rkey; + guint32 frame_num = pinfo->num; + + nvme_build_done_cmd_key(cmd_key, &key, &frame_num); + + return wmem_tree_lookup32_array(q_ctx->data_responses, cmd_key); +} + void nvme_add_cmd_cqe_to_done_list(struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx, guint16 cmd_id) @@ -382,7 +439,7 @@ nvme_publish_cmd_to_cqe_link(proto_tree *cmd_tree, tvbuff_t *cmd_tvb, } void dissect_nvme_cmd_sgl(tvbuff_t *cmd_tvb, proto_tree *cmd_tree, - int field_index) + int field_index, struct nvme_cmd_ctx *cmd_ctx) { proto_item *ti, *sgl_tree, *type_item, *sub_type_item; guint8 sgl_identifier, desc_type, desc_sub_type; @@ -427,6 +484,9 @@ void dissect_nvme_cmd_sgl(tvbuff_t *cmd_tvb, proto_tree *cmd_tree, offset + 12, 3, ENC_NA); break; case NVME_CMD_SGL_KEYED_DATA_DESC: + if (cmd_ctx) + cmd_ctx->remote_key = tvb_get_guint32(cmd_tvb, offset + 11, + ENC_LITTLE_ENDIAN); proto_tree_add_item(sgl_tree, hf_nvme_cmd_sgl_desc_addr, cmd_tvb, offset, 8, ENC_LITTLE_ENDIAN); proto_tree_add_item(sgl_tree, hf_nvme_cmd_sgl_desc_len, cmd_tvb, @@ -486,8 +546,10 @@ dissect_nvme_rwc_common_word_10_11_12_14_15(tvbuff_t *cmd_tvb, proto_tree *cmd_t 62, 2, ENC_LITTLE_ENDIAN); } -static void dissect_nvme_identify_cmd(tvbuff_t *cmd_tvb, proto_tree *cmd_tree) +static void dissect_nvme_identify_cmd(tvbuff_t *cmd_tvb, proto_tree *cmd_tree, + struct nvme_cmd_ctx *cmd_ctx) { + cmd_ctx->resp_type = tvb_get_guint16(cmd_tvb, 40, ENC_LITTLE_ENDIAN); proto_tree_add_item(cmd_tree, hf_nvme_identify_cns, cmd_tvb, 40, 2, ENC_LITTLE_ENDIAN); proto_tree_add_item(cmd_tree, hf_nvme_identify_rsvd, cmd_tvb, @@ -527,13 +589,51 @@ static void dissect_nvme_rw_cmd(tvbuff_t *cmd_tvb, proto_tree *cmd_tree) 53, 3, ENC_NA); } +void +dissect_nvme_data_response(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, + struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx, guint len) +{ + proto_tree *cmd_tree; + proto_item *ti; + const guint8 *str_opcode; + + col_set_str(pinfo->cinfo, COL_PROTOCOL, "NVMe"); + ti = proto_tree_add_item(root_tree, proto_nvme, nvme_tvb, 0, + len, ENC_NA); + cmd_tree = proto_item_add_subtree(ti, ett_data); + if (q_ctx->qid) { //IOQ + str_opcode = val_to_str(cmd_ctx->opcode, ioq_opc_tbl, + "Unknown IOQ Opcode"); + switch (cmd_ctx->opcode) { + case NVME_IOQ_OPC_READ: + case NVME_IOQ_OPC_WRITE: + default: + proto_tree_add_bytes_format_value(cmd_tree, hf_nvme_gen_data, + nvme_tvb, 0, len, NULL, + "%s", str_opcode); + break; + } + } else { //AQ + str_opcode = val_to_str(cmd_ctx->opcode, aq_opc_tbl, + "Unknown AQ Opcode"); + switch (cmd_ctx->opcode) { + default: + proto_tree_add_bytes_format_value(cmd_tree, hf_nvme_gen_data, + nvme_tvb, 0, len, NULL, + "%s", str_opcode); + break; + } + } + col_add_lstr(pinfo->cinfo, COL_INFO, "NVMe ", str_opcode, ": Data", + COL_ADD_LSTR_TERMINATOR); +} + void dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx) { proto_tree *cmd_tree; proto_item *ti, *opc_item; - guint8 opcode; col_set_str(pinfo->cinfo, COL_PROTOCOL, "NVMe"); ti = proto_tree_add_item(root_tree, proto_nvme, nvme_tvb, 0, @@ -541,15 +641,15 @@ dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, proto_item_append_text(ti, " (Cmd)"); cmd_tree = proto_item_add_subtree(ti, ett_data); - opcode = tvb_get_guint8(nvme_tvb, 0); + cmd_ctx->opcode = tvb_get_guint8(nvme_tvb, 0); opc_item = proto_tree_add_item(cmd_tree, hf_nvme_cmd_opc, nvme_tvb, 0, 1, ENC_LITTLE_ENDIAN); if (q_ctx->qid) proto_item_append_text(opc_item, " %s", - val_to_str(opcode, ioq_opc_tbl, "Reserved")); + val_to_str(cmd_ctx->opcode, ioq_opc_tbl, "Reserved")); else proto_item_append_text(opc_item, " %s", - val_to_str(opcode, aq_opc_tbl, "Reserved")); + val_to_str(cmd_ctx->opcode, aq_opc_tbl, "Reserved")); nvme_publish_cmd_to_cqe_link(cmd_tree, nvme_tvb, hf_nvme_cqe_pkt, cmd_ctx); @@ -568,10 +668,10 @@ dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, proto_tree_add_item(cmd_tree, hf_nvme_cmd_mptr, nvme_tvb, 16, 8, ENC_LITTLE_ENDIAN); - dissect_nvme_cmd_sgl(nvme_tvb, cmd_tree, hf_nvme_cmd_sgl); + dissect_nvme_cmd_sgl(nvme_tvb, cmd_tree, hf_nvme_cmd_sgl, cmd_ctx); - if (q_ctx->qid) { - switch (opcode) { + if (q_ctx->qid) { //IOQ + switch (cmd_ctx->opcode) { case NVME_IOQ_OPC_READ: case NVME_IOQ_OPC_WRITE: dissect_nvme_rw_cmd(nvme_tvb, cmd_tree); @@ -579,10 +679,10 @@ dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, default: break; } - } else { - switch (opcode) { + } else { //AQ + switch (cmd_ctx->opcode) { case NVME_AQ_OPC_IDENTIFY: - dissect_nvme_identify_cmd(nvme_tvb, cmd_tree); + dissect_nvme_identify_cmd(nvme_tvb, cmd_tree, cmd_ctx); break; default: break; @@ -823,6 +923,10 @@ proto_register_nvme(void) FT_DOUBLE, BASE_NONE, NULL, 0x0, "The time between the command and completion, in usec", HFILL } }, + { &hf_nvme_gen_data, + { "Nvme Data", "nvme.data", + FT_BYTES, BASE_NONE, NULL, 0, NULL, HFILL} + }, }; static gint *ett[] = { &ett_data, diff --git a/epan/dissectors/packet-nvme.h b/epan/dissectors/packet-nvme.h index 7f2fcdd2d2..d127e91338 100644 --- a/epan/dissectors/packet-nvme.h +++ b/epan/dissectors/packet-nvme.h @@ -30,6 +30,8 @@ struct nvme_q_ctx { wmem_tree_t *pending_cmds; wmem_tree_t *done_cmds; + wmem_tree_t *data_requests; + wmem_tree_t *data_responses; guint16 qid; }; @@ -37,9 +39,16 @@ struct nvme_cmd_ctx { guint32 cmd_pkt_num; /* pkt number of the cmd */ guint32 cqe_pkt_num; /* pkt number of the cqe */ + guint32 data_req_pkt_num; + guint32 data_resp_pkt_num; + nstime_t cmd_start_time; nstime_t cmd_end_time; gboolean fabric; /* indicate whether cmd fabric type or not */ + + guint8 opcode; + guint32 remote_key; + guint16 resp_type; }; void @@ -63,6 +72,17 @@ nvme_add_cmd_to_pending_list(packet_info *pinfo, struct nvme_q_ctx *q_ctx, void *ctx, guint16 cmd_id); void* nvme_lookup_cmd_in_pending_list(struct nvme_q_ctx *q_ctx, guint16 cmd_id); +void nvme_add_data_request(packet_info *pinfo, struct nvme_q_ctx *q_ctx, + struct nvme_cmd_ctx *cmd_ctx, void *ctx); +void* nvme_lookup_data_request(struct nvme_q_ctx *q_ctx, guint32 key); + +void +nvme_add_data_response(struct nvme_q_ctx *q_ctx, + struct nvme_cmd_ctx *cmd_ctx, guint32 rkey); +void* +nvme_lookup_data_response(packet_info *pinfo, struct nvme_q_ctx *q_ctx, + guint32 rkey); + void nvme_add_cmd_cqe_to_done_list(struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx, guint16 cmd_id); @@ -71,10 +91,16 @@ nvme_lookup_cmd_in_done_list(packet_info *pinfo, struct nvme_q_ctx *q_ctx, guint16 cmd_id); void dissect_nvme_cmd_sgl(tvbuff_t *cmd_tvb, proto_tree *cmd_tree, - int field_index); + int field_index, struct nvme_cmd_ctx *cmd_ctx); + void dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx); + +void +dissect_nvme_data_response(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, + struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx, guint len); + void dissect_nvme_cqe(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree, struct nvme_cmd_ctx *cmd_ctx); -- cgit v1.2.1