nvme: add NVMe Data responses (via RDMA)

The commit contains a general framework for parsing NVMe Fabrics data
responses, which contain only "pure" data. These packets are received
as a response for Data requests inside the SGLs in NVMe commands.

Change-Id: I05f8130df6eef37795d258be680f673930ab6e34
Signed-off-by: Nitzan Carmi <nitzanc@mellanox.com>
Reviewed-by: Parav Pandit <parav@mellanox.com>
Tested-by: Nitzan Carmi <nitzanc@mellanox.com>
Reviewed-on: https://code.wireshark.org/review/22207
Petri-Dish: Alexis La Goutte <alexis.lagoutte@gmail.com>
Tested-by: Petri Dish Buildbot <buildbot-no-reply@wireshark.org>
Reviewed-by: Anders Broman <a.broman58@gmail.com>

3 files changed, 172 insertions, 14 deletions

diff --git a/epan/dissectors/packet-nvme-rdma.c b/epan/dissectors/packet-nvme-rdma.c
index 6d94505893..3d34b9aa54 100644
--- a/epan/dissectors/packet-nvme-rdma.c
+++ b/epan/dissectors/packet-nvme-rdma.c
@@ -312,6 +312,8 @@ find_add_q_ctx(packet_info *pinfo, conversation_t *conv)
         q_ctx = wmem_new(wmem_file_scope(), struct nvme_rdma_q_ctx);
         q_ctx->n_q_ctx.pending_cmds = wmem_tree_new(wmem_file_scope());
         q_ctx->n_q_ctx.done_cmds = wmem_tree_new(wmem_file_scope());
+        q_ctx->n_q_ctx.data_requests = wmem_tree_new(wmem_file_scope());
+        q_ctx->n_q_ctx.data_responses = wmem_tree_new(wmem_file_scope());
         q_ctx->n_q_ctx.qid = qid;
         conversation_add_proto_data(conv, proto_nvme_rdma, q_ctx);
     }
@@ -431,7 +433,7 @@ static void dissect_nvme_fabric_connect_cmd(proto_tree *cmd_tree, tvbuff_t *cmd_
 {
     proto_tree_add_item(cmd_tree, hf_nvme_rdma_cmd_connect_rsvd1, cmd_tvb,
                         5, 19, ENC_NA);
-    dissect_nvme_cmd_sgl(cmd_tvb, cmd_tree, hf_nvme_rdma_cmd_connect_sgl1);
+    dissect_nvme_cmd_sgl(cmd_tvb, cmd_tree, hf_nvme_rdma_cmd_connect_sgl1, NULL);
     proto_tree_add_item(cmd_tree, hf_nvme_rdma_cmd_connect_recfmt, cmd_tvb,
                         40, 2, ENC_LITTLE_ENDIAN);
     proto_tree_add_item(cmd_tree, hf_nvme_rdma_cmd_connect_qid, cmd_tvb,
@@ -645,6 +647,10 @@ dissect_nvme_rdma_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_t
         cmd_ctx->n_cmd_ctx.fabric = FALSE;
         dissect_nvme_cmd(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx,
                          &cmd_ctx->n_cmd_ctx);
+        if (cmd_ctx->n_cmd_ctx.remote_key) {
+            nvme_add_data_request(pinfo, &q_ctx->n_q_ctx,
+                                  &cmd_ctx->n_cmd_ctx, (void*)cmd_ctx);
+        }
     }
 }
 
@@ -783,6 +789,8 @@ dissect_nvme_to_host(tvbuff_t *nvme_tvb, packet_info *pinfo,
                      struct infinibandinfo *info,
                      struct nvme_rdma_q_ctx *q_ctx, guint len)
 {
+    struct nvme_rdma_cmd_ctx *cmd_ctx;
+
     switch (info->opCode) {
     case RC_SEND_ONLY:
     case RC_SEND_ONLY_INVAL:
@@ -792,6 +800,26 @@ dissect_nvme_to_host(tvbuff_t *nvme_tvb, packet_info *pinfo,
             proto_tree_add_item(nvme_tree, hf_nvme_rdma_to_host_unknown_data, nvme_tvb,
                     0, len, ENC_NA);
         break;
+    case RC_RDMA_WRITE_ONLY:
+    case RC_RDMA_WRITE_FIRST:
+        if (!PINFO_FD_VISITED(pinfo)) {
+            cmd_ctx = (struct nvme_rdma_cmd_ctx*)
+                       nvme_lookup_data_request(&q_ctx->n_q_ctx,
+                                                info->reth_remote_key);
+            if (cmd_ctx) {
+                cmd_ctx->n_cmd_ctx.data_resp_pkt_num = pinfo->num;
+                nvme_add_data_response(&q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx,
+                                       info->reth_remote_key);
+            }
+        } else {
+            cmd_ctx = (struct nvme_rdma_cmd_ctx*)
+                       nvme_lookup_data_response(pinfo, &q_ctx->n_q_ctx,
+                                                 info->reth_remote_key);
+        }
+        if (cmd_ctx)
+            dissect_nvme_data_response(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx,
+                                       &cmd_ctx->n_cmd_ctx, len);
+        break;
     default:
         proto_tree_add_item(nvme_tree, hf_nvme_rdma_to_host_unknown_data, nvme_tvb,
                 0, len, ENC_NA);
diff --git a/epan/dissectors/packet-nvme.c b/epan/dissectors/packet-nvme.c
index f846368f3f..7d192bca38 100644
--- a/epan/dissectors/packet-nvme.c
+++ b/epan/dissectors/packet-nvme.c
@@ -95,6 +95,9 @@ static int hf_nvme_cmd_pkt = -1;
 static int hf_nvme_cqe_pkt = -1;
 static int hf_nvme_cmd_latency = -1;
 
+/* Data response fields */
+static int hf_nvme_gen_data = -1;
+
 /* Initialize the subtree pointers */
 static gint ett_data = -1;
 
@@ -287,6 +290,7 @@ nvme_add_cmd_to_pending_list(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
     cmd_ctx->cqe_pkt_num = 0;
     cmd_ctx->cmd_start_time = pinfo->abs_ts;
     nstime_set_zero(&cmd_ctx->cmd_end_time);
+    cmd_ctx->remote_key = 0;
 
     /* this is a new cmd, create a new command context and map it to the
        unmatched table
@@ -304,6 +308,59 @@ void* nvme_lookup_cmd_in_pending_list(struct nvme_q_ctx *q_ctx, guint16 cmd_id)
     return wmem_tree_lookup32_array(q_ctx->pending_cmds, cmd_key);
 }
 
+void nvme_add_data_request(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
+                           struct nvme_cmd_ctx *cmd_ctx, void *ctx)
+{
+    wmem_tree_key_t cmd_key[3];
+    guint32 key = cmd_ctx->remote_key;
+
+    cmd_ctx->data_req_pkt_num = pinfo->num;
+    cmd_ctx->data_resp_pkt_num = 0;
+    nvme_build_pending_cmd_key(cmd_key, &key);
+    wmem_tree_insert32_array(q_ctx->data_requests, cmd_key, (void *)ctx);
+}
+
+void* nvme_lookup_data_request(struct nvme_q_ctx *q_ctx, guint32 key)
+{
+    wmem_tree_key_t cmd_key[3];
+
+    nvme_build_pending_cmd_key(cmd_key, &key);
+    return wmem_tree_lookup32_array(q_ctx->data_requests, cmd_key);
+}
+
+void
+nvme_add_data_response(struct nvme_q_ctx *q_ctx,
+                       struct nvme_cmd_ctx *cmd_ctx, guint32 rkey)
+{
+    wmem_tree_key_t cmd_key[3];
+    guint32 key = rkey;
+    guint32 frame_num;
+
+    nvme_build_done_cmd_key(cmd_key, &key, &frame_num);
+
+    /* Found matching data response packet. Add entries to the matched table
+     * for cmd and response packets
+     */
+    frame_num = cmd_ctx->data_req_pkt_num;
+    wmem_tree_insert32_array(q_ctx->data_responses, cmd_key, (void*)cmd_ctx);
+
+    frame_num = cmd_ctx->data_resp_pkt_num;
+    wmem_tree_insert32_array(q_ctx->data_responses, cmd_key, (void*)cmd_ctx);
+}
+
+void*
+nvme_lookup_data_response(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
+                          guint32 rkey)
+{
+    wmem_tree_key_t cmd_key[3];
+    guint32 key = rkey;
+    guint32 frame_num = pinfo->num;
+
+    nvme_build_done_cmd_key(cmd_key, &key, &frame_num);
+
+    return wmem_tree_lookup32_array(q_ctx->data_responses, cmd_key);
+}
+
 void
 nvme_add_cmd_cqe_to_done_list(struct nvme_q_ctx *q_ctx,
                               struct nvme_cmd_ctx *cmd_ctx, guint16 cmd_id)
@@ -382,7 +439,7 @@ nvme_publish_cmd_to_cqe_link(proto_tree *cmd_tree, tvbuff_t *cmd_tvb,
 }
 
 void dissect_nvme_cmd_sgl(tvbuff_t *cmd_tvb, proto_tree *cmd_tree,
-                          int field_index)
+                          int field_index, struct nvme_cmd_ctx *cmd_ctx)
 {
     proto_item *ti, *sgl_tree, *type_item, *sub_type_item;
     guint8 sgl_identifier, desc_type, desc_sub_type;
@@ -427,6 +484,9 @@ void dissect_nvme_cmd_sgl(tvbuff_t *cmd_tvb, proto_tree *cmd_tree,
                             offset + 12, 3, ENC_NA);
         break;
     case NVME_CMD_SGL_KEYED_DATA_DESC:
+        if (cmd_ctx)
+            cmd_ctx->remote_key = tvb_get_guint32(cmd_tvb, offset + 11,
+                                                  ENC_LITTLE_ENDIAN);
         proto_tree_add_item(sgl_tree, hf_nvme_cmd_sgl_desc_addr, cmd_tvb,
                             offset, 8, ENC_LITTLE_ENDIAN);
         proto_tree_add_item(sgl_tree, hf_nvme_cmd_sgl_desc_len, cmd_tvb,
@@ -486,8 +546,10 @@ dissect_nvme_rwc_common_word_10_11_12_14_15(tvbuff_t *cmd_tvb, proto_tree *cmd_t
                         62, 2, ENC_LITTLE_ENDIAN);
 }
 
-static void dissect_nvme_identify_cmd(tvbuff_t *cmd_tvb, proto_tree *cmd_tree)
+static void dissect_nvme_identify_cmd(tvbuff_t *cmd_tvb, proto_tree *cmd_tree,
+                                      struct nvme_cmd_ctx *cmd_ctx)
 {
+    cmd_ctx->resp_type = tvb_get_guint16(cmd_tvb, 40, ENC_LITTLE_ENDIAN);
     proto_tree_add_item(cmd_tree, hf_nvme_identify_cns, cmd_tvb,
                         40, 2, ENC_LITTLE_ENDIAN);
     proto_tree_add_item(cmd_tree, hf_nvme_identify_rsvd, cmd_tvb,
@@ -528,12 +590,50 @@ static void dissect_nvme_rw_cmd(tvbuff_t *cmd_tvb, proto_tree *cmd_tree)
 }
 
 void
+dissect_nvme_data_response(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
+                 struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx, guint len)
+{
+    proto_tree *cmd_tree;
+    proto_item *ti;
+    const guint8 *str_opcode;
+
+    col_set_str(pinfo->cinfo, COL_PROTOCOL, "NVMe");
+    ti = proto_tree_add_item(root_tree, proto_nvme, nvme_tvb, 0,
+                             len, ENC_NA);
+    cmd_tree = proto_item_add_subtree(ti, ett_data);
+    if (q_ctx->qid) { //IOQ
+        str_opcode = val_to_str(cmd_ctx->opcode, ioq_opc_tbl,
+                                "Unknown IOQ Opcode");
+        switch (cmd_ctx->opcode) {
+        case NVME_IOQ_OPC_READ:
+        case NVME_IOQ_OPC_WRITE:
+        default:
+            proto_tree_add_bytes_format_value(cmd_tree, hf_nvme_gen_data,
+                                              nvme_tvb, 0, len, NULL,
+                                              "%s", str_opcode);
+            break;
+        }
+    } else { //AQ
+        str_opcode = val_to_str(cmd_ctx->opcode, aq_opc_tbl,
+                                "Unknown AQ Opcode");
+        switch (cmd_ctx->opcode) {
+        default:
+            proto_tree_add_bytes_format_value(cmd_tree, hf_nvme_gen_data,
+                                              nvme_tvb, 0, len, NULL,
+                                              "%s", str_opcode);
+            break;
+        }
+    }
+    col_add_lstr(pinfo->cinfo, COL_INFO, "NVMe ", str_opcode, ": Data",
+                 COL_ADD_LSTR_TERMINATOR);
+}
+
+void
 dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
                  struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx)
 {
     proto_tree *cmd_tree;
     proto_item *ti, *opc_item;
-    guint8 opcode;
 
     col_set_str(pinfo->cinfo, COL_PROTOCOL, "NVMe");
     ti = proto_tree_add_item(root_tree, proto_nvme, nvme_tvb, 0,
@@ -541,15 +641,15 @@ dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
     proto_item_append_text(ti, " (Cmd)");
     cmd_tree = proto_item_add_subtree(ti, ett_data);
 
-    opcode = tvb_get_guint8(nvme_tvb, 0);
+    cmd_ctx->opcode = tvb_get_guint8(nvme_tvb, 0);
     opc_item = proto_tree_add_item(cmd_tree, hf_nvme_cmd_opc, nvme_tvb,
                         0, 1, ENC_LITTLE_ENDIAN);
     if (q_ctx->qid)
         proto_item_append_text(opc_item, " %s",
-                               val_to_str(opcode, ioq_opc_tbl, "Reserved"));
+                               val_to_str(cmd_ctx->opcode, ioq_opc_tbl, "Reserved"));
     else
         proto_item_append_text(opc_item, " %s",
-                               val_to_str(opcode, aq_opc_tbl, "Reserved"));
+                               val_to_str(cmd_ctx->opcode, aq_opc_tbl, "Reserved"));
 
     nvme_publish_cmd_to_cqe_link(cmd_tree, nvme_tvb, hf_nvme_cqe_pkt, cmd_ctx);
 
@@ -568,10 +668,10 @@ dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
     proto_tree_add_item(cmd_tree, hf_nvme_cmd_mptr, nvme_tvb,
                         16, 8, ENC_LITTLE_ENDIAN);
 
-    dissect_nvme_cmd_sgl(nvme_tvb, cmd_tree, hf_nvme_cmd_sgl);
+    dissect_nvme_cmd_sgl(nvme_tvb, cmd_tree, hf_nvme_cmd_sgl, cmd_ctx);
 
-    if (q_ctx->qid) {
-        switch (opcode) {
+    if (q_ctx->qid) { //IOQ
+        switch (cmd_ctx->opcode) {
         case NVME_IOQ_OPC_READ:
         case NVME_IOQ_OPC_WRITE:
             dissect_nvme_rw_cmd(nvme_tvb, cmd_tree);
@@ -579,10 +679,10 @@ dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
         default:
             break;
         }
-    } else {
-        switch (opcode) {
+    } else { //AQ
+        switch (cmd_ctx->opcode) {
         case NVME_AQ_OPC_IDENTIFY:
-            dissect_nvme_identify_cmd(nvme_tvb, cmd_tree);
+            dissect_nvme_identify_cmd(nvme_tvb, cmd_tree, cmd_ctx);
             break;
         default:
             break;
@@ -823,6 +923,10 @@ proto_register_nvme(void)
               FT_DOUBLE, BASE_NONE, NULL, 0x0,
               "The time between the command and completion, in usec", HFILL }
         },
+        { &hf_nvme_gen_data,
+            { "Nvme Data", "nvme.data",
+              FT_BYTES, BASE_NONE, NULL, 0, NULL, HFILL}
+        },
     };
     static gint *ett[] = {
         &ett_data,
diff --git a/epan/dissectors/packet-nvme.h b/epan/dissectors/packet-nvme.h
index 7f2fcdd2d2..d127e91338 100644
--- a/epan/dissectors/packet-nvme.h
+++ b/epan/dissectors/packet-nvme.h
@@ -30,6 +30,8 @@
 struct nvme_q_ctx {
     wmem_tree_t *pending_cmds;
     wmem_tree_t *done_cmds;
+    wmem_tree_t *data_requests;
+    wmem_tree_t *data_responses;
     guint16     qid;
 };
 
@@ -37,9 +39,16 @@ struct nvme_cmd_ctx {
     guint32 cmd_pkt_num;  /* pkt number of the cmd */
     guint32 cqe_pkt_num;  /* pkt number of the cqe */
 
+    guint32 data_req_pkt_num;
+    guint32 data_resp_pkt_num;
+
     nstime_t cmd_start_time;
     nstime_t cmd_end_time;
     gboolean fabric;     /* indicate whether cmd fabric type or not */
+
+    guint8  opcode;
+    guint32 remote_key;
+    guint16 resp_type;
 };
 
 void
@@ -63,6 +72,17 @@ nvme_add_cmd_to_pending_list(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
                              void *ctx, guint16 cmd_id);
 void* nvme_lookup_cmd_in_pending_list(struct nvme_q_ctx *q_ctx, guint16 cmd_id);
 
+void nvme_add_data_request(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
+                           struct nvme_cmd_ctx *cmd_ctx, void *ctx);
+void* nvme_lookup_data_request(struct nvme_q_ctx *q_ctx, guint32 key);
+
+void
+nvme_add_data_response(struct nvme_q_ctx *q_ctx,
+                       struct nvme_cmd_ctx *cmd_ctx, guint32 rkey);
+void*
+nvme_lookup_data_response(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
+                          guint32 rkey);
+
 void
 nvme_add_cmd_cqe_to_done_list(struct nvme_q_ctx *q_ctx,
                               struct nvme_cmd_ctx *cmd_ctx, guint16 cmd_id);
@@ -71,10 +91,16 @@ nvme_lookup_cmd_in_done_list(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
                              guint16 cmd_id);
 
 void dissect_nvme_cmd_sgl(tvbuff_t *cmd_tvb, proto_tree *cmd_tree,
-                          int field_index);
+                          int field_index, struct nvme_cmd_ctx *cmd_ctx);
+
 void
 dissect_nvme_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
                  struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx);
+
+void
+dissect_nvme_data_response(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
+                 struct nvme_q_ctx *q_ctx, struct nvme_cmd_ctx *cmd_ctx, guint len);
+
 void
 dissect_nvme_cqe(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
                  struct nvme_cmd_ctx *cmd_ctx);