libmultipath: ANA prioritzer: use nvme wrapper library
authorMartin Wilck <mwilck@suse.com>
Sun, 23 Dec 2018 22:21:21 +0000 (23:21 +0100)
committerChristophe Varoqui <christophe.varoqui@opensvc.com>
Mon, 7 Jan 2019 10:46:38 +0000 (11:46 +0100)
Use the previously introduced NVME wrapper library for
the passthrough commands from the ANA prioritizer. Discard
code duplicated from nvme-cli from the ana code itself.

Furthermore, make additional cleanups in the ANA prioritizer:

 - don't use the same enum for priorities and error codes
 - use char* arrays for error messages and state names
 - return -1 prio to libmultipath for all error cases
 - check for overflow in check_ana_state()
 - get_ana_info(): improve readability with is_anagrpid_const
 - priorities: PERSISTENT_LOSS state is worse than INACCESSIBLE
 and CHANGE

Cc: lijie <lijie34@huawei.com>
Signed-off-by: Martin Wilck <mwilck@suse.com>
libmultipath/prioritizers/Makefile
libmultipath/prioritizers/ana.c
libmultipath/prioritizers/ana.h [deleted file]

index 15afaba..4d80c20 100644 (file)
@@ -19,9 +19,13 @@ LIBS = \
        libpriordac.so \
        libprioweightedpath.so \
        libpriopath_latency.so \
-       libprioana.so \
        libpriosysfs.so
 
+ifneq ($(call check_file,/usr/include/linux/nvme_ioctl.h),0)
+       LIBS += libprioana.so
+       CFLAGS += -I../nvme
+endif
+
 all: $(LIBS)
 
 libprioalua.so: alua.o alua_rtpg.o
index c5aaa5f..6000d54 100644 (file)
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <stdbool.h>
+#include <libudev.h>
 
 #include "debug.h"
+#include "nvme-lib.h"
 #include "prio.h"
+#include "util.h"
 #include "structs.h"
-#include "ana.h"
 
 enum {
-       ANA_PRIO_OPTIMIZED              = 50,
-       ANA_PRIO_NONOPTIMIZED           = 10,
-       ANA_PRIO_INACCESSIBLE           = 5,
-       ANA_PRIO_PERSISTENT_LOSS        = 1,
-       ANA_PRIO_CHANGE                 = 0,
-       ANA_PRIO_RESERVED               = 0,
-       ANA_PRIO_GETCTRL_FAILED         = -1,
-       ANA_PRIO_NOT_SUPPORTED          = -2,
-       ANA_PRIO_GETANAS_FAILED         = -3,
-       ANA_PRIO_GETANALOG_FAILED       = -4,
-       ANA_PRIO_GETNSID_FAILED         = -5,
-       ANA_PRIO_GETNS_FAILED           = -6,
-       ANA_PRIO_NO_MEMORY              = -7,
-       ANA_PRIO_NO_INFORMATION         = -8,
+       ANA_ERR_GETCTRL_FAILED          = 1,
+       ANA_ERR_NOT_NVME,
+       ANA_ERR_NOT_SUPPORTED,
+       ANA_ERR_GETANAS_OVERFLOW,
+       ANA_ERR_GETANAS_NOTFOUND,
+       ANA_ERR_GETANALOG_FAILED,
+       ANA_ERR_GETNSID_FAILED,
+       ANA_ERR_GETNS_FAILED,
+       ANA_ERR_NO_MEMORY,
+       ANA_ERR_NO_INFORMATION,
 };
 
-static const char * anas_string[] = {
+static const char *ana_errmsg[] = {
+       [ANA_ERR_GETCTRL_FAILED]        = "couldn't get ctrl info",
+       [ANA_ERR_NOT_NVME]              = "not an NVMe device",
+       [ANA_ERR_NOT_SUPPORTED]         = "ANA not supported",
+       [ANA_ERR_GETANAS_OVERFLOW]      = "buffer overflow in ANA log",
+       [ANA_ERR_GETANAS_NOTFOUND]      = "NSID or ANAGRPID not found",
+       [ANA_ERR_GETANALOG_FAILED]      = "couldn't get ana log",
+       [ANA_ERR_GETNSID_FAILED]        = "couldn't get NSID",
+       [ANA_ERR_GETNS_FAILED]          = "couldn't get namespace info",
+       [ANA_ERR_NO_MEMORY]             = "out of memory",
+       [ANA_ERR_NO_INFORMATION]        = "invalid fd",
+};
+
+static const char *anas_string[] = {
        [NVME_ANA_OPTIMIZED]                    = "ANA Optimized State",
        [NVME_ANA_NONOPTIMIZED]                 = "ANA Non-Optimized State",
        [NVME_ANA_INACCESSIBLE]                 = "ANA Inaccessible State",
        [NVME_ANA_PERSISTENT_LOSS]              = "ANA Persistent Loss State",
        [NVME_ANA_CHANGE]                       = "ANA Change state",
-       [NVME_ANA_RESERVED]                     = "Invalid namespace group state!",
 };
 
 static const char *aas_print_string(int rc)
 {
        rc &= 0xff;
-
-       switch(rc) {
-       case NVME_ANA_OPTIMIZED:
-       case NVME_ANA_NONOPTIMIZED:
-       case NVME_ANA_INACCESSIBLE:
-       case NVME_ANA_PERSISTENT_LOSS:
-       case NVME_ANA_CHANGE:
+       if (rc >= 0 && rc < ARRAY_SIZE(anas_string) &&
+           anas_string[rc] != NULL)
                return anas_string[rc];
-       default:
-               return anas_string[NVME_ANA_RESERVED];
-       }
-
-       return anas_string[NVME_ANA_RESERVED];
-}
-
-static int nvme_get_nsid(int fd, unsigned *nsid)
-{
-       static struct stat nvme_stat;
-       int err = fstat(fd, &nvme_stat);
-       if (err < 0)
-               return 1;
-
-       if (!S_ISBLK(nvme_stat.st_mode)) {
-               condlog(0, "Error: requesting namespace-id from non-block device\n");
-               return 1;
-       }
-
-       *nsid = ioctl(fd, NVME_IOCTL_ID);
-       return 0;
-}
-
-static int nvme_submit_admin_passthru(int fd, struct nvme_passthru_cmd *cmd)
-{
-       return ioctl(fd, NVME_IOCTL_ADMIN_CMD, cmd);
-}
-
-int nvme_get_log13(int fd, __u32 nsid, __u8 log_id, __u8 lsp, __u64 lpo,
-                 __u16 lsi, bool rae, __u32 data_len, void *data)
-{
-       struct nvme_admin_cmd cmd = {
-               .opcode         = nvme_admin_get_log_page,
-               .nsid           = nsid,
-               .addr           = (__u64)(uintptr_t) data,
-               .data_len       = data_len,
-       };
-       __u32 numd = (data_len >> 2) - 1;
-       __u16 numdu = numd >> 16, numdl = numd & 0xffff;
-
-       cmd.cdw10 = log_id | (numdl << 16) | (rae ? 1 << 15 : 0);
-       if (lsp)
-               cmd.cdw10 |= lsp << 8;
-
-       cmd.cdw11 = numdu | (lsi << 16);
-       cmd.cdw12 = lpo;
-       cmd.cdw13 = (lpo >> 32);
-
-       return nvme_submit_admin_passthru(fd, &cmd);
-
-}
-
-int nvme_identify13(int fd, __u32 nsid, __u32 cdw10, __u32 cdw11, void *data)
-{
-       struct nvme_admin_cmd cmd = {
-               .opcode         = nvme_admin_identify,
-               .nsid           = nsid,
-               .addr           = (__u64)(uintptr_t) data,
-               .data_len       = NVME_IDENTIFY_DATA_SIZE,
-               .cdw10          = cdw10,
-               .cdw11          = cdw11,
-       };
 
-       return nvme_submit_admin_passthru(fd, &cmd);
+       return "invalid ANA state";
 }
 
-int nvme_identify(int fd, __u32 nsid, __u32 cdw10, void *data)
+static int get_ana_state(__u32 nsid, __u32 anagrpid, void *ana_log,
+                        size_t ana_log_len)
 {
-       return nvme_identify13(fd, nsid, cdw10, 0, data);
-}
-
-int nvme_identify_ctrl(int fd, void *data)
-{
-       return nvme_identify(fd, 0, NVME_ID_CNS_CTRL, data);
-}
-
-int nvme_identify_ns(int fd, __u32 nsid, void *data)
-{
-       return nvme_identify(fd, nsid, NVME_ID_CNS_NS, data);
-}
-
-int nvme_ana_log(int fd, void *ana_log, size_t ana_log_len, int rgo)
-{
-       __u64 lpo = 0;
-
-       return nvme_get_log13(fd, NVME_NSID_ALL, NVME_LOG_ANA, rgo, lpo, 0,
-                       true, ana_log_len, ana_log);
-}
-
-static int get_ana_state(__u32 nsid, __u32 anagrpid, void *ana_log)
-{
-       int     rc = ANA_PRIO_GETANAS_FAILED;
        void *base = ana_log;
        struct nvme_ana_rsp_hdr *hdr = base;
        struct nvme_ana_group_desc *ana_desc;
-       int offset = sizeof(struct nvme_ana_rsp_hdr);
+       size_t offset = sizeof(struct nvme_ana_rsp_hdr);
        __u32 nr_nsids;
        size_t nsid_buf_size;
        int i, j;
 
        for (i = 0; i < le16_to_cpu(hdr->ngrps); i++) {
                ana_desc = base + offset;
+
+               offset += sizeof(*ana_desc);
+               if (offset > ana_log_len)
+                       return -ANA_ERR_GETANAS_OVERFLOW;
+
                nr_nsids = le32_to_cpu(ana_desc->nnsids);
                nsid_buf_size = nr_nsids * sizeof(__le32);
 
-               offset += sizeof(*ana_desc);
+               offset += nsid_buf_size;
+               if (offset > ana_log_len)
+                       return -ANA_ERR_GETANAS_OVERFLOW;
 
                for (j = 0; j < nr_nsids; j++) {
                        if (nsid == le32_to_cpu(ana_desc->nsids[j]))
@@ -173,12 +100,10 @@ static int get_ana_state(__u32 nsid, __u32 anagrpid, void *ana_log)
                }
 
                if (anagrpid != 0 && anagrpid == le32_to_cpu(ana_desc->grpid))
-                       rc = ana_desc->state;
+                       return ana_desc->state;
 
-               offset += nsid_buf_size;
        }
-
-       return rc;
+       return -ANA_ERR_GETANAS_NOTFOUND;
 }
 
 int get_ana_info(struct path * pp, unsigned int timeout)
@@ -189,104 +114,121 @@ int get_ana_info(struct path * pp, unsigned int timeout)
        struct nvme_id_ns ns;
        void *ana_log;
        size_t ana_log_len;
+       bool is_anagrpid_const;
 
        rc = nvme_identify_ctrl(pp->fd, &ctrl);
-       if (rc)
-               return ANA_PRIO_GETCTRL_FAILED;
+       if (rc < 0) {
+               log_nvme_errcode(rc, pp->dev, "nvme_identify_ctrl");
+               return -ANA_ERR_GETCTRL_FAILED;
+       }
 
        if(!(ctrl.cmic & (1 << 3)))
-               return ANA_PRIO_NOT_SUPPORTED;
-
-       rc = nvme_get_nsid(pp->fd, &nsid);
-       if (rc)
-               return ANA_PRIO_GETNSID_FAILED;
+               return -ANA_ERR_NOT_SUPPORTED;
 
-       rc = nvme_identify_ns(pp->fd, nsid, &ns);
-       if (rc)
-               return ANA_PRIO_GETNS_FAILED;
+       nsid = nvme_get_nsid(pp->fd);
+       if (nsid <= 0) {
+               log_nvme_errcode(rc, pp->dev, "nvme_get_nsid");
+               return -ANA_ERR_GETNSID_FAILED;
+       }
+       is_anagrpid_const = ctrl.anacap & (1 << 6);
 
+       /*
+        * Code copied from nvme-cli/nvme.c. We don't need to allocate an
+        * [nanagrpid*mnan] array of NSIDs because each NSID can occur at most
+        * in one ANA group.
+        */
        ana_log_len = sizeof(struct nvme_ana_rsp_hdr) +
-               le32_to_cpu(ctrl.nanagrpid) * sizeof(struct nvme_ana_group_desc);
-       if (!(ctrl.anacap & (1 << 6)))
+               le32_to_cpu(ctrl.nanagrpid)
+               * sizeof(struct nvme_ana_group_desc);
+
+       if (is_anagrpid_const) {
+               rc = nvme_identify_ns(pp->fd, nsid, 0, &ns);
+               if (rc) {
+                       log_nvme_errcode(rc, pp->dev, "nvme_identify_ns");
+                       return -ANA_ERR_GETNS_FAILED;
+               }
+       } else
                ana_log_len += le32_to_cpu(ctrl.mnan) * sizeof(__le32);
 
        ana_log = malloc(ana_log_len);
        if (!ana_log)
-               return ANA_PRIO_NO_MEMORY;
-
+               return -ANA_ERR_NO_MEMORY;
+       pthread_cleanup_push(free, ana_log);
        rc = nvme_ana_log(pp->fd, ana_log, ana_log_len,
-               (ctrl.anacap & (1 << 6)) ? NVME_ANA_LOG_RGO : 0);
+                         is_anagrpid_const ? NVME_ANA_LOG_RGO : 0);
        if (rc) {
-               free(ana_log);
-               return ANA_PRIO_GETANALOG_FAILED;
-       }
-
-       rc = get_ana_state(nsid, le32_to_cpu(ns.anagrpid), ana_log);
-       if (rc < 0){
-               free(ana_log);
-               return ANA_PRIO_GETANAS_FAILED;
-       }
-
-       free(ana_log);
-       condlog(3, "%s: ana state = %02x [%s]", pp->dev, rc, aas_print_string(rc));
-
+               log_nvme_errcode(rc, pp->dev, "nvme_ana_log");
+               rc = -ANA_ERR_GETANALOG_FAILED;
+       } else
+               rc = get_ana_state(nsid,
+                                  is_anagrpid_const ?
+                                  le32_to_cpu(ns.anagrpid) : 0,
+                                  ana_log, ana_log_len);
+       pthread_cleanup_pop(1);
+       if (rc >= 0)
+               condlog(3, "%s: ana state = %02x [%s]", pp->dev, rc,
+                       aas_print_string(rc));
        return rc;
 }
 
-int getprio(struct path * pp, char * args, unsigned int timeout)
+/*
+ * Priorities modeled roughly after the ALUA model (alua.c/sysfs.c)
+ * Reference: ANA Base Protocol (NVMe TP 4004a, 11/13/2018).
+ *
+ * Differences:
+ *
+ * - The ANA base spec defines no implicit or explicit (STPG) state management.
+ *   If a state is encountered that doesn't allow normal I/O (all except
+ *   OPTIMIZED and NON_OPTIMIZED), we can't do anything but either wait for a
+ *   Access State Change Notice (can't do that in multipathd as we don't receive
+ *   those), or retry commands in regular time intervals until ANATT is expired
+ *   (not implemented). Mapping UNAVAILABLE state to ALUA STANDBY is the best we
+ *   can currently do.
+ *
+ *   FIXME: Waiting for ANATT could be implemented with a "delayed failback"
+ *   mechanism. The current "failback" method can't be used, as it would
+ *   affect failback to every state, and here only failback to UNAVAILABLE
+ *   should be delayed.
+ *
+ * - PERSISTENT_LOSS state is even below ALUA's UNAVAILABLE state.
+ *   FIXME: According to the ANA TP, accessing paths in PERSISTENT_LOSS state
+ *   in any way makes no sense (e.g. §8.19.6 - paths in this state shouldn't
+ *   even be checked under "all paths down" conditions). Device mapper can,
+ *   and will, select a PG for IO if it has non-failed paths, even if the
+ *   PG has priority 0. We could avoid that only with an "ANA path checker".
+ *
+ * - ALUA has no CHANGE state. The ANA TP §8.18.3 / §8.19.4 suggests
+ *   that CHANGE state should be treated in roughly the same way as
+ *   INACCESSIBLE. Therefore we assign the same prio to it.
+ *
+ * - ALUA's LBA-dependent state has no ANA equivalent.
+ */
+
+int getprio(struct path *pp, char *args, unsigned int timeout)
 {
        int rc;
 
        if (pp->fd < 0)
-               return ANA_PRIO_NO_INFORMATION;
+               rc = -ANA_ERR_NO_INFORMATION;
+       else
+               rc = get_ana_info(pp, timeout);
 
-       rc = get_ana_info(pp, timeout);
-       if (rc >= 0) {
-               rc &= 0x0f;
-               switch(rc) {
-               case NVME_ANA_OPTIMIZED:
-                       rc = ANA_PRIO_OPTIMIZED;
-                       break;
-               case NVME_ANA_NONOPTIMIZED:
-                       rc = ANA_PRIO_NONOPTIMIZED;
-                       break;
-               case NVME_ANA_INACCESSIBLE:
-                       rc = ANA_PRIO_INACCESSIBLE;
-                       break;
-               case NVME_ANA_PERSISTENT_LOSS:
-                       rc = ANA_PRIO_PERSISTENT_LOSS;
-                       break;
-               case NVME_ANA_CHANGE:
-                       rc = ANA_PRIO_CHANGE;
-                       break;
-               default:
-                       rc = ANA_PRIO_RESERVED;
-               }
-       } else {
-               switch(rc) {
-               case ANA_PRIO_GETCTRL_FAILED:
-                       condlog(0, "%s: couldn't get ctrl info", pp->dev);
-                       break;
-               case ANA_PRIO_NOT_SUPPORTED:
-                       condlog(0, "%s: ana not supported", pp->dev);
-                       break;
-               case ANA_PRIO_GETANAS_FAILED:
-                       condlog(0, "%s: couldn't get ana state", pp->dev);
-                       break;
-               case ANA_PRIO_GETANALOG_FAILED:
-                       condlog(0, "%s: couldn't get ana log", pp->dev);
-                       break;
-               case ANA_PRIO_GETNS_FAILED:
-                       condlog(0, "%s: couldn't get namespace", pp->dev);
-                       break;
-               case ANA_PRIO_GETNSID_FAILED:
-                       condlog(0, "%s: couldn't get namespace id", pp->dev);
-                       break;
-               case ANA_PRIO_NO_MEMORY:
-                       condlog(0, "%s: couldn't alloc memory", pp->dev);
-                       break;
-               }
+       switch (rc) {
+       case NVME_ANA_OPTIMIZED:
+               return 50;
+       case NVME_ANA_NONOPTIMIZED:
+               return 10;
+       case NVME_ANA_INACCESSIBLE:
+       case NVME_ANA_CHANGE:
+               return 1;
+       case NVME_ANA_PERSISTENT_LOSS:
+               return 0;
+       default:
+               break;
        }
-       return rc;
+       if (rc < 0 && -rc < ARRAY_SIZE(ana_errmsg))
+               condlog(2, "%s: ANA error: %s", pp->dev, ana_errmsg[-rc]);
+       else
+               condlog(1, "%s: invalid ANA rc code %d", pp->dev, rc);
+       return -1;
 }
-
diff --git a/libmultipath/prioritizers/ana.h b/libmultipath/prioritizers/ana.h
deleted file mode 100644 (file)
index 92cfa9e..0000000
+++ /dev/null
@@ -1,221 +0,0 @@
-#ifndef _ANA_H
-#define _ANA_H
-
-#include <linux/types.h>
-
-#define NVME_NSID_ALL                  0xffffffff
-#define NVME_IDENTIFY_DATA_SIZE        4096
-
-#define NVME_LOG_ANA                   0x0c
-
-/* Admin commands */
-enum nvme_admin_opcode {
-       nvme_admin_get_log_page         = 0x02,
-       nvme_admin_identify             = 0x06,
-};
-
-enum {
-       NVME_ID_CNS_NS                  = 0x00,
-       NVME_ID_CNS_CTRL                = 0x01,
-};
-
-/* nvme ioctl start */
-struct nvme_passthru_cmd {
-       __u8    opcode;
-       __u8    flags;
-       __u16   rsvd1;
-       __u32   nsid;
-       __u32   cdw2;
-       __u32   cdw3;
-       __u64   metadata;
-       __u64   addr;
-       __u32   metadata_len;
-       __u32   data_len;
-       __u32   cdw10;
-       __u32   cdw11;
-       __u32   cdw12;
-       __u32   cdw13;
-       __u32   cdw14;
-       __u32   cdw15;
-       __u32   timeout_ms;
-       __u32   result;
-};
-
-#define nvme_admin_cmd nvme_passthru_cmd
-
-#define NVME_IOCTL_ID          _IO('N', 0x40)
-#define NVME_IOCTL_ADMIN_CMD   _IOWR('N', 0x41, struct nvme_admin_cmd)
-/* nvme ioctl end */
-
-/* nvme id ctrl start */
-struct nvme_id_power_state {
-       __le16                  max_power;      /* centiwatts */
-       __u8                    rsvd2;
-       __u8                    flags;
-       __le32                  entry_lat;      /* microseconds */
-       __le32                  exit_lat;       /* microseconds */
-       __u8                    read_tput;
-       __u8                    read_lat;
-       __u8                    write_tput;
-       __u8                    write_lat;
-       __le16                  idle_power;
-       __u8                    idle_scale;
-       __u8                    rsvd19;
-       __le16                  active_power;
-       __u8                    active_work_scale;
-       __u8                    rsvd23[9];
-};
-
-struct nvme_id_ctrl {
-       __le16                  vid;
-       __le16                  ssvid;
-       char                    sn[20];
-       char                    mn[40];
-       char                    fr[8];
-       __u8                    rab;
-       __u8                    ieee[3];
-       __u8                    cmic;
-       __u8                    mdts;
-       __le16                  cntlid;
-       __le32                  ver;
-       __le32                  rtd3r;
-       __le32                  rtd3e;
-       __le32                  oaes;
-       __le32                  ctratt;
-       __u8                    rsvd100[156];
-       __le16                  oacs;
-       __u8                    acl;
-       __u8                    aerl;
-       __u8                    frmw;
-       __u8                    lpa;
-       __u8                    elpe;
-       __u8                    npss;
-       __u8                    avscc;
-       __u8                    apsta;
-       __le16                  wctemp;
-       __le16                  cctemp;
-       __le16                  mtfa;
-       __le32                  hmpre;
-       __le32                  hmmin;
-       __u8                    tnvmcap[16];
-       __u8                    unvmcap[16];
-       __le32                  rpmbs;
-       __le16                  edstt;
-       __u8                    dsto;
-       __u8                    fwug;
-       __le16                  kas;
-       __le16                  hctma;
-       __le16                  mntmt;
-       __le16                  mxtmt;
-       __le32                  sanicap;
-       __le32                  hmminds;
-       __le16                  hmmaxd;
-       __u8                    rsvd338[4];
-       __u8                    anatt;
-       __u8                    anacap;
-       __le32                  anagrpmax;
-       __le32                  nanagrpid;
-       __u8                    rsvd352[160];
-       __u8                    sqes;
-       __u8                    cqes;
-       __le16                  maxcmd;
-       __le32                  nn;
-       __le16                  oncs;
-       __le16                  fuses;
-       __u8                    fna;
-       __u8                    vwc;
-       __le16                  awun;
-       __le16                  awupf;
-       __u8                    nvscc;
-       __u8                    nwpc;
-       __le16                  acwu;
-       __u8                    rsvd534[2];
-       __le32                  sgls;
-       __le32                  mnan;
-       __u8                    rsvd544[224];
-       char                    subnqn[256];
-       __u8                    rsvd1024[768];
-       __le32                  ioccsz;
-       __le32                  iorcsz;
-       __le16                  icdoff;
-       __u8                    ctrattr;
-       __u8                    msdbd;
-       __u8                    rsvd1804[244];
-       struct nvme_id_power_state      psd[32];
-       __u8                    vs[1024];
-};
-/* nvme id ctrl end */
-
-/* nvme id ns start */
-struct nvme_lbaf {
-       __le16                  ms;
-       __u8                    ds;
-       __u8                    rp;
-};
-
-struct nvme_id_ns {
-       __le64                  nsze;
-       __le64                  ncap;
-       __le64                  nuse;
-       __u8                    nsfeat;
-       __u8                    nlbaf;
-       __u8                    flbas;
-       __u8                    mc;
-       __u8                    dpc;
-       __u8                    dps;
-       __u8                    nmic;
-       __u8                    rescap;
-       __u8                    fpi;
-       __u8                    rsvd33;
-       __le16                  nawun;
-       __le16                  nawupf;
-       __le16                  nacwu;
-       __le16                  nabsn;
-       __le16                  nabo;
-       __le16                  nabspf;
-       __le16                  noiob;
-       __u8                    nvmcap[16];
-       __u8                    rsvd64[28];
-       __le32                  anagrpid;
-       __u8                    rsvd96[3];
-       __u8                    nsattr;
-       __u8                    rsvd100[4];
-       __u8                    nguid[16];
-       __u8                    eui64[8];
-       struct nvme_lbaf        lbaf[16];
-       __u8                    rsvd192[192];
-       __u8                    vs[3712];
-};
-/* nvme id ns end */
-
-/* nvme ana start */
-enum nvme_ana_state {
-       NVME_ANA_OPTIMIZED              = 0x01,
-       NVME_ANA_NONOPTIMIZED           = 0x02,
-       NVME_ANA_INACCESSIBLE           = 0x03,
-       NVME_ANA_PERSISTENT_LOSS        = 0x04,
-       NVME_ANA_CHANGE                 = 0x0f,
-       NVME_ANA_RESERVED               = 0x05,
-};
-
-struct nvme_ana_rsp_hdr {
-       __le64  chgcnt;
-       __le16  ngrps;
-       __le16  rsvd10[3];
-};
-
-struct nvme_ana_group_desc {
-       __le32  grpid;
-       __le32  nnsids;
-       __le64  chgcnt;
-       __u8    state;
-       __u8    rsvd17[15];
-       __le32  nsids[];
-};
-
-/* flag for the log specific field of the ANA log */
-#define NVME_ANA_LOG_RGO       (1 << 0)
-
-/* nvme ana end */
-
-#endif