drivers/infiniband/hw/cxgb3/cxio_wr.h - LeafOS-Devices/android_kernel_realme_mt6785 - Gitiles

 /*
  * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
 #ifndef __CXIO_WR_H__
 #define __CXIO_WR_H__

 #include <asm/io.h>
 #include <linux/pci.h>
 #include <linux/timer.h>
 #include "firmware_exports.h"

 #define T3_MAX_SGE      4
 #define T3_MAX_INLINE	64

 #define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
 #define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))>>(size_log2)) && \
 				       ((rptr)!=(wptr)) )
 #define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
 #define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
 #define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
 #define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))

 static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)
 {
 	writel(((1<<31) | qpid), doorbell);
 }

 #define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))

 enum t3_wr_flags {
 	T3_COMPLETION_FLAG = 0x01,
 	T3_NOTIFY_FLAG = 0x02,
 	T3_SOLICITED_EVENT_FLAG = 0x04,
 	T3_READ_FENCE_FLAG = 0x08,
 	T3_LOCAL_FENCE_FLAG = 0x10
 } __attribute__ ((packed));

 enum t3_wr_opcode {
 	T3_WR_BP = FW_WROPCODE_RI_BYPASS,
 	T3_WR_SEND = FW_WROPCODE_RI_SEND,
 	T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
 	T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
 	T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
 	T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
 	T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
 	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
 	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
 } __attribute__ ((packed));

 enum t3_rdma_opcode {
 	T3_RDMA_WRITE,		/* IETF RDMAP v1.0 ... */
 	T3_READ_REQ,
 	T3_READ_RESP,
 	T3_SEND,
 	T3_SEND_WITH_INV,
 	T3_SEND_WITH_SE,
 	T3_SEND_WITH_SE_INV,
 	T3_TERMINATE,
 	T3_RDMA_INIT,		/* CHELSIO RI specific ... */
 	T3_BIND_MW,
 	T3_FAST_REGISTER,
 	T3_LOCAL_INV,
 	T3_QP_MOD,
 	T3_BYPASS
 } __attribute__ ((packed));

 static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
 {
 	switch (wrop) {
 		case T3_WR_BP: return T3_BYPASS;
 		case T3_WR_SEND: return T3_SEND;
 		case T3_WR_WRITE: return T3_RDMA_WRITE;
 		case T3_WR_READ: return T3_READ_REQ;
 		case T3_WR_INV_STAG: return T3_LOCAL_INV;
 		case T3_WR_BIND: return T3_BIND_MW;
 		case T3_WR_INIT: return T3_RDMA_INIT;
 		case T3_WR_QP_MOD: return T3_QP_MOD;
 		default: break;
 	}
 	return -1;
 }


 /* Work request id */
 union t3_wrid {
 	struct {
 		u32 hi;
 		u32 low;
 	} id0;
 	u64 id1;
 };

 #define WRID(wrid)		(wrid.id1)
 #define WRID_GEN(wrid)		(wrid.id0.wr_gen)
 #define WRID_IDX(wrid)		(wrid.id0.wr_idx)
 #define WRID_LO(wrid)		(wrid.id0.wr_lo)

 struct fw_riwrh {
 	__be32 op_seop_flags;
 	__be32 gen_tid_len;
 };

 #define S_FW_RIWR_OP		24
 #define M_FW_RIWR_OP		0xff
 #define V_FW_RIWR_OP(x)		((x) << S_FW_RIWR_OP)
 #define G_FW_RIWR_OP(x)	((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)

 #define S_FW_RIWR_SOPEOP	22
 #define M_FW_RIWR_SOPEOP	0x3
 #define V_FW_RIWR_SOPEOP(x)	((x) << S_FW_RIWR_SOPEOP)

 #define S_FW_RIWR_FLAGS		8
 #define M_FW_RIWR_FLAGS		0x3fffff
 #define V_FW_RIWR_FLAGS(x)	((x) << S_FW_RIWR_FLAGS)
 #define G_FW_RIWR_FLAGS(x)	((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)

 #define S_FW_RIWR_TID		8
 #define V_FW_RIWR_TID(x)	((x) << S_FW_RIWR_TID)

 #define S_FW_RIWR_LEN		0
 #define V_FW_RIWR_LEN(x)	((x) << S_FW_RIWR_LEN)

 #define S_FW_RIWR_GEN           31
 #define V_FW_RIWR_GEN(x)        ((x)  << S_FW_RIWR_GEN)

 struct t3_sge {
 	__be32 stag;
 	__be32 len;
 	__be64 to;
 };

 /* If num_sgle is zero, flit 5+ contains immediate data.*/
 struct t3_send_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */

 	u8 rdmaop;		/* 2 */
 	u8 reserved[3];
 	__be32 rem_stag;
 	__be32 plen;		/* 3 */
 	__be32 num_sgle;
 	struct t3_sge sgl[T3_MAX_SGE];	/* 4+ */
 };

 struct t3_local_inv_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	__be32 stag;		/* 2 */
 	__be32 reserved3;
 };

 struct t3_rdma_write_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	u8 rdmaop;		/* 2 */
 	u8 reserved[3];
 	__be32 stag_sink;
 	__be64 to_sink;		/* 3 */
 	__be32 plen;		/* 4 */
 	__be32 num_sgle;
 	struct t3_sge sgl[T3_MAX_SGE];	/* 5+ */
 };

 struct t3_rdma_read_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	u8 rdmaop;		/* 2 */
 	u8 reserved[3];
 	__be32 rem_stag;
 	__be64 rem_to;		/* 3 */
 	__be32 local_stag;	/* 4 */
 	__be32 local_len;
 	__be64 local_to;	/* 5 */
 };

 enum t3_addr_type {
 	T3_VA_BASED_TO = 0x0,
 	T3_ZERO_BASED_TO = 0x1
 } __attribute__ ((packed));

 enum t3_mem_perms {
 	T3_MEM_ACCESS_LOCAL_READ = 0x1,
 	T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
 	T3_MEM_ACCESS_REM_READ = 0x4,
 	T3_MEM_ACCESS_REM_WRITE = 0x8
 } __attribute__ ((packed));

 struct t3_bind_mw_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	u16 reserved;		/* 2 */
 	u8 type;
 	u8 perms;
 	__be32 mr_stag;
 	__be32 mw_stag;		/* 3 */
 	__be32 mw_len;
 	__be64 mw_va;		/* 4 */
 	__be32 mr_pbl_addr;	/* 5 */
 	u8 reserved2[3];
 	u8 mr_pagesz;
 };

 struct t3_receive_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	u8 pagesz[T3_MAX_SGE];
 	__be32 num_sgle;		/* 2 */
 	struct t3_sge sgl[T3_MAX_SGE];	/* 3+ */
 	__be32 pbl_addr[T3_MAX_SGE];
 };

 struct t3_bypass_wr {
 	struct fw_riwrh wrh;
 	union t3_wrid wrid;	/* 1 */
 };

 struct t3_modify_qp_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	__be32 flags;		/* 2 */
 	__be32 quiesce;		/* 2 */
 	__be32 max_ird;		/* 3 */
 	__be32 max_ord;		/* 3 */
 	__be64 sge_cmd;		/* 4 */
 	__be64 ctx1;		/* 5 */
 	__be64 ctx0;		/* 6 */
 };

 enum t3_modify_qp_flags {
 	MODQP_QUIESCE  = 0x01,
 	MODQP_MAX_IRD  = 0x02,
 	MODQP_MAX_ORD  = 0x04,
 	MODQP_WRITE_EC = 0x08,
 	MODQP_READ_EC  = 0x10,
 };


 enum t3_mpa_attrs {
 	uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
 	uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
 	uP_RI_MPA_CRC_ENABLE = 0x4,
 	uP_RI_MPA_IETF_ENABLE = 0x8
 } __attribute__ ((packed));

 enum t3_qp_caps {
 	uP_RI_QP_RDMA_READ_ENABLE = 0x01,
 	uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
 	uP_RI_QP_BIND_ENABLE = 0x04,
 	uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
 	uP_RI_QP_STAG0_ENABLE = 0x10
 } __attribute__ ((packed));

 struct t3_rdma_init_attr {
 	u32 tid;
 	u32 qpid;
 	u32 pdid;
 	u32 scqid;
 	u32 rcqid;
 	u32 rq_addr;
 	u32 rq_size;
 	enum t3_mpa_attrs mpaattrs;
 	enum t3_qp_caps qpcaps;
 	u16 tcp_emss;
 	u32 ord;
 	u32 ird;
 	u64 qp_dma_addr;
 	u32 qp_dma_size;
 	u32 flags;
 	u32 irs;
 };

 struct t3_rdma_init_wr {
 	struct fw_riwrh wrh;	/* 0 */
 	union t3_wrid wrid;	/* 1 */
 	__be32 qpid;		/* 2 */
 	__be32 pdid;
 	__be32 scqid;		/* 3 */
 	__be32 rcqid;
 	__be32 rq_addr;		/* 4 */
 	__be32 rq_size;
 	u8 mpaattrs;		/* 5 */
 	u8 qpcaps;
 	__be16 ulpdu_size;
 	__be32 flags;		/* bits 31-1 - reservered */
 				/* bit     0 - set if RECV posted */
 	__be32 ord;		/* 6 */
 	__be32 ird;
 	__be64 qp_dma_addr;	/* 7 */
 	__be32 qp_dma_size;	/* 8 */
 	__be32 irs;
 };

 struct t3_genbit {
 	u64 flit[15];
 	__be64 genbit;
 };

 enum rdma_init_wr_flags {
 	RECVS_POSTED = (1<<0),
 	PRIV_QP = (1<<1),
 };

 union t3_wr {
 	struct t3_send_wr send;
 	struct t3_rdma_write_wr write;
 	struct t3_rdma_read_wr read;
 	struct t3_receive_wr recv;
 	struct t3_local_inv_wr local_inv;
 	struct t3_bind_mw_wr bind;
 	struct t3_bypass_wr bypass;
 	struct t3_rdma_init_wr init;
 	struct t3_modify_qp_wr qp_mod;
 	struct t3_genbit genbit;
 	u64 flit[16];
 };

 #define T3_SQ_CQE_FLIT	  13
 #define T3_SQ_COOKIE_FLIT 14

 #define T3_RQ_COOKIE_FLIT 13
 #define T3_RQ_CQE_FLIT	  14

 static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
 {
 	return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
 }

 static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
 				  enum t3_wr_flags flags, u8 genbit, u32 tid,
 				  u8 len)
 {
 	wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
 					 V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
 					 V_FW_RIWR_FLAGS(flags));
 	wmb();
 	wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
 				       V_FW_RIWR_TID(tid) |
 				       V_FW_RIWR_LEN(len));
 	/* 2nd gen bit... */
 	((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit);
 }

 /*
  * T3 ULP2_TX commands
  */
 enum t3_utx_mem_op {
 	T3_UTX_MEM_READ = 2,
 	T3_UTX_MEM_WRITE = 3
 };

 /* T3 MC7 RDMA TPT entry format */

 enum tpt_mem_type {
 	TPT_NON_SHARED_MR = 0x0,
 	TPT_SHARED_MR = 0x1,
 	TPT_MW = 0x2,
 	TPT_MW_RELAXED_PROTECTION = 0x3
 };

 enum tpt_addr_type {
 	TPT_ZBTO = 0,
 	TPT_VATO = 1
 };

 enum tpt_mem_perm {
 	TPT_LOCAL_READ = 0x8,
 	TPT_LOCAL_WRITE = 0x4,
 	TPT_REMOTE_READ = 0x2,
 	TPT_REMOTE_WRITE = 0x1
 };

 struct tpt_entry {
 	__be32 valid_stag_pdid;
 	__be32 flags_pagesize_qpid;

 	__be32 rsvd_pbl_addr;
 	__be32 len;
 	__be32 va_hi;
 	__be32 va_low_or_fbo;

 	__be32 rsvd_bind_cnt_or_pstag;
 	__be32 rsvd_pbl_size;
 };

 #define S_TPT_VALID		31
 #define V_TPT_VALID(x)		((x) << S_TPT_VALID)
 #define F_TPT_VALID		V_TPT_VALID(1U)

 #define S_TPT_STAG_KEY		23
 #define M_TPT_STAG_KEY		0xFF
 #define V_TPT_STAG_KEY(x)	((x) << S_TPT_STAG_KEY)
 #define G_TPT_STAG_KEY(x)	(((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)

 #define S_TPT_STAG_STATE	22
 #define V_TPT_STAG_STATE(x)	((x) << S_TPT_STAG_STATE)
 #define F_TPT_STAG_STATE	V_TPT_STAG_STATE(1U)

 #define S_TPT_STAG_TYPE		20
 #define M_TPT_STAG_TYPE		0x3
 #define V_TPT_STAG_TYPE(x)	((x) << S_TPT_STAG_TYPE)
 #define G_TPT_STAG_TYPE(x)	(((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)

 #define S_TPT_PDID		0
 #define M_TPT_PDID		0xFFFFF
 #define V_TPT_PDID(x)		((x) << S_TPT_PDID)
 #define G_TPT_PDID(x)		(((x) >> S_TPT_PDID) & M_TPT_PDID)

 #define S_TPT_PERM		28
 #define M_TPT_PERM		0xF
 #define V_TPT_PERM(x)		((x) << S_TPT_PERM)
 #define G_TPT_PERM(x)		(((x) >> S_TPT_PERM) & M_TPT_PERM)

 #define S_TPT_REM_INV_DIS	27
 #define V_TPT_REM_INV_DIS(x)	((x) << S_TPT_REM_INV_DIS)
 #define F_TPT_REM_INV_DIS	V_TPT_REM_INV_DIS(1U)

 #define S_TPT_ADDR_TYPE		26
 #define V_TPT_ADDR_TYPE(x)	((x) << S_TPT_ADDR_TYPE)
 #define F_TPT_ADDR_TYPE		V_TPT_ADDR_TYPE(1U)

 #define S_TPT_MW_BIND_ENABLE	25
 #define V_TPT_MW_BIND_ENABLE(x)	((x) << S_TPT_MW_BIND_ENABLE)
 #define F_TPT_MW_BIND_ENABLE    V_TPT_MW_BIND_ENABLE(1U)

 #define S_TPT_PAGE_SIZE		20
 #define M_TPT_PAGE_SIZE		0x1F
 #define V_TPT_PAGE_SIZE(x)	((x) << S_TPT_PAGE_SIZE)
 #define G_TPT_PAGE_SIZE(x)	(((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)

 #define S_TPT_PBL_ADDR		0
 #define M_TPT_PBL_ADDR		0x1FFFFFFF
 #define V_TPT_PBL_ADDR(x)	((x) << S_TPT_PBL_ADDR)
 #define G_TPT_PBL_ADDR(x)       (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)

 #define S_TPT_QPID		0
 #define M_TPT_QPID		0xFFFFF
 #define V_TPT_QPID(x)		((x) << S_TPT_QPID)
 #define G_TPT_QPID(x)		(((x) >> S_TPT_QPID) & M_TPT_QPID)

 #define S_TPT_PSTAG		0
 #define M_TPT_PSTAG		0xFFFFFF
 #define V_TPT_PSTAG(x)		((x) << S_TPT_PSTAG)
 #define G_TPT_PSTAG(x)		(((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)

 #define S_TPT_PBL_SIZE		0
 #define M_TPT_PBL_SIZE		0xFFFFF
 #define V_TPT_PBL_SIZE(x)	((x) << S_TPT_PBL_SIZE)
 #define G_TPT_PBL_SIZE(x)	(((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)

 /*
  * CQE defs
  */
 struct t3_cqe {
 	__be32 header;
 	__be32 len;
 	union {
 		struct {
 			__be32 stag;
 			__be32 msn;
 		} rcqe;
 		struct {
 			u32 wrid_hi;
 			u32 wrid_low;
 		} scqe;
 	} u;
 };

 #define S_CQE_OOO	  31
 #define M_CQE_OOO	  0x1
 #define G_CQE_OOO(x)	  ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
 #define V_CEQ_OOO(x)	  ((x)<<S_CQE_OOO)

 #define S_CQE_QPID        12
 #define M_CQE_QPID        0x7FFFF
 #define G_CQE_QPID(x)     ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
 #define V_CQE_QPID(x)	  ((x)<<S_CQE_QPID)

 #define S_CQE_SWCQE       11
 #define M_CQE_SWCQE       0x1
 #define G_CQE_SWCQE(x)    ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
 #define V_CQE_SWCQE(x)	  ((x)<<S_CQE_SWCQE)

 #define S_CQE_GENBIT      10
 #define M_CQE_GENBIT      0x1
 #define G_CQE_GENBIT(x)   (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
 #define V_CQE_GENBIT(x)	  ((x)<<S_CQE_GENBIT)

 #define S_CQE_STATUS      5
 #define M_CQE_STATUS      0x1F
 #define G_CQE_STATUS(x)   ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
 #define V_CQE_STATUS(x)   ((x)<<S_CQE_STATUS)

 #define S_CQE_TYPE        4
 #define M_CQE_TYPE        0x1
 #define G_CQE_TYPE(x)     ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
 #define V_CQE_TYPE(x)     ((x)<<S_CQE_TYPE)

 #define S_CQE_OPCODE      0
 #define M_CQE_OPCODE      0xF
 #define G_CQE_OPCODE(x)   ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
 #define V_CQE_OPCODE(x)   ((x)<<S_CQE_OPCODE)

 #define SW_CQE(x)         (G_CQE_SWCQE(be32_to_cpu((x).header)))
 #define CQE_OOO(x)        (G_CQE_OOO(be32_to_cpu((x).header)))
 #define CQE_QPID(x)       (G_CQE_QPID(be32_to_cpu((x).header)))
 #define CQE_GENBIT(x)     (G_CQE_GENBIT(be32_to_cpu((x).header)))
 #define CQE_TYPE(x)       (G_CQE_TYPE(be32_to_cpu((x).header)))
 #define SQ_TYPE(x)	  (CQE_TYPE((x)))
 #define RQ_TYPE(x)	  (!CQE_TYPE((x)))
 #define CQE_STATUS(x)     (G_CQE_STATUS(be32_to_cpu((x).header)))
 #define CQE_OPCODE(x)     (G_CQE_OPCODE(be32_to_cpu((x).header)))

 #define CQE_LEN(x)        (be32_to_cpu((x).len))

 /* used for RQ completion processing */
 #define CQE_WRID_STAG(x)  (be32_to_cpu((x).u.rcqe.stag))
 #define CQE_WRID_MSN(x)   (be32_to_cpu((x).u.rcqe.msn))

 /* used for SQ completion processing */
 #define CQE_WRID_SQ_WPTR(x)	((x).u.scqe.wrid_hi)
 #define CQE_WRID_WPTR(x)	((x).u.scqe.wrid_low)

 /* generic accessor macros */
 #define CQE_WRID_HI(x)		((x).u.scqe.wrid_hi)
 #define CQE_WRID_LOW(x)		((x).u.scqe.wrid_low)

 #define TPT_ERR_SUCCESS                     0x0
 #define TPT_ERR_STAG                        0x1	 /* STAG invalid: either the */
 						 /* STAG is offlimt, being 0, */
 						 /* or STAG_key mismatch */
 #define TPT_ERR_PDID                        0x2	 /* PDID mismatch */
 #define TPT_ERR_QPID                        0x3	 /* QPID mismatch */
 #define TPT_ERR_ACCESS                      0x4	 /* Invalid access right */
 #define TPT_ERR_WRAP                        0x5	 /* Wrap error */
 #define TPT_ERR_BOUND                       0x6	 /* base and bounds voilation */
 #define TPT_ERR_INVALIDATE_SHARED_MR        0x7	 /* attempt to invalidate a  */
 						 /* shared memory region */
 #define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8	 /* attempt to invalidate a  */
 						 /* shared memory region */
 #define TPT_ERR_ECC                         0x9	 /* ECC error detected */
 #define TPT_ERR_ECC_PSTAG                   0xA	 /* ECC error detected when  */
 						 /* reading PSTAG for a MW  */
 						 /* Invalidate */
 #define TPT_ERR_PBL_ADDR_BOUND              0xB	 /* pbl addr out of bounds:  */
 						 /* software error */
 #define TPT_ERR_SWFLUSH			    0xC	 /* SW FLUSHED */
 #define TPT_ERR_CRC                         0x10 /* CRC error */
 #define TPT_ERR_MARKER                      0x11 /* Marker error */
 #define TPT_ERR_PDU_LEN_ERR                 0x12 /* invalid PDU length */
 #define TPT_ERR_OUT_OF_RQE                  0x13 /* out of RQE */
 #define TPT_ERR_DDP_VERSION                 0x14 /* wrong DDP version */
 #define TPT_ERR_RDMA_VERSION                0x15 /* wrong RDMA version */
 #define TPT_ERR_OPCODE                      0x16 /* invalid rdma opcode */
 #define TPT_ERR_DDP_QUEUE_NUM               0x17 /* invalid ddp queue number */
 #define TPT_ERR_MSN                         0x18 /* MSN error */
 #define TPT_ERR_TBIT                        0x19 /* tag bit not set correctly */
 #define TPT_ERR_MO                          0x1A /* MO not 0 for TERMINATE  */
 						 /* or READ_REQ */
 #define TPT_ERR_MSN_GAP                     0x1B
 #define TPT_ERR_MSN_RANGE                   0x1C
 #define TPT_ERR_IRD_OVERFLOW                0x1D
 #define TPT_ERR_RQE_ADDR_BOUND              0x1E /* RQE addr out of bounds:  */
 						 /* software error */
 #define TPT_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */
 						 /* mismatch) */

 struct t3_swsq {
 	__u64			wr_id;
 	struct t3_cqe		cqe;
 	__u32			sq_wptr;
 	__be32			read_len;
 	int			opcode;
 	int			complete;
 	int			signaled;
 };

 /*
  * A T3 WQ implements both the SQ and RQ.
  */
 struct t3_wq {
 	union t3_wr *queue;		/* DMA accessable memory */
 	dma_addr_t dma_addr;		/* DMA address for HW */
 	DECLARE_PCI_UNMAP_ADDR(mapping)	/* unmap kruft */
 	u32 error;			/* 1 once we go to ERROR */
 	u32 qpid;
 	u32 wptr;			/* idx to next available WR slot */
 	u32 size_log2;			/* total wq size */
 	struct t3_swsq *sq;		/* SW SQ */
 	struct t3_swsq *oldest_read;	/* tracks oldest pending read */
 	u32 sq_wptr;			/* sq_wptr - sq_rptr == count of */
 	u32 sq_rptr;			/* pending wrs */
 	u32 sq_size_log2;		/* sq size */
 	u64 *rq;			/* SW RQ (holds consumer wr_ids */
 	u32 rq_wptr;			/* rq_wptr - rq_rptr == count of */
 	u32 rq_rptr;			/* pending wrs */
 	u64 *rq_oldest_wr;		/* oldest wr on the SW RQ */
 	u32 rq_size_log2;		/* rq size */
 	u32 rq_addr;			/* rq adapter address */
 	void __iomem *doorbell;		/* kernel db */
 	u64 udb;			/* user db if any */
 };

 struct t3_cq {
 	u32 cqid;
 	u32 rptr;
 	u32 wptr;
 	u32 size_log2;
 	dma_addr_t dma_addr;
 	DECLARE_PCI_UNMAP_ADDR(mapping)
 	struct t3_cqe *queue;
 	struct t3_cqe *sw_queue;
 	u32 sw_rptr;
 	u32 sw_wptr;
 };

 #define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
 					 CQE_GENBIT(*cqe))

 static inline void cxio_set_wq_in_error(struct t3_wq *wq)
 {
 	wq->queue->flit[13] = 1;
 }

 static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
 {
 	struct t3_cqe *cqe;

 	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
 	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
 		return cqe;
 	return NULL;
 }

 static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
 {
 	struct t3_cqe *cqe;

 	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
 		cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
 		return cqe;
 	}
 	return NULL;
 }

 static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
 {
 	struct t3_cqe *cqe;

 	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
 		cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
 		return cqe;
 	}
 	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
 	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
 		return cqe;
 	return NULL;
 }

 #endif
	/*
	* Copyright (c) 2006 Chelsio, Inc. All rights reserved.
	*
	* This software is available to you under a choice of one of two
	* licenses. You may choose to be licensed under the terms of the GNU
	* General Public License (GPL) Version 2, available from the file
	* COPYING in the main directory of this source tree, or the
	* OpenIB.org BSD license below:
	*
	* Redistribution and use in source and binary forms, with or
	* without modification, are permitted provided that the following
	* conditions are met:
	*
	* - Redistributions of source code must retain the above
	* copyright notice, this list of conditions and the following
	* disclaimer.
	*
	* - Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following
	* disclaimer in the documentation and/or other materials
	* provided with the distribution.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*/
	#ifndef __CXIO_WR_H__
	#define __CXIO_WR_H__

	#include <asm/io.h>
	#include <linux/pci.h>
	#include <linux/timer.h>
	#include "firmware_exports.h"

	#define T3_MAX_SGE 4
	#define T3_MAX_INLINE 64

	#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
	#define Q_FULL(rptr,wptr,size_log2) ( (((wptr)-(rptr))>>(size_log2)) && \
	((rptr)!=(wptr)) )
	#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
	#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
	#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
	#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))

	static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)
	{
	writel(((1<<31) \| qpid), doorbell);
	}

	#define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))

	enum t3_wr_flags {
	T3_COMPLETION_FLAG = 0x01,
	T3_NOTIFY_FLAG = 0x02,
	T3_SOLICITED_EVENT_FLAG = 0x04,
	T3_READ_FENCE_FLAG = 0x08,
	T3_LOCAL_FENCE_FLAG = 0x10
	} __attribute__ ((packed));

	enum t3_wr_opcode {
	T3_WR_BP = FW_WROPCODE_RI_BYPASS,
	T3_WR_SEND = FW_WROPCODE_RI_SEND,
	T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
	T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
	T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
	T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
	T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
	} __attribute__ ((packed));

	enum t3_rdma_opcode {
	T3_RDMA_WRITE, /* IETF RDMAP v1.0 ... */
	T3_READ_REQ,
	T3_READ_RESP,
	T3_SEND,
	T3_SEND_WITH_INV,
	T3_SEND_WITH_SE,
	T3_SEND_WITH_SE_INV,
	T3_TERMINATE,
	T3_RDMA_INIT, /* CHELSIO RI specific ... */
	T3_BIND_MW,
	T3_FAST_REGISTER,
	T3_LOCAL_INV,
	T3_QP_MOD,
	T3_BYPASS
	} __attribute__ ((packed));

	static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
	{
	switch (wrop) {
	case T3_WR_BP: return T3_BYPASS;
	case T3_WR_SEND: return T3_SEND;
	case T3_WR_WRITE: return T3_RDMA_WRITE;
	case T3_WR_READ: return T3_READ_REQ;
	case T3_WR_INV_STAG: return T3_LOCAL_INV;
	case T3_WR_BIND: return T3_BIND_MW;
	case T3_WR_INIT: return T3_RDMA_INIT;
	case T3_WR_QP_MOD: return T3_QP_MOD;
	default: break;
	}
	return -1;
	}


	/* Work request id */
	union t3_wrid {
	struct {
	u32 hi;
	u32 low;
	} id0;
	u64 id1;
	};

	#define WRID(wrid) (wrid.id1)
	#define WRID_GEN(wrid) (wrid.id0.wr_gen)
	#define WRID_IDX(wrid) (wrid.id0.wr_idx)
	#define WRID_LO(wrid) (wrid.id0.wr_lo)

	struct fw_riwrh {
	__be32 op_seop_flags;
	__be32 gen_tid_len;
	};

	#define S_FW_RIWR_OP 24
	#define M_FW_RIWR_OP 0xff
	#define V_FW_RIWR_OP(x) ((x) << S_FW_RIWR_OP)
	#define G_FW_RIWR_OP(x) ((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)

	#define S_FW_RIWR_SOPEOP 22
	#define M_FW_RIWR_SOPEOP 0x3
	#define V_FW_RIWR_SOPEOP(x) ((x) << S_FW_RIWR_SOPEOP)

	#define S_FW_RIWR_FLAGS 8
	#define M_FW_RIWR_FLAGS 0x3fffff
	#define V_FW_RIWR_FLAGS(x) ((x) << S_FW_RIWR_FLAGS)
	#define G_FW_RIWR_FLAGS(x) ((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)

	#define S_FW_RIWR_TID 8
	#define V_FW_RIWR_TID(x) ((x) << S_FW_RIWR_TID)

	#define S_FW_RIWR_LEN 0
	#define V_FW_RIWR_LEN(x) ((x) << S_FW_RIWR_LEN)

	#define S_FW_RIWR_GEN 31
	#define V_FW_RIWR_GEN(x) ((x) << S_FW_RIWR_GEN)

	struct t3_sge {
	__be32 stag;
	__be32 len;
	__be64 to;
	};

	/* If num_sgle is zero, flit 5+ contains immediate data.*/
	struct t3_send_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */

	u8 rdmaop; /* 2 */
	u8 reserved[3];
	__be32 rem_stag;
	__be32 plen; /* 3 */
	__be32 num_sgle;
	struct t3_sge sgl[T3_MAX_SGE]; /* 4+ */
	};

	struct t3_local_inv_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	__be32 stag; /* 2 */
	__be32 reserved3;
	};

	struct t3_rdma_write_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	u8 rdmaop; /* 2 */
	u8 reserved[3];
	__be32 stag_sink;
	__be64 to_sink; /* 3 */
	__be32 plen; /* 4 */
	__be32 num_sgle;
	struct t3_sge sgl[T3_MAX_SGE]; /* 5+ */
	};

	struct t3_rdma_read_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	u8 rdmaop; /* 2 */
	u8 reserved[3];
	__be32 rem_stag;
	__be64 rem_to; /* 3 */
	__be32 local_stag; /* 4 */
	__be32 local_len;
	__be64 local_to; /* 5 */
	};

	enum t3_addr_type {
	T3_VA_BASED_TO = 0x0,
	T3_ZERO_BASED_TO = 0x1
	} __attribute__ ((packed));

	enum t3_mem_perms {
	T3_MEM_ACCESS_LOCAL_READ = 0x1,
	T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
	T3_MEM_ACCESS_REM_READ = 0x4,
	T3_MEM_ACCESS_REM_WRITE = 0x8
	} __attribute__ ((packed));

	struct t3_bind_mw_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	u16 reserved; /* 2 */
	u8 type;
	u8 perms;
	__be32 mr_stag;
	__be32 mw_stag; /* 3 */
	__be32 mw_len;
	__be64 mw_va; /* 4 */
	__be32 mr_pbl_addr; /* 5 */
	u8 reserved2[3];
	u8 mr_pagesz;
	};

	struct t3_receive_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	u8 pagesz[T3_MAX_SGE];
	__be32 num_sgle; /* 2 */
	struct t3_sge sgl[T3_MAX_SGE]; /* 3+ */
	__be32 pbl_addr[T3_MAX_SGE];
	};

	struct t3_bypass_wr {
	struct fw_riwrh wrh;
	union t3_wrid wrid; /* 1 */
	};

	struct t3_modify_qp_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	__be32 flags; /* 2 */
	__be32 quiesce; /* 2 */
	__be32 max_ird; /* 3 */
	__be32 max_ord; /* 3 */
	__be64 sge_cmd; /* 4 */
	__be64 ctx1; /* 5 */
	__be64 ctx0; /* 6 */
	};

	enum t3_modify_qp_flags {
	MODQP_QUIESCE = 0x01,
	MODQP_MAX_IRD = 0x02,
	MODQP_MAX_ORD = 0x04,
	MODQP_WRITE_EC = 0x08,
	MODQP_READ_EC = 0x10,
	};


	enum t3_mpa_attrs {
	uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
	uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
	uP_RI_MPA_CRC_ENABLE = 0x4,
	uP_RI_MPA_IETF_ENABLE = 0x8
	} __attribute__ ((packed));

	enum t3_qp_caps {
	uP_RI_QP_RDMA_READ_ENABLE = 0x01,
	uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
	uP_RI_QP_BIND_ENABLE = 0x04,
	uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
	uP_RI_QP_STAG0_ENABLE = 0x10
	} __attribute__ ((packed));

	struct t3_rdma_init_attr {
	u32 tid;
	u32 qpid;
	u32 pdid;
	u32 scqid;
	u32 rcqid;
	u32 rq_addr;
	u32 rq_size;
	enum t3_mpa_attrs mpaattrs;
	enum t3_qp_caps qpcaps;
	u16 tcp_emss;
	u32 ord;
	u32 ird;
	u64 qp_dma_addr;
	u32 qp_dma_size;
	u32 flags;
	u32 irs;
	};

	struct t3_rdma_init_wr {
	struct fw_riwrh wrh; /* 0 */
	union t3_wrid wrid; /* 1 */
	__be32 qpid; /* 2 */
	__be32 pdid;
	__be32 scqid; /* 3 */
	__be32 rcqid;
	__be32 rq_addr; /* 4 */
	__be32 rq_size;
	u8 mpaattrs; /* 5 */
	u8 qpcaps;
	__be16 ulpdu_size;
	__be32 flags; /* bits 31-1 - reservered */
	/* bit 0 - set if RECV posted */
	__be32 ord; /* 6 */
	__be32 ird;
	__be64 qp_dma_addr; /* 7 */
	__be32 qp_dma_size; /* 8 */
	__be32 irs;
	};

	struct t3_genbit {
	u64 flit[15];
	__be64 genbit;
	};

	enum rdma_init_wr_flags {
	RECVS_POSTED = (1<<0),
	PRIV_QP = (1<<1),
	};

	union t3_wr {
	struct t3_send_wr send;
	struct t3_rdma_write_wr write;
	struct t3_rdma_read_wr read;
	struct t3_receive_wr recv;
	struct t3_local_inv_wr local_inv;
	struct t3_bind_mw_wr bind;
	struct t3_bypass_wr bypass;
	struct t3_rdma_init_wr init;
	struct t3_modify_qp_wr qp_mod;
	struct t3_genbit genbit;
	u64 flit[16];
	};

	#define T3_SQ_CQE_FLIT 13
	#define T3_SQ_COOKIE_FLIT 14

	#define T3_RQ_COOKIE_FLIT 13
	#define T3_RQ_CQE_FLIT 14

	static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
	{
	return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
	}

	static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
	enum t3_wr_flags flags, u8 genbit, u32 tid,
	u8 len)
	{
	wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) \|
	V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) \|
	V_FW_RIWR_FLAGS(flags));
	wmb();
	wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) \|
	V_FW_RIWR_TID(tid) \|
	V_FW_RIWR_LEN(len));
	/* 2nd gen bit... */
	((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit);
	}

	/*
	* T3 ULP2_TX commands
	*/
	enum t3_utx_mem_op {
	T3_UTX_MEM_READ = 2,
	T3_UTX_MEM_WRITE = 3
	};

	/* T3 MC7 RDMA TPT entry format */

	enum tpt_mem_type {
	TPT_NON_SHARED_MR = 0x0,
	TPT_SHARED_MR = 0x1,
	TPT_MW = 0x2,
	TPT_MW_RELAXED_PROTECTION = 0x3
	};

	enum tpt_addr_type {
	TPT_ZBTO = 0,
	TPT_VATO = 1
	};

	enum tpt_mem_perm {
	TPT_LOCAL_READ = 0x8,
	TPT_LOCAL_WRITE = 0x4,
	TPT_REMOTE_READ = 0x2,
	TPT_REMOTE_WRITE = 0x1
	};

	struct tpt_entry {
	__be32 valid_stag_pdid;
	__be32 flags_pagesize_qpid;

	__be32 rsvd_pbl_addr;
	__be32 len;
	__be32 va_hi;
	__be32 va_low_or_fbo;

	__be32 rsvd_bind_cnt_or_pstag;
	__be32 rsvd_pbl_size;
	};

	#define S_TPT_VALID 31
	#define V_TPT_VALID(x) ((x) << S_TPT_VALID)
	#define F_TPT_VALID V_TPT_VALID(1U)

	#define S_TPT_STAG_KEY 23
	#define M_TPT_STAG_KEY 0xFF
	#define V_TPT_STAG_KEY(x) ((x) << S_TPT_STAG_KEY)
	#define G_TPT_STAG_KEY(x) (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)

	#define S_TPT_STAG_STATE 22
	#define V_TPT_STAG_STATE(x) ((x) << S_TPT_STAG_STATE)
	#define F_TPT_STAG_STATE V_TPT_STAG_STATE(1U)

	#define S_TPT_STAG_TYPE 20
	#define M_TPT_STAG_TYPE 0x3
	#define V_TPT_STAG_TYPE(x) ((x) << S_TPT_STAG_TYPE)
	#define G_TPT_STAG_TYPE(x) (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)

	#define S_TPT_PDID 0
	#define M_TPT_PDID 0xFFFFF
	#define V_TPT_PDID(x) ((x) << S_TPT_PDID)
	#define G_TPT_PDID(x) (((x) >> S_TPT_PDID) & M_TPT_PDID)

	#define S_TPT_PERM 28
	#define M_TPT_PERM 0xF
	#define V_TPT_PERM(x) ((x) << S_TPT_PERM)
	#define G_TPT_PERM(x) (((x) >> S_TPT_PERM) & M_TPT_PERM)

	#define S_TPT_REM_INV_DIS 27
	#define V_TPT_REM_INV_DIS(x) ((x) << S_TPT_REM_INV_DIS)
	#define F_TPT_REM_INV_DIS V_TPT_REM_INV_DIS(1U)

	#define S_TPT_ADDR_TYPE 26
	#define V_TPT_ADDR_TYPE(x) ((x) << S_TPT_ADDR_TYPE)
	#define F_TPT_ADDR_TYPE V_TPT_ADDR_TYPE(1U)

	#define S_TPT_MW_BIND_ENABLE 25
	#define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE)
	#define F_TPT_MW_BIND_ENABLE V_TPT_MW_BIND_ENABLE(1U)

	#define S_TPT_PAGE_SIZE 20
	#define M_TPT_PAGE_SIZE 0x1F
	#define V_TPT_PAGE_SIZE(x) ((x) << S_TPT_PAGE_SIZE)
	#define G_TPT_PAGE_SIZE(x) (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)

	#define S_TPT_PBL_ADDR 0
	#define M_TPT_PBL_ADDR 0x1FFFFFFF
	#define V_TPT_PBL_ADDR(x) ((x) << S_TPT_PBL_ADDR)
	#define G_TPT_PBL_ADDR(x) (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)

	#define S_TPT_QPID 0
	#define M_TPT_QPID 0xFFFFF
	#define V_TPT_QPID(x) ((x) << S_TPT_QPID)
	#define G_TPT_QPID(x) (((x) >> S_TPT_QPID) & M_TPT_QPID)

	#define S_TPT_PSTAG 0
	#define M_TPT_PSTAG 0xFFFFFF
	#define V_TPT_PSTAG(x) ((x) << S_TPT_PSTAG)
	#define G_TPT_PSTAG(x) (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)

	#define S_TPT_PBL_SIZE 0
	#define M_TPT_PBL_SIZE 0xFFFFF
	#define V_TPT_PBL_SIZE(x) ((x) << S_TPT_PBL_SIZE)
	#define G_TPT_PBL_SIZE(x) (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)

	/*
	* CQE defs
	*/
	struct t3_cqe {
	__be32 header;
	__be32 len;
	union {
	struct {
	__be32 stag;
	__be32 msn;
	} rcqe;
	struct {
	u32 wrid_hi;
	u32 wrid_low;
	} scqe;
	} u;
	};

	#define S_CQE_OOO 31
	#define M_CQE_OOO 0x1
	#define G_CQE_OOO(x) ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
	#define V_CEQ_OOO(x) ((x)<<S_CQE_OOO)

	#define S_CQE_QPID 12
	#define M_CQE_QPID 0x7FFFF
	#define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
	#define V_CQE_QPID(x) ((x)<<S_CQE_QPID)

	#define S_CQE_SWCQE 11
	#define M_CQE_SWCQE 0x1
	#define G_CQE_SWCQE(x) ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
	#define V_CQE_SWCQE(x) ((x)<<S_CQE_SWCQE)

	#define S_CQE_GENBIT 10
	#define M_CQE_GENBIT 0x1
	#define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
	#define V_CQE_GENBIT(x) ((x)<<S_CQE_GENBIT)

	#define S_CQE_STATUS 5
	#define M_CQE_STATUS 0x1F
	#define G_CQE_STATUS(x) ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
	#define V_CQE_STATUS(x) ((x)<<S_CQE_STATUS)

	#define S_CQE_TYPE 4
	#define M_CQE_TYPE 0x1
	#define G_CQE_TYPE(x) ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
	#define V_CQE_TYPE(x) ((x)<<S_CQE_TYPE)

	#define S_CQE_OPCODE 0
	#define M_CQE_OPCODE 0xF
	#define G_CQE_OPCODE(x) ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
	#define V_CQE_OPCODE(x) ((x)<<S_CQE_OPCODE)

	#define SW_CQE(x) (G_CQE_SWCQE(be32_to_cpu((x).header)))
	#define CQE_OOO(x) (G_CQE_OOO(be32_to_cpu((x).header)))
	#define CQE_QPID(x) (G_CQE_QPID(be32_to_cpu((x).header)))
	#define CQE_GENBIT(x) (G_CQE_GENBIT(be32_to_cpu((x).header)))
	#define CQE_TYPE(x) (G_CQE_TYPE(be32_to_cpu((x).header)))
	#define SQ_TYPE(x) (CQE_TYPE((x)))
	#define RQ_TYPE(x) (!CQE_TYPE((x)))
	#define CQE_STATUS(x) (G_CQE_STATUS(be32_to_cpu((x).header)))
	#define CQE_OPCODE(x) (G_CQE_OPCODE(be32_to_cpu((x).header)))

	#define CQE_LEN(x) (be32_to_cpu((x).len))

	/* used for RQ completion processing */
	#define CQE_WRID_STAG(x) (be32_to_cpu((x).u.rcqe.stag))
	#define CQE_WRID_MSN(x) (be32_to_cpu((x).u.rcqe.msn))

	/* used for SQ completion processing */
	#define CQE_WRID_SQ_WPTR(x) ((x).u.scqe.wrid_hi)
	#define CQE_WRID_WPTR(x) ((x).u.scqe.wrid_low)

	/* generic accessor macros */
	#define CQE_WRID_HI(x) ((x).u.scqe.wrid_hi)
	#define CQE_WRID_LOW(x) ((x).u.scqe.wrid_low)

	#define TPT_ERR_SUCCESS 0x0
	#define TPT_ERR_STAG 0x1 /* STAG invalid: either the */
	/* STAG is offlimt, being 0, */
	/* or STAG_key mismatch */
	#define TPT_ERR_PDID 0x2 /* PDID mismatch */
	#define TPT_ERR_QPID 0x3 /* QPID mismatch */
	#define TPT_ERR_ACCESS 0x4 /* Invalid access right */
	#define TPT_ERR_WRAP 0x5 /* Wrap error */
	#define TPT_ERR_BOUND 0x6 /* base and bounds voilation */
	#define TPT_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */
	/* shared memory region */
	#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */
	/* shared memory region */
	#define TPT_ERR_ECC 0x9 /* ECC error detected */
	#define TPT_ERR_ECC_PSTAG 0xA /* ECC error detected when */
	/* reading PSTAG for a MW */
	/* Invalidate */
	#define TPT_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */
	/* software error */
	#define TPT_ERR_SWFLUSH 0xC /* SW FLUSHED */
	#define TPT_ERR_CRC 0x10 /* CRC error */
	#define TPT_ERR_MARKER 0x11 /* Marker error */
	#define TPT_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */
	#define TPT_ERR_OUT_OF_RQE 0x13 /* out of RQE */
	#define TPT_ERR_DDP_VERSION 0x14 /* wrong DDP version */
	#define TPT_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */
	#define TPT_ERR_OPCODE 0x16 /* invalid rdma opcode */
	#define TPT_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */
	#define TPT_ERR_MSN 0x18 /* MSN error */
	#define TPT_ERR_TBIT 0x19 /* tag bit not set correctly */
	#define TPT_ERR_MO 0x1A /* MO not 0 for TERMINATE */
	/* or READ_REQ */
	#define TPT_ERR_MSN_GAP 0x1B
	#define TPT_ERR_MSN_RANGE 0x1C
	#define TPT_ERR_IRD_OVERFLOW 0x1D
	#define TPT_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */
	/* software error */
	#define TPT_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */
	/* mismatch) */

	struct t3_swsq {
	__u64 wr_id;
	struct t3_cqe cqe;
	__u32 sq_wptr;
	__be32 read_len;
	int opcode;
	int complete;
	int signaled;
	};

	/*
	* A T3 WQ implements both the SQ and RQ.
	*/
	struct t3_wq {
	union t3_wr queue; / DMA accessable memory */
	dma_addr_t dma_addr; /* DMA address for HW */
	DECLARE_PCI_UNMAP_ADDR(mapping) /* unmap kruft */
	u32 error; /* 1 once we go to ERROR */
	u32 qpid;
	u32 wptr; /* idx to next available WR slot */
	u32 size_log2; /* total wq size */
	struct t3_swsq sq; / SW SQ */
	struct t3_swsq oldest_read; / tracks oldest pending read */
	u32 sq_wptr; /* sq_wptr - sq_rptr == count of */
	u32 sq_rptr; /* pending wrs */
	u32 sq_size_log2; /* sq size */
	u64 rq; / SW RQ (holds consumer wr_ids */
	u32 rq_wptr; /* rq_wptr - rq_rptr == count of */
	u32 rq_rptr; /* pending wrs */
	u64 rq_oldest_wr; / oldest wr on the SW RQ */
	u32 rq_size_log2; /* rq size */
	u32 rq_addr; /* rq adapter address */
	void __iomem doorbell; / kernel db */
	u64 udb; /* user db if any */
	};

	struct t3_cq {
	u32 cqid;
	u32 rptr;
	u32 wptr;
	u32 size_log2;
	dma_addr_t dma_addr;
	DECLARE_PCI_UNMAP_ADDR(mapping)
	struct t3_cqe *queue;
	struct t3_cqe *sw_queue;
	u32 sw_rptr;
	u32 sw_wptr;
	};

	#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
	CQE_GENBIT(*cqe))

	static inline void cxio_set_wq_in_error(struct t3_wq *wq)
	{
	wq->queue->flit[13] = 1;
	}

	static inline struct t3_cqe cxio_next_hw_cqe(struct t3_cq cq)
	{
	struct t3_cqe *cqe;

	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
	return cqe;
	return NULL;
	}

	static inline struct t3_cqe cxio_next_sw_cqe(struct t3_cq cq)
	{
	struct t3_cqe *cqe;

	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
	cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
	return cqe;
	}
	return NULL;
	}

	static inline struct t3_cqe cxio_next_cqe(struct t3_cq cq)
	{
	struct t3_cqe *cqe;

	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
	cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
	return cqe;
	}
	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
	return cqe;
	return NULL;
	}

	#endif