Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 1 | /* |
John Gregor | 87427da | 2007-06-11 10:21:14 -0700 | [diff] [blame] | 2 | * Copyright (c) 2006, 2007 QLogic Corporation. All rights reserved. |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 3 | * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. |
| 4 | * |
| 5 | * This software is available to you under a choice of one of two |
| 6 | * licenses. You may choose to be licensed under the terms of the GNU |
| 7 | * General Public License (GPL) Version 2, available from the file |
| 8 | * COPYING in the main directory of this source tree, or the |
| 9 | * OpenIB.org BSD license below: |
| 10 | * |
| 11 | * Redistribution and use in source and binary forms, with or |
| 12 | * without modification, are permitted provided that the following |
| 13 | * conditions are met: |
| 14 | * |
| 15 | * - Redistributions of source code must retain the above |
| 16 | * copyright notice, this list of conditions and the following |
| 17 | * disclaimer. |
| 18 | * |
| 19 | * - Redistributions in binary form must reproduce the above |
| 20 | * copyright notice, this list of conditions and the following |
| 21 | * disclaimer in the documentation and/or other materials |
| 22 | * provided with the distribution. |
| 23 | * |
| 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 31 | * SOFTWARE. |
| 32 | */ |
| 33 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 34 | #include <linux/spinlock.h> |
| 35 | |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 36 | #include "ipath_verbs.h" |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 37 | #include "ipath_kernel.h" |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 38 | |
| 39 | /* |
| 40 | * Convert the AETH RNR timeout code into the number of milliseconds. |
| 41 | */ |
| 42 | const u32 ib_ipath_rnr_table[32] = { |
| 43 | 656, /* 0 */ |
| 44 | 1, /* 1 */ |
| 45 | 1, /* 2 */ |
| 46 | 1, /* 3 */ |
| 47 | 1, /* 4 */ |
| 48 | 1, /* 5 */ |
| 49 | 1, /* 6 */ |
| 50 | 1, /* 7 */ |
| 51 | 1, /* 8 */ |
| 52 | 1, /* 9 */ |
| 53 | 1, /* A */ |
| 54 | 1, /* B */ |
| 55 | 1, /* C */ |
| 56 | 1, /* D */ |
| 57 | 2, /* E */ |
| 58 | 2, /* F */ |
| 59 | 3, /* 10 */ |
| 60 | 4, /* 11 */ |
| 61 | 6, /* 12 */ |
| 62 | 8, /* 13 */ |
| 63 | 11, /* 14 */ |
| 64 | 16, /* 15 */ |
| 65 | 21, /* 16 */ |
| 66 | 31, /* 17 */ |
| 67 | 41, /* 18 */ |
| 68 | 62, /* 19 */ |
| 69 | 82, /* 1A */ |
| 70 | 123, /* 1B */ |
| 71 | 164, /* 1C */ |
| 72 | 246, /* 1D */ |
| 73 | 328, /* 1E */ |
| 74 | 492 /* 1F */ |
| 75 | }; |
| 76 | |
| 77 | /** |
| 78 | * ipath_insert_rnr_queue - put QP on the RNR timeout list for the device |
| 79 | * @qp: the QP |
| 80 | * |
| 81 | * XXX Use a simple list for now. We might need a priority |
| 82 | * queue if we have lots of QPs waiting for RNR timeouts |
| 83 | * but that should be rare. |
| 84 | */ |
| 85 | void ipath_insert_rnr_queue(struct ipath_qp *qp) |
| 86 | { |
| 87 | struct ipath_ibdev *dev = to_idev(qp->ibqp.device); |
| 88 | unsigned long flags; |
| 89 | |
| 90 | spin_lock_irqsave(&dev->pending_lock, flags); |
| 91 | if (list_empty(&dev->rnrwait)) |
| 92 | list_add(&qp->timerwait, &dev->rnrwait); |
| 93 | else { |
| 94 | struct list_head *l = &dev->rnrwait; |
| 95 | struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp, |
| 96 | timerwait); |
| 97 | |
| 98 | while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) { |
| 99 | qp->s_rnr_timeout -= nqp->s_rnr_timeout; |
| 100 | l = l->next; |
Ralph Campbell | cc65edc | 2007-12-14 19:22:34 -0800 | [diff] [blame] | 101 | if (l->next == &dev->rnrwait) { |
| 102 | nqp = NULL; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 103 | break; |
Ralph Campbell | cc65edc | 2007-12-14 19:22:34 -0800 | [diff] [blame] | 104 | } |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 105 | nqp = list_entry(l->next, struct ipath_qp, |
| 106 | timerwait); |
| 107 | } |
Ralph Campbell | cc65edc | 2007-12-14 19:22:34 -0800 | [diff] [blame] | 108 | if (nqp) |
| 109 | nqp->s_rnr_timeout -= qp->s_rnr_timeout; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 110 | list_add(&qp->timerwait, l); |
| 111 | } |
| 112 | spin_unlock_irqrestore(&dev->pending_lock, flags); |
| 113 | } |
| 114 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 115 | /** |
| 116 | * ipath_init_sge - Validate a RWQE and fill in the SGE state |
| 117 | * @qp: the QP |
| 118 | * |
| 119 | * Return 1 if OK. |
| 120 | */ |
| 121 | int ipath_init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe, |
| 122 | u32 *lengthp, struct ipath_sge_state *ss) |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 123 | { |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 124 | int i, j, ret; |
| 125 | struct ib_wc wc; |
| 126 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 127 | *lengthp = 0; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 128 | for (i = j = 0; i < wqe->num_sge; i++) { |
| 129 | if (wqe->sg_list[i].length == 0) |
| 130 | continue; |
| 131 | /* Check LKEY */ |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 132 | if (!ipath_lkey_ok(qp, j ? &ss->sg_list[j - 1] : &ss->sge, |
| 133 | &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE)) |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 134 | goto bad_lkey; |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 135 | *lengthp += wqe->sg_list[i].length; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 136 | j++; |
| 137 | } |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 138 | ss->num_sge = j; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 139 | ret = 1; |
| 140 | goto bail; |
| 141 | |
| 142 | bad_lkey: |
| 143 | wc.wr_id = wqe->wr_id; |
| 144 | wc.status = IB_WC_LOC_PROT_ERR; |
| 145 | wc.opcode = IB_WC_RECV; |
| 146 | wc.vendor_err = 0; |
| 147 | wc.byte_len = 0; |
| 148 | wc.imm_data = 0; |
Michael S. Tsirkin | 062dbb6 | 2006-12-31 21:09:42 +0200 | [diff] [blame] | 149 | wc.qp = &qp->ibqp; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 150 | wc.src_qp = 0; |
| 151 | wc.wc_flags = 0; |
| 152 | wc.pkey_index = 0; |
| 153 | wc.slid = 0; |
| 154 | wc.sl = 0; |
| 155 | wc.dlid_path_bits = 0; |
| 156 | wc.port_num = 0; |
| 157 | /* Signal solicited completion event. */ |
| 158 | ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1); |
| 159 | ret = 0; |
| 160 | bail: |
| 161 | return ret; |
| 162 | } |
| 163 | |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 164 | /** |
| 165 | * ipath_get_rwqe - copy the next RWQE into the QP's RWQE |
| 166 | * @qp: the QP |
| 167 | * @wr_id_only: update wr_id only, not SGEs |
| 168 | * |
| 169 | * Return 0 if no RWQE is available, otherwise return 1. |
| 170 | * |
Bryan O'Sullivan | 12eef41 | 2006-07-01 04:36:10 -0700 | [diff] [blame] | 171 | * Can be called from interrupt level. |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 172 | */ |
| 173 | int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only) |
| 174 | { |
Bryan O'Sullivan | 12eef41 | 2006-07-01 04:36:10 -0700 | [diff] [blame] | 175 | unsigned long flags; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 176 | struct ipath_rq *rq; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 177 | struct ipath_rwq *wq; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 178 | struct ipath_srq *srq; |
| 179 | struct ipath_rwqe *wqe; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 180 | void (*handler)(struct ib_event *, void *); |
| 181 | u32 tail; |
| 182 | int ret; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 183 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 184 | qp->r_sge.sg_list = qp->r_sg_list; |
| 185 | |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 186 | if (qp->ibqp.srq) { |
| 187 | srq = to_isrq(qp->ibqp.srq); |
| 188 | handler = srq->ibsrq.event_handler; |
| 189 | rq = &srq->rq; |
| 190 | } else { |
| 191 | srq = NULL; |
| 192 | handler = NULL; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 193 | rq = &qp->r_rq; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 194 | } |
| 195 | |
Bryan O'Sullivan | 12eef41 | 2006-07-01 04:36:10 -0700 | [diff] [blame] | 196 | spin_lock_irqsave(&rq->lock, flags); |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 197 | wq = rq->wq; |
| 198 | tail = wq->tail; |
| 199 | /* Validate tail before using it since it is user writable. */ |
| 200 | if (tail >= rq->size) |
| 201 | tail = 0; |
| 202 | do { |
| 203 | if (unlikely(tail == wq->head)) { |
| 204 | spin_unlock_irqrestore(&rq->lock, flags); |
| 205 | ret = 0; |
| 206 | goto bail; |
| 207 | } |
Ralph Campbell | 4fc570b | 2007-07-06 12:48:23 -0700 | [diff] [blame] | 208 | /* Make sure entry is read after head index is read. */ |
| 209 | smp_rmb(); |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 210 | wqe = get_rwqe_ptr(rq, tail); |
| 211 | if (++tail >= rq->size) |
| 212 | tail = 0; |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 213 | } while (!wr_id_only && !ipath_init_sge(qp, wqe, &qp->r_len, |
| 214 | &qp->r_sge)); |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 215 | qp->r_wr_id = wqe->wr_id; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 216 | wq->tail = tail; |
| 217 | |
| 218 | ret = 1; |
Ralph Campbell | 9f9630d | 2007-03-15 14:44:49 -0700 | [diff] [blame] | 219 | qp->r_wrid_valid = 1; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 220 | if (handler) { |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 221 | u32 n; |
| 222 | |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 223 | /* |
| 224 | * validate head pointer value and compute |
| 225 | * the number of remaining WQEs. |
| 226 | */ |
| 227 | n = wq->head; |
| 228 | if (n >= rq->size) |
| 229 | n = 0; |
| 230 | if (n < tail) |
| 231 | n += rq->size - tail; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 232 | else |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 233 | n -= tail; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 234 | if (n < srq->limit) { |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 235 | struct ib_event ev; |
| 236 | |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 237 | srq->limit = 0; |
Bryan O'Sullivan | 12eef41 | 2006-07-01 04:36:10 -0700 | [diff] [blame] | 238 | spin_unlock_irqrestore(&rq->lock, flags); |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 239 | ev.device = qp->ibqp.device; |
| 240 | ev.element.srq = qp->ibqp.srq; |
| 241 | ev.event = IB_EVENT_SRQ_LIMIT_REACHED; |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 242 | handler(&ev, srq->ibsrq.srq_context); |
Bryan O'Sullivan | 12eef41 | 2006-07-01 04:36:10 -0700 | [diff] [blame] | 243 | goto bail; |
| 244 | } |
| 245 | } |
Bryan O'Sullivan | 12eef41 | 2006-07-01 04:36:10 -0700 | [diff] [blame] | 246 | spin_unlock_irqrestore(&rq->lock, flags); |
Ralph Campbell | 373d991 | 2006-09-22 15:22:26 -0700 | [diff] [blame] | 247 | |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 248 | bail: |
| 249 | return ret; |
| 250 | } |
| 251 | |
| 252 | /** |
| 253 | * ipath_ruc_loopback - handle UC and RC lookback requests |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 254 | * @sqp: the sending QP |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 255 | * |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 256 | * This is called from ipath_do_send() to |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 257 | * forward a WQE addressed to the same HCA. |
| 258 | * Note that although we are single threaded due to the tasklet, we still |
| 259 | * have to protect against post_send(). We don't have to worry about |
| 260 | * receive interrupts since this is a connected protocol and all packets |
| 261 | * will pass through here. |
| 262 | */ |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 263 | static void ipath_ruc_loopback(struct ipath_qp *sqp) |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 264 | { |
| 265 | struct ipath_ibdev *dev = to_idev(sqp->ibqp.device); |
| 266 | struct ipath_qp *qp; |
| 267 | struct ipath_swqe *wqe; |
| 268 | struct ipath_sge *sge; |
| 269 | unsigned long flags; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 270 | struct ib_wc wc; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 271 | u64 sdata; |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 272 | atomic64_t *maddr; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 273 | |
| 274 | qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn); |
| 275 | if (!qp) { |
| 276 | dev->n_pkt_drops++; |
| 277 | return; |
| 278 | } |
| 279 | |
| 280 | again: |
| 281 | spin_lock_irqsave(&sqp->s_lock, flags); |
| 282 | |
Ralph Campbell | 7b21d26 | 2007-03-15 14:44:50 -0700 | [diff] [blame] | 283 | if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_SEND_OK) || |
Ralph Campbell | 6d2fad0 | 2007-06-18 14:24:38 -0700 | [diff] [blame] | 284 | sqp->s_rnr_timeout) { |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 285 | spin_unlock_irqrestore(&sqp->s_lock, flags); |
| 286 | goto done; |
| 287 | } |
| 288 | |
| 289 | /* Get the next send request. */ |
| 290 | if (sqp->s_last == sqp->s_head) { |
| 291 | /* Send work queue is empty. */ |
| 292 | spin_unlock_irqrestore(&sqp->s_lock, flags); |
| 293 | goto done; |
| 294 | } |
| 295 | |
| 296 | /* |
| 297 | * We can rely on the entry not changing without the s_lock |
| 298 | * being held until we update s_last. |
| 299 | */ |
| 300 | wqe = get_swqe_ptr(sqp, sqp->s_last); |
| 301 | spin_unlock_irqrestore(&sqp->s_lock, flags); |
| 302 | |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 303 | wc.wc_flags = 0; |
| 304 | wc.imm_data = 0; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 305 | |
| 306 | sqp->s_sge.sge = wqe->sg_list[0]; |
| 307 | sqp->s_sge.sg_list = wqe->sg_list + 1; |
| 308 | sqp->s_sge.num_sge = wqe->wr.num_sge; |
| 309 | sqp->s_len = wqe->length; |
| 310 | switch (wqe->wr.opcode) { |
| 311 | case IB_WR_SEND_WITH_IMM: |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 312 | wc.wc_flags = IB_WC_WITH_IMM; |
| 313 | wc.imm_data = wqe->wr.imm_data; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 314 | /* FALLTHROUGH */ |
| 315 | case IB_WR_SEND: |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 316 | if (!ipath_get_rwqe(qp, 0)) { |
| 317 | rnr_nak: |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 318 | /* Handle RNR NAK */ |
| 319 | if (qp->ibqp.qp_type == IB_QPT_UC) |
| 320 | goto send_comp; |
| 321 | if (sqp->s_rnr_retry == 0) { |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 322 | wc.status = IB_WC_RNR_RETRY_EXC_ERR; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 323 | goto err; |
| 324 | } |
| 325 | if (sqp->s_rnr_retry_cnt < 7) |
| 326 | sqp->s_rnr_retry--; |
| 327 | dev->n_rnr_naks++; |
| 328 | sqp->s_rnr_timeout = |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 329 | ib_ipath_rnr_table[qp->r_min_rnr_timer]; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 330 | ipath_insert_rnr_queue(sqp); |
| 331 | goto done; |
| 332 | } |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 333 | break; |
| 334 | |
| 335 | case IB_WR_RDMA_WRITE_WITH_IMM: |
Robert Walsh | b506e1d | 2007-06-18 14:24:48 -0700 | [diff] [blame] | 336 | if (unlikely(!(qp->qp_access_flags & |
| 337 | IB_ACCESS_REMOTE_WRITE))) { |
| 338 | wc.status = IB_WC_REM_INV_REQ_ERR; |
| 339 | goto err; |
| 340 | } |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 341 | wc.wc_flags = IB_WC_WITH_IMM; |
| 342 | wc.imm_data = wqe->wr.imm_data; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 343 | if (!ipath_get_rwqe(qp, 1)) |
| 344 | goto rnr_nak; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 345 | /* FALLTHROUGH */ |
| 346 | case IB_WR_RDMA_WRITE: |
Robert Walsh | b506e1d | 2007-06-18 14:24:48 -0700 | [diff] [blame] | 347 | if (unlikely(!(qp->qp_access_flags & |
| 348 | IB_ACCESS_REMOTE_WRITE))) { |
| 349 | wc.status = IB_WC_REM_INV_REQ_ERR; |
| 350 | goto err; |
| 351 | } |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 352 | if (wqe->length == 0) |
| 353 | break; |
Bryan O'Sullivan | 6a553af | 2006-09-28 09:00:07 -0700 | [diff] [blame] | 354 | if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, wqe->length, |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 355 | wqe->wr.wr.rdma.remote_addr, |
| 356 | wqe->wr.wr.rdma.rkey, |
| 357 | IB_ACCESS_REMOTE_WRITE))) { |
| 358 | acc_err: |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 359 | wc.status = IB_WC_REM_ACCESS_ERR; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 360 | err: |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 361 | wc.wr_id = wqe->wr.wr_id; |
| 362 | wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; |
| 363 | wc.vendor_err = 0; |
| 364 | wc.byte_len = 0; |
Michael S. Tsirkin | 062dbb6 | 2006-12-31 21:09:42 +0200 | [diff] [blame] | 365 | wc.qp = &sqp->ibqp; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 366 | wc.src_qp = sqp->remote_qpn; |
| 367 | wc.pkey_index = 0; |
| 368 | wc.slid = sqp->remote_ah_attr.dlid; |
| 369 | wc.sl = sqp->remote_ah_attr.sl; |
| 370 | wc.dlid_path_bits = 0; |
| 371 | wc.port_num = 0; |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 372 | spin_lock_irqsave(&sqp->s_lock, flags); |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 373 | ipath_sqerror_qp(sqp, &wc); |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 374 | spin_unlock_irqrestore(&sqp->s_lock, flags); |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 375 | goto done; |
| 376 | } |
| 377 | break; |
| 378 | |
| 379 | case IB_WR_RDMA_READ: |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 380 | if (unlikely(!(qp->qp_access_flags & |
Robert Walsh | b506e1d | 2007-06-18 14:24:48 -0700 | [diff] [blame] | 381 | IB_ACCESS_REMOTE_READ))) { |
| 382 | wc.status = IB_WC_REM_INV_REQ_ERR; |
| 383 | goto err; |
| 384 | } |
Bryan O'Sullivan | 6a553af | 2006-09-28 09:00:07 -0700 | [diff] [blame] | 385 | if (unlikely(!ipath_rkey_ok(qp, &sqp->s_sge, wqe->length, |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 386 | wqe->wr.wr.rdma.remote_addr, |
| 387 | wqe->wr.wr.rdma.rkey, |
| 388 | IB_ACCESS_REMOTE_READ))) |
| 389 | goto acc_err; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 390 | qp->r_sge.sge = wqe->sg_list[0]; |
| 391 | qp->r_sge.sg_list = wqe->sg_list + 1; |
| 392 | qp->r_sge.num_sge = wqe->wr.num_sge; |
| 393 | break; |
| 394 | |
| 395 | case IB_WR_ATOMIC_CMP_AND_SWP: |
| 396 | case IB_WR_ATOMIC_FETCH_AND_ADD: |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 397 | if (unlikely(!(qp->qp_access_flags & |
Robert Walsh | b506e1d | 2007-06-18 14:24:48 -0700 | [diff] [blame] | 398 | IB_ACCESS_REMOTE_ATOMIC))) { |
| 399 | wc.status = IB_WC_REM_INV_REQ_ERR; |
| 400 | goto err; |
| 401 | } |
Bryan O'Sullivan | 6a553af | 2006-09-28 09:00:07 -0700 | [diff] [blame] | 402 | if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, sizeof(u64), |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 403 | wqe->wr.wr.atomic.remote_addr, |
| 404 | wqe->wr.wr.atomic.rkey, |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 405 | IB_ACCESS_REMOTE_ATOMIC))) |
| 406 | goto acc_err; |
| 407 | /* Perform atomic OP and save result. */ |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 408 | maddr = (atomic64_t *) qp->r_sge.sge.vaddr; |
| 409 | sdata = wqe->wr.wr.atomic.compare_add; |
| 410 | *(u64 *) sqp->s_sge.sge.vaddr = |
| 411 | (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? |
| 412 | (u64) atomic64_add_return(sdata, maddr) - sdata : |
| 413 | (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, |
| 414 | sdata, wqe->wr.wr.atomic.swap); |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 415 | goto send_comp; |
| 416 | |
| 417 | default: |
| 418 | goto done; |
| 419 | } |
| 420 | |
| 421 | sge = &sqp->s_sge.sge; |
| 422 | while (sqp->s_len) { |
| 423 | u32 len = sqp->s_len; |
| 424 | |
| 425 | if (len > sge->length) |
| 426 | len = sge->length; |
Ralph Campbell | 30d149a | 2007-06-18 14:24:44 -0700 | [diff] [blame] | 427 | if (len > sge->sge_length) |
| 428 | len = sge->sge_length; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 429 | BUG_ON(len == 0); |
| 430 | ipath_copy_sge(&qp->r_sge, sge->vaddr, len); |
| 431 | sge->vaddr += len; |
| 432 | sge->length -= len; |
| 433 | sge->sge_length -= len; |
| 434 | if (sge->sge_length == 0) { |
| 435 | if (--sqp->s_sge.num_sge) |
| 436 | *sge = *sqp->s_sge.sg_list++; |
| 437 | } else if (sge->length == 0 && sge->mr != NULL) { |
| 438 | if (++sge->n >= IPATH_SEGSZ) { |
| 439 | if (++sge->m >= sge->mr->mapsz) |
| 440 | break; |
| 441 | sge->n = 0; |
| 442 | } |
| 443 | sge->vaddr = |
| 444 | sge->mr->map[sge->m]->segs[sge->n].vaddr; |
| 445 | sge->length = |
| 446 | sge->mr->map[sge->m]->segs[sge->n].length; |
| 447 | } |
| 448 | sqp->s_len -= len; |
| 449 | } |
| 450 | |
| 451 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE || |
| 452 | wqe->wr.opcode == IB_WR_RDMA_READ) |
| 453 | goto send_comp; |
| 454 | |
| 455 | if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 456 | wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 457 | else |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 458 | wc.opcode = IB_WC_RECV; |
| 459 | wc.wr_id = qp->r_wr_id; |
| 460 | wc.status = IB_WC_SUCCESS; |
| 461 | wc.vendor_err = 0; |
| 462 | wc.byte_len = wqe->length; |
Michael S. Tsirkin | 062dbb6 | 2006-12-31 21:09:42 +0200 | [diff] [blame] | 463 | wc.qp = &qp->ibqp; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 464 | wc.src_qp = qp->remote_qpn; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 465 | wc.pkey_index = 0; |
| 466 | wc.slid = qp->remote_ah_attr.dlid; |
| 467 | wc.sl = qp->remote_ah_attr.sl; |
| 468 | wc.dlid_path_bits = 0; |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 469 | wc.port_num = 1; |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 470 | /* Signal completion event if the solicited bit is set. */ |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 471 | ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 472 | wqe->wr.send_flags & IB_SEND_SOLICITED); |
| 473 | |
| 474 | send_comp: |
| 475 | sqp->s_rnr_retry = sqp->s_rnr_retry_cnt; |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 476 | ipath_send_complete(sqp, wqe, IB_WC_SUCCESS); |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 477 | goto again; |
| 478 | |
| 479 | done: |
| 480 | if (atomic_dec_and_test(&qp->refcount)) |
| 481 | wake_up(&qp->wait); |
| 482 | } |
| 483 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 484 | static void want_buffer(struct ipath_devdata *dd) |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 485 | { |
John Gregor | e342c11 | 2007-09-05 01:57:14 -0700 | [diff] [blame] | 486 | unsigned long flags; |
| 487 | |
| 488 | spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); |
| 489 | dd->ipath_sendctrl |= INFINIPATH_S_PIOINTBUFAVAIL; |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 490 | ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, |
| 491 | dd->ipath_sendctrl); |
John Gregor | e342c11 | 2007-09-05 01:57:14 -0700 | [diff] [blame] | 492 | ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); |
| 493 | spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 494 | } |
| 495 | |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 496 | /** |
| 497 | * ipath_no_bufs_available - tell the layer driver we need buffers |
| 498 | * @qp: the QP that caused the problem |
| 499 | * @dev: the device we ran out of buffers on |
| 500 | * |
| 501 | * Called when we run out of PIO buffers. |
| 502 | */ |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 503 | static void ipath_no_bufs_available(struct ipath_qp *qp, |
| 504 | struct ipath_ibdev *dev) |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 505 | { |
| 506 | unsigned long flags; |
| 507 | |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 508 | /* |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 509 | * Note that as soon as want_buffer() is called and |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 510 | * possibly before it returns, ipath_ib_piobufavail() |
| 511 | * could be called. If we are still in the tasklet function, |
| 512 | * tasklet_hi_schedule() will not call us until the next time |
| 513 | * tasklet_hi_schedule() is called. |
Ralph Campbell | db5518c | 2007-06-18 14:24:43 -0700 | [diff] [blame] | 514 | * We leave the busy flag set so that another post send doesn't |
| 515 | * try to put the same QP on the piowait list again. |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 516 | */ |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 517 | spin_lock_irqsave(&dev->pending_lock, flags); |
| 518 | list_add_tail(&qp->piowait, &dev->piowait); |
| 519 | spin_unlock_irqrestore(&dev->pending_lock, flags); |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 520 | want_buffer(dev->dd); |
Bryan O'Sullivan | e28c00a | 2006-03-29 15:23:37 -0800 | [diff] [blame] | 521 | dev->n_piowait++; |
| 522 | } |
| 523 | |
| 524 | /** |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 525 | * ipath_make_grh - construct a GRH header |
| 526 | * @dev: a pointer to the ipath device |
| 527 | * @hdr: a pointer to the GRH header being constructed |
| 528 | * @grh: the global route address to send to |
| 529 | * @hwords: the number of 32 bit words of header being sent |
| 530 | * @nwords: the number of 32 bit words of data being sent |
| 531 | * |
| 532 | * Return the size of the header in 32 bit words. |
| 533 | */ |
| 534 | u32 ipath_make_grh(struct ipath_ibdev *dev, struct ib_grh *hdr, |
| 535 | struct ib_global_route *grh, u32 hwords, u32 nwords) |
| 536 | { |
| 537 | hdr->version_tclass_flow = |
| 538 | cpu_to_be32((6 << 28) | |
| 539 | (grh->traffic_class << 20) | |
| 540 | grh->flow_label); |
| 541 | hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2); |
| 542 | /* next_hdr is defined by C8-7 in ch. 8.4.1 */ |
| 543 | hdr->next_hdr = 0x1B; |
| 544 | hdr->hop_limit = grh->hop_limit; |
| 545 | /* The SGID is 32-bit aligned. */ |
| 546 | hdr->sgid.global.subnet_prefix = dev->gid_prefix; |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 547 | hdr->sgid.global.interface_id = dev->dd->ipath_guid; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 548 | hdr->dgid = grh->dgid; |
| 549 | |
| 550 | /* GRH header size in 32-bit words. */ |
| 551 | return sizeof(struct ib_grh) / sizeof(u32); |
| 552 | } |
| 553 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 554 | void ipath_make_ruc_header(struct ipath_ibdev *dev, struct ipath_qp *qp, |
| 555 | struct ipath_other_headers *ohdr, |
| 556 | u32 bth0, u32 bth2) |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 557 | { |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 558 | u16 lrh0; |
| 559 | u32 nwords; |
| 560 | u32 extra_bytes; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 561 | |
| 562 | /* Construct the header. */ |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 563 | extra_bytes = -qp->s_cur_size & 3; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 564 | nwords = (qp->s_cur_size + extra_bytes) >> 2; |
Bryan O'Sullivan | 27b678d | 2006-07-01 04:36:17 -0700 | [diff] [blame] | 565 | lrh0 = IPATH_LRH_BTH; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 566 | if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) { |
| 567 | qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh, |
| 568 | &qp->remote_ah_attr.grh, |
| 569 | qp->s_hdrwords, nwords); |
Bryan O'Sullivan | 27b678d | 2006-07-01 04:36:17 -0700 | [diff] [blame] | 570 | lrh0 = IPATH_LRH_GRH; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 571 | } |
| 572 | lrh0 |= qp->remote_ah_attr.sl << 4; |
| 573 | qp->s_hdr.lrh[0] = cpu_to_be16(lrh0); |
| 574 | qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid); |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 575 | qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC); |
Bryan O'Sullivan | 34b2aaf | 2006-08-25 11:24:32 -0700 | [diff] [blame] | 576 | qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid); |
| 577 | bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index); |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 578 | bth0 |= extra_bytes << 20; |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 579 | ohdr->bth[0] = cpu_to_be32(bth0 | (1 << 22)); |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 580 | ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); |
| 581 | ohdr->bth[2] = cpu_to_be32(bth2); |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 582 | } |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 583 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 584 | /** |
| 585 | * ipath_do_send - perform a send on a QP |
| 586 | * @data: contains a pointer to the QP |
| 587 | * |
| 588 | * Process entries in the send work queue until credit or queue is |
| 589 | * exhausted. Only allow one CPU to send a packet per QP (tasklet). |
| 590 | * Otherwise, two threads could send packets out of order. |
| 591 | */ |
| 592 | void ipath_do_send(unsigned long data) |
| 593 | { |
| 594 | struct ipath_qp *qp = (struct ipath_qp *)data; |
| 595 | struct ipath_ibdev *dev = to_idev(qp->ibqp.device); |
| 596 | int (*make_req)(struct ipath_qp *qp); |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 597 | |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 598 | if (test_and_set_bit(IPATH_S_BUSY, &qp->s_busy)) |
| 599 | goto bail; |
| 600 | |
| 601 | if ((qp->ibqp.qp_type == IB_QPT_RC || |
| 602 | qp->ibqp.qp_type == IB_QPT_UC) && |
| 603 | qp->remote_ah_attr.dlid == dev->dd->ipath_lid) { |
| 604 | ipath_ruc_loopback(qp); |
| 605 | goto clear; |
| 606 | } |
| 607 | |
| 608 | if (qp->ibqp.qp_type == IB_QPT_RC) |
| 609 | make_req = ipath_make_rc_req; |
| 610 | else if (qp->ibqp.qp_type == IB_QPT_UC) |
| 611 | make_req = ipath_make_uc_req; |
| 612 | else |
| 613 | make_req = ipath_make_ud_req; |
| 614 | |
| 615 | again: |
| 616 | /* Check for a constructed packet to be sent. */ |
| 617 | if (qp->s_hdrwords != 0) { |
| 618 | /* |
| 619 | * If no PIO bufs are available, return. An interrupt will |
| 620 | * call ipath_ib_piobufavail() when one is available. |
| 621 | */ |
| 622 | if (ipath_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords, |
| 623 | qp->s_cur_sge, qp->s_cur_size)) { |
| 624 | ipath_no_bufs_available(qp, dev); |
| 625 | goto bail; |
| 626 | } |
| 627 | dev->n_unicast_xmit++; |
| 628 | /* Record that we sent the packet and s_hdr is empty. */ |
| 629 | qp->s_hdrwords = 0; |
| 630 | } |
| 631 | |
| 632 | if (make_req(qp)) |
| 633 | goto again; |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 634 | clear: |
Ralph Campbell | 3859e39 | 2007-03-15 14:44:51 -0700 | [diff] [blame] | 635 | clear_bit(IPATH_S_BUSY, &qp->s_busy); |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 636 | bail:; |
| 637 | } |
| 638 | |
| 639 | void ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe, |
| 640 | enum ib_wc_status status) |
| 641 | { |
Ralph Campbell | fffbfea | 2007-10-19 15:04:10 -0700 | [diff] [blame] | 642 | unsigned long flags; |
| 643 | u32 last; |
Ralph Campbell | 4ee9718 | 2007-07-25 11:08:28 -0700 | [diff] [blame] | 644 | |
| 645 | /* See ch. 11.2.4.1 and 10.7.3.1 */ |
| 646 | if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) || |
| 647 | (wqe->wr.send_flags & IB_SEND_SIGNALED) || |
| 648 | status != IB_WC_SUCCESS) { |
| 649 | struct ib_wc wc; |
| 650 | |
| 651 | wc.wr_id = wqe->wr.wr_id; |
| 652 | wc.status = status; |
| 653 | wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; |
| 654 | wc.vendor_err = 0; |
| 655 | wc.byte_len = wqe->length; |
| 656 | wc.imm_data = 0; |
| 657 | wc.qp = &qp->ibqp; |
| 658 | wc.src_qp = 0; |
| 659 | wc.wc_flags = 0; |
| 660 | wc.pkey_index = 0; |
| 661 | wc.slid = 0; |
| 662 | wc.sl = 0; |
| 663 | wc.dlid_path_bits = 0; |
| 664 | wc.port_num = 0; |
| 665 | ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0); |
| 666 | } |
Ralph Campbell | fffbfea | 2007-10-19 15:04:10 -0700 | [diff] [blame] | 667 | |
| 668 | spin_lock_irqsave(&qp->s_lock, flags); |
| 669 | last = qp->s_last; |
| 670 | if (++last >= qp->s_size) |
| 671 | last = 0; |
| 672 | qp->s_last = last; |
| 673 | spin_unlock_irqrestore(&qp->s_lock, flags); |
Bryan O'Sullivan | ddd4bb2 | 2006-07-01 04:35:50 -0700 | [diff] [blame] | 674 | } |