Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. |
| 3 | * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. |
Or Gerlitz | 3ee07d2 | 2014-04-01 16:28:41 +0300 | [diff] [blame] | 4 | * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 5 | * |
| 6 | * This software is available to you under a choice of one of two |
| 7 | * licenses. You may choose to be licensed under the terms of the GNU |
| 8 | * General Public License (GPL) Version 2, available from the file |
| 9 | * COPYING in the main directory of this source tree, or the |
| 10 | * OpenIB.org BSD license below: |
| 11 | * |
| 12 | * Redistribution and use in source and binary forms, with or |
| 13 | * without modification, are permitted provided that the following |
| 14 | * conditions are met: |
| 15 | * |
| 16 | * - Redistributions of source code must retain the above |
| 17 | * copyright notice, this list of conditions and the following |
| 18 | * disclaimer. |
| 19 | * |
| 20 | * - Redistributions in binary form must reproduce the above |
| 21 | * copyright notice, this list of conditions and the following |
| 22 | * disclaimer in the documentation and/or other materials |
| 23 | * provided with the distribution. |
| 24 | * |
| 25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 | * SOFTWARE. |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 33 | */ |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 34 | #include <linux/kernel.h> |
| 35 | #include <linux/module.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 36 | #include <linux/slab.h> |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 37 | #include <linux/delay.h> |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 38 | |
| 39 | #include "iscsi_iser.h" |
| 40 | |
| 41 | #define ISCSI_ISER_MAX_CONN 8 |
Sagi Grimberg | 6aabfa7 | 2014-10-01 14:02:09 +0300 | [diff] [blame] | 42 | #define ISER_MAX_RX_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN) |
| 43 | #define ISER_MAX_TX_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN) |
Sagi Grimberg | ff3dd52 | 2014-10-01 14:02:10 +0300 | [diff] [blame] | 44 | #define ISER_MAX_CQ_LEN (ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \ |
| 45 | ISCSI_ISER_MAX_CONN) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 46 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 47 | static void iser_qp_event_callback(struct ib_event *cause, void *context) |
| 48 | { |
Sagi Grimberg | 871e00a | 2015-05-18 13:40:30 +0300 | [diff] [blame] | 49 | iser_err("qp event %s (%d)\n", |
| 50 | ib_event_msg(cause->event), cause->event); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 51 | } |
| 52 | |
Or Gerlitz | 2110f9b | 2010-05-05 17:30:10 +0300 | [diff] [blame] | 53 | static void iser_event_handler(struct ib_event_handler *handler, |
| 54 | struct ib_event *event) |
| 55 | { |
Sagi Grimberg | 871e00a | 2015-05-18 13:40:30 +0300 | [diff] [blame] | 56 | iser_err("async event %s (%d) on device %s port %d\n", |
| 57 | ib_event_msg(event->event), event->event, |
| 58 | event->device->name, event->element.port_num); |
Or Gerlitz | 2110f9b | 2010-05-05 17:30:10 +0300 | [diff] [blame] | 59 | } |
| 60 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 61 | /** |
| 62 | * iser_create_device_ib_res - creates Protection Domain (PD), Completion |
| 63 | * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with |
| 64 | * the adapator. |
| 65 | * |
| 66 | * returns 0 on success, -1 on failure |
| 67 | */ |
| 68 | static int iser_create_device_ib_res(struct iser_device *device) |
| 69 | { |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 70 | struct ib_device *ib_dev = device->ib_device; |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 71 | int ret, i, max_cqe; |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 72 | |
Sagi Grimberg | 48afbff | 2015-08-06 18:32:56 +0300 | [diff] [blame] | 73 | ret = iser_assign_reg_ops(device); |
| 74 | if (ret) |
| 75 | return ret; |
Sagi Grimberg | b4e155f | 2013-07-28 12:35:39 +0300 | [diff] [blame] | 76 | |
Sagi Grimberg | da64bdb | 2014-12-07 16:10:03 +0200 | [diff] [blame] | 77 | device->comps_used = min_t(int, num_online_cpus(), |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 78 | ib_dev->num_comp_vectors); |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 79 | |
Sagi Grimberg | da64bdb | 2014-12-07 16:10:03 +0200 | [diff] [blame] | 80 | device->comps = kcalloc(device->comps_used, sizeof(*device->comps), |
| 81 | GFP_KERNEL); |
| 82 | if (!device->comps) |
| 83 | goto comps_err; |
| 84 | |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 85 | max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe); |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 86 | |
| 87 | iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n", |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 88 | device->comps_used, ib_dev->name, |
| 89 | ib_dev->num_comp_vectors, max_cqe); |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 90 | |
Christoph Hellwig | 8e61212 | 2016-09-05 12:56:18 +0200 | [diff] [blame] | 91 | device->pd = ib_alloc_pd(ib_dev, |
| 92 | iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 93 | if (IS_ERR(device->pd)) |
| 94 | goto pd_err; |
| 95 | |
Sagi Grimberg | bf17554 | 2014-10-01 14:02:07 +0300 | [diff] [blame] | 96 | for (i = 0; i < device->comps_used; i++) { |
| 97 | struct iser_comp *comp = &device->comps[i]; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 98 | |
Doug Ledford | 882f3b3 | 2015-12-22 17:03:15 -0500 | [diff] [blame] | 99 | comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i, |
| 100 | IB_POLL_SOFTIRQ); |
Sagi Grimberg | 6aabfa7 | 2014-10-01 14:02:09 +0300 | [diff] [blame] | 101 | if (IS_ERR(comp->cq)) { |
| 102 | comp->cq = NULL; |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 103 | goto cq_err; |
Roi Dayan | c33b15f | 2014-09-02 17:08:41 +0300 | [diff] [blame] | 104 | } |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 105 | } |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 106 | |
Doug Ledford | 882f3b3 | 2015-12-22 17:03:15 -0500 | [diff] [blame] | 107 | INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev, |
| 108 | iser_event_handler); |
Or Gerlitz | 2110f9b | 2010-05-05 17:30:10 +0300 | [diff] [blame] | 109 | if (ib_register_event_handler(&device->event_handler)) |
Christoph Hellwig | 8e61212 | 2016-09-05 12:56:18 +0200 | [diff] [blame] | 110 | goto cq_err; |
Or Gerlitz | 2110f9b | 2010-05-05 17:30:10 +0300 | [diff] [blame] | 111 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 112 | return 0; |
| 113 | |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 114 | cq_err: |
Sagi Grimberg | bf17554 | 2014-10-01 14:02:07 +0300 | [diff] [blame] | 115 | for (i = 0; i < device->comps_used; i++) { |
| 116 | struct iser_comp *comp = &device->comps[i]; |
| 117 | |
Sagi Grimberg | 6aabfa7 | 2014-10-01 14:02:09 +0300 | [diff] [blame] | 118 | if (comp->cq) |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 119 | ib_free_cq(comp->cq); |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 120 | } |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 121 | ib_dealloc_pd(device->pd); |
| 122 | pd_err: |
Sagi Grimberg | da64bdb | 2014-12-07 16:10:03 +0200 | [diff] [blame] | 123 | kfree(device->comps); |
| 124 | comps_err: |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 125 | iser_err("failed to allocate an IB resource\n"); |
| 126 | return -1; |
| 127 | } |
| 128 | |
| 129 | /** |
Oliver Pinter | 38dc732 | 2008-01-25 14:15:32 -0800 | [diff] [blame] | 130 | * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR, |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 131 | * CQ and PD created with the device associated with the adapator. |
| 132 | */ |
| 133 | static void iser_free_device_ib_res(struct iser_device *device) |
| 134 | { |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 135 | int i; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 136 | |
Sagi Grimberg | bf17554 | 2014-10-01 14:02:07 +0300 | [diff] [blame] | 137 | for (i = 0; i < device->comps_used; i++) { |
| 138 | struct iser_comp *comp = &device->comps[i]; |
| 139 | |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 140 | ib_free_cq(comp->cq); |
Sagi Grimberg | 6aabfa7 | 2014-10-01 14:02:09 +0300 | [diff] [blame] | 141 | comp->cq = NULL; |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 142 | } |
| 143 | |
Or Gerlitz | 2110f9b | 2010-05-05 17:30:10 +0300 | [diff] [blame] | 144 | (void)ib_unregister_event_handler(&device->event_handler); |
Jason Gunthorpe | 7dd7864 | 2015-08-05 14:34:31 -0600 | [diff] [blame] | 145 | ib_dealloc_pd(device->pd); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 146 | |
Sagi Grimberg | da64bdb | 2014-12-07 16:10:03 +0200 | [diff] [blame] | 147 | kfree(device->comps); |
| 148 | device->comps = NULL; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 149 | device->pd = NULL; |
| 150 | } |
| 151 | |
| 152 | /** |
Sagi Grimberg | 48afbff | 2015-08-06 18:32:56 +0300 | [diff] [blame] | 153 | * iser_alloc_fmr_pool - Creates FMR pool and page_vector |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 154 | * |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 155 | * returns 0 on success, or errno code on failure |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 156 | */ |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 157 | int iser_alloc_fmr_pool(struct ib_conn *ib_conn, |
| 158 | unsigned cmds_max, |
| 159 | unsigned int size) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 160 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 161 | struct iser_device *device = ib_conn->device; |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 162 | struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; |
| 163 | struct iser_page_vec *page_vec; |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 164 | struct iser_fr_desc *desc; |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 165 | struct ib_fmr_pool *fmr_pool; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 166 | struct ib_fmr_pool_param params; |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 167 | int ret; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 168 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 169 | INIT_LIST_HEAD(&fr_pool->list); |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 170 | spin_lock_init(&fr_pool->lock); |
| 171 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 172 | desc = kzalloc(sizeof(*desc), GFP_KERNEL); |
| 173 | if (!desc) |
| 174 | return -ENOMEM; |
| 175 | |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 176 | page_vec = kmalloc(sizeof(*page_vec) + (sizeof(u64) * size), |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 177 | GFP_KERNEL); |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 178 | if (!page_vec) { |
| 179 | ret = -ENOMEM; |
| 180 | goto err_frpl; |
| 181 | } |
Dan Carpenter | 9fda1ac | 2010-05-06 16:22:21 +0300 | [diff] [blame] | 182 | |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 183 | page_vec->pages = (u64 *)(page_vec + 1); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 184 | |
Erez Zilber | 8dfa087 | 2006-09-11 12:22:30 +0300 | [diff] [blame] | 185 | params.page_shift = SHIFT_4K; |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 186 | params.max_pages_per_fmr = size; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 187 | /* make the pool size twice the max number of SCSI commands * |
| 188 | * the ML is expected to queue, watermark for unmap at 50% */ |
Shlomo Pongratz | b7f0451 | 2013-07-28 12:35:38 +0300 | [diff] [blame] | 189 | params.pool_size = cmds_max * 2; |
| 190 | params.dirty_watermark = cmds_max; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 191 | params.cache = 0; |
| 192 | params.flush_function = NULL; |
| 193 | params.access = (IB_ACCESS_LOCAL_WRITE | |
| 194 | IB_ACCESS_REMOTE_WRITE | |
| 195 | IB_ACCESS_REMOTE_READ); |
| 196 | |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 197 | fmr_pool = ib_create_fmr_pool(device->pd, ¶ms); |
| 198 | if (IS_ERR(fmr_pool)) { |
| 199 | ret = PTR_ERR(fmr_pool); |
Sagi Grimberg | 8c18ed0 | 2015-08-06 18:32:55 +0300 | [diff] [blame] | 200 | iser_err("FMR allocation failed, err %d\n", ret); |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 201 | goto err_fmr; |
Sagi Grimberg | 8c18ed0 | 2015-08-06 18:32:55 +0300 | [diff] [blame] | 202 | } |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 203 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 204 | desc->rsc.page_vec = page_vec; |
| 205 | desc->rsc.fmr_pool = fmr_pool; |
| 206 | list_add(&desc->list, &fr_pool->list); |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 207 | |
Sagi Grimberg | 8c18ed0 | 2015-08-06 18:32:55 +0300 | [diff] [blame] | 208 | return 0; |
| 209 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 210 | err_fmr: |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 211 | kfree(page_vec); |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 212 | err_frpl: |
| 213 | kfree(desc); |
| 214 | |
Sagi Grimberg | 8c18ed0 | 2015-08-06 18:32:55 +0300 | [diff] [blame] | 215 | return ret; |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 216 | } |
| 217 | |
| 218 | /** |
| 219 | * iser_free_fmr_pool - releases the FMR pool and page vec |
| 220 | */ |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 221 | void iser_free_fmr_pool(struct ib_conn *ib_conn) |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 222 | { |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 223 | struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 224 | struct iser_fr_desc *desc; |
| 225 | |
| 226 | desc = list_first_entry(&fr_pool->list, |
| 227 | struct iser_fr_desc, list); |
| 228 | list_del(&desc->list); |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 229 | |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 230 | iser_info("freeing conn %p fmr pool %p\n", |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 231 | ib_conn, desc->rsc.fmr_pool); |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 232 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 233 | ib_destroy_fmr_pool(desc->rsc.fmr_pool); |
| 234 | kfree(desc->rsc.page_vec); |
| 235 | kfree(desc); |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 236 | } |
| 237 | |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 238 | static int |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 239 | iser_alloc_reg_res(struct iser_device *device, |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 240 | struct ib_pd *pd, |
| 241 | struct iser_reg_resources *res, |
| 242 | unsigned int size) |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 243 | { |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 244 | struct ib_device *ib_dev = device->ib_device; |
| 245 | enum ib_mr_type mr_type; |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 246 | int ret; |
| 247 | |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 248 | if (ib_dev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG) |
| 249 | mr_type = IB_MR_TYPE_SG_GAPS; |
| 250 | else |
| 251 | mr_type = IB_MR_TYPE_MEM_REG; |
| 252 | |
| 253 | res->mr = ib_alloc_mr(pd, mr_type, size); |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 254 | if (IS_ERR(res->mr)) { |
| 255 | ret = PTR_ERR(res->mr); |
| 256 | iser_err("Failed to allocate ib_fast_reg_mr err=%d\n", ret); |
Sagi Grimberg | 3940588 | 2015-10-13 19:11:33 +0300 | [diff] [blame] | 257 | return ret; |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 258 | } |
Jenny Derzhavetz | 1caa70d8 | 2015-12-09 14:12:02 +0200 | [diff] [blame] | 259 | res->mr_valid = 0; |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 260 | |
| 261 | return 0; |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 262 | } |
| 263 | |
| 264 | static void |
| 265 | iser_free_reg_res(struct iser_reg_resources *rsc) |
| 266 | { |
| 267 | ib_dereg_mr(rsc->mr); |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 268 | } |
| 269 | |
| 270 | static int |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 271 | iser_alloc_pi_ctx(struct iser_device *device, |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 272 | struct ib_pd *pd, |
| 273 | struct iser_fr_desc *desc, |
| 274 | unsigned int size) |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 275 | { |
| 276 | struct iser_pi_context *pi_ctx = NULL; |
Sagi Grimberg | 9bee178 | 2015-07-30 10:32:35 +0300 | [diff] [blame] | 277 | int ret; |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 278 | |
| 279 | desc->pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); |
| 280 | if (!desc->pi_ctx) |
| 281 | return -ENOMEM; |
| 282 | |
| 283 | pi_ctx = desc->pi_ctx; |
| 284 | |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 285 | ret = iser_alloc_reg_res(device, pd, &pi_ctx->rsc, size); |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 286 | if (ret) { |
| 287 | iser_err("failed to allocate reg_resources\n"); |
| 288 | goto alloc_reg_res_err; |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 289 | } |
| 290 | |
Sagi Grimberg | 9bee178 | 2015-07-30 10:32:35 +0300 | [diff] [blame] | 291 | pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2); |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 292 | if (IS_ERR(pi_ctx->sig_mr)) { |
| 293 | ret = PTR_ERR(pi_ctx->sig_mr); |
| 294 | goto sig_mr_failure; |
| 295 | } |
Jenny Derzhavetz | 1caa70d8 | 2015-12-09 14:12:02 +0200 | [diff] [blame] | 296 | pi_ctx->sig_mr_valid = 0; |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 297 | desc->pi_ctx->sig_protected = 0; |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 298 | |
| 299 | return 0; |
| 300 | |
| 301 | sig_mr_failure: |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 302 | iser_free_reg_res(&pi_ctx->rsc); |
| 303 | alloc_reg_res_err: |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 304 | kfree(desc->pi_ctx); |
| 305 | |
| 306 | return ret; |
| 307 | } |
| 308 | |
| 309 | static void |
| 310 | iser_free_pi_ctx(struct iser_pi_context *pi_ctx) |
| 311 | { |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 312 | iser_free_reg_res(&pi_ctx->rsc); |
Sagi Grimberg | 8b91ffc | 2015-07-30 10:32:34 +0300 | [diff] [blame] | 313 | ib_dereg_mr(pi_ctx->sig_mr); |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 314 | kfree(pi_ctx); |
| 315 | } |
| 316 | |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 317 | static struct iser_fr_desc * |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 318 | iser_create_fastreg_desc(struct iser_device *device, |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 319 | struct ib_pd *pd, |
| 320 | bool pi_enable, |
| 321 | unsigned int size) |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 322 | { |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 323 | struct iser_fr_desc *desc; |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 324 | int ret; |
| 325 | |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 326 | desc = kzalloc(sizeof(*desc), GFP_KERNEL); |
| 327 | if (!desc) |
| 328 | return ERR_PTR(-ENOMEM); |
| 329 | |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 330 | ret = iser_alloc_reg_res(device, pd, &desc->rsc, size); |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 331 | if (ret) |
| 332 | goto reg_res_alloc_failure; |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 333 | |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 334 | if (pi_enable) { |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 335 | ret = iser_alloc_pi_ctx(device, pd, desc, size); |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 336 | if (ret) |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 337 | goto pi_ctx_alloc_failure; |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 338 | } |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 339 | |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 340 | return desc; |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 341 | |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 342 | pi_ctx_alloc_failure: |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 343 | iser_free_reg_res(&desc->rsc); |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 344 | reg_res_alloc_failure: |
| 345 | kfree(desc); |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 346 | |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 347 | return ERR_PTR(ret); |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 348 | } |
| 349 | |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 350 | /** |
Sagi Grimberg | 48afbff | 2015-08-06 18:32:56 +0300 | [diff] [blame] | 351 | * iser_alloc_fastreg_pool - Creates pool of fast_reg descriptors |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 352 | * for fast registration work requests. |
| 353 | * returns 0 on success, or errno code on failure |
| 354 | */ |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 355 | int iser_alloc_fastreg_pool(struct ib_conn *ib_conn, |
| 356 | unsigned cmds_max, |
| 357 | unsigned int size) |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 358 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 359 | struct iser_device *device = ib_conn->device; |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 360 | struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; |
Sagi Grimberg | 5190cc2 | 2015-08-06 18:32:54 +0300 | [diff] [blame] | 361 | struct iser_fr_desc *desc; |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 362 | int i, ret; |
| 363 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 364 | INIT_LIST_HEAD(&fr_pool->list); |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 365 | spin_lock_init(&fr_pool->lock); |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 366 | fr_pool->size = 0; |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 367 | for (i = 0; i < cmds_max; i++) { |
Sagi Grimberg | 318d311 | 2016-02-29 19:07:34 +0200 | [diff] [blame] | 368 | desc = iser_create_fastreg_desc(device, device->pd, |
Sagi Grimberg | f8db651 | 2015-08-06 18:33:03 +0300 | [diff] [blame] | 369 | ib_conn->pi_support, size); |
Sagi Grimberg | eb6ea8c3 | 2015-08-06 18:32:57 +0300 | [diff] [blame] | 370 | if (IS_ERR(desc)) { |
| 371 | ret = PTR_ERR(desc); |
Sagi Grimberg | 310b347 | 2014-03-05 19:43:41 +0200 | [diff] [blame] | 372 | goto err; |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 373 | } |
| 374 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 375 | list_add_tail(&desc->list, &fr_pool->list); |
| 376 | fr_pool->size++; |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 377 | } |
| 378 | |
| 379 | return 0; |
Roi Dayan | 27ae2d1 | 2013-08-19 16:41:51 +0300 | [diff] [blame] | 380 | |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 381 | err: |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 382 | iser_free_fastreg_pool(ib_conn); |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 383 | return ret; |
| 384 | } |
| 385 | |
| 386 | /** |
Sagi Grimberg | 7306b8f | 2014-03-05 19:43:39 +0200 | [diff] [blame] | 387 | * iser_free_fastreg_pool - releases the pool of fast_reg descriptors |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 388 | */ |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 389 | void iser_free_fastreg_pool(struct ib_conn *ib_conn) |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 390 | { |
Sagi Grimberg | 385ad87 | 2015-08-06 18:32:58 +0300 | [diff] [blame] | 391 | struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; |
Sagi Grimberg | 5190cc2 | 2015-08-06 18:32:54 +0300 | [diff] [blame] | 392 | struct iser_fr_desc *desc, *tmp; |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 393 | int i = 0; |
| 394 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 395 | if (list_empty(&fr_pool->list)) |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 396 | return; |
| 397 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 398 | iser_info("freeing conn %p fr pool\n", ib_conn); |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 399 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 400 | list_for_each_entry_safe(desc, tmp, &fr_pool->list, list) { |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 401 | list_del(&desc->list); |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 402 | iser_free_reg_res(&desc->rsc); |
Sagi Grimberg | 4dec2a2 | 2015-04-14 18:08:22 +0300 | [diff] [blame] | 403 | if (desc->pi_ctx) |
| 404 | iser_free_pi_ctx(desc->pi_ctx); |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 405 | kfree(desc); |
| 406 | ++i; |
| 407 | } |
| 408 | |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 409 | if (i < fr_pool->size) |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 410 | iser_warn("pool still has %d regions registered\n", |
Adir Lev | 2b3bf95 | 2015-08-06 18:32:59 +0300 | [diff] [blame] | 411 | fr_pool->size - i); |
Sagi Grimberg | 5587856 | 2013-07-28 12:35:42 +0300 | [diff] [blame] | 412 | } |
| 413 | |
| 414 | /** |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 415 | * iser_create_ib_conn_res - Queue-Pair (QP) |
| 416 | * |
| 417 | * returns 0 on success, -1 on failure |
| 418 | */ |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 419 | static int iser_create_ib_conn_res(struct ib_conn *ib_conn) |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 420 | { |
Sagi Grimberg | 7edc5a9 | 2015-11-04 10:50:32 +0200 | [diff] [blame] | 421 | struct iser_conn *iser_conn = to_iser_conn(ib_conn); |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 422 | struct iser_device *device; |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 423 | struct ib_device *ib_dev; |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 424 | struct ib_qp_init_attr init_attr; |
| 425 | int ret = -ENOMEM; |
| 426 | int index, min_index = 0; |
| 427 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 428 | BUG_ON(ib_conn->device == NULL); |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 429 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 430 | device = ib_conn->device; |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 431 | ib_dev = device->ib_device; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 432 | |
| 433 | memset(&init_attr, 0, sizeof init_attr); |
| 434 | |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 435 | mutex_lock(&ig.connlist_mutex); |
| 436 | /* select the CQ with the minimal number of usages */ |
Sagi Grimberg | bf17554 | 2014-10-01 14:02:07 +0300 | [diff] [blame] | 437 | for (index = 0; index < device->comps_used; index++) { |
| 438 | if (device->comps[index].active_qps < |
| 439 | device->comps[min_index].active_qps) |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 440 | min_index = index; |
Sagi Grimberg | bf17554 | 2014-10-01 14:02:07 +0300 | [diff] [blame] | 441 | } |
| 442 | ib_conn->comp = &device->comps[min_index]; |
| 443 | ib_conn->comp->active_qps++; |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 444 | mutex_unlock(&ig.connlist_mutex); |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 445 | iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn); |
Alex Tabachnik | 5a33a66 | 2012-09-23 15:17:44 +0000 | [diff] [blame] | 446 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 447 | init_attr.event_handler = iser_qp_event_callback; |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 448 | init_attr.qp_context = (void *)ib_conn; |
Sagi Grimberg | 6aabfa7 | 2014-10-01 14:02:09 +0300 | [diff] [blame] | 449 | init_attr.send_cq = ib_conn->comp->cq; |
| 450 | init_attr.recv_cq = ib_conn->comp->cq; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 451 | init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; |
Or Gerlitz | f19624a | 2010-02-08 13:19:56 +0000 | [diff] [blame] | 452 | init_attr.cap.max_send_sge = 2; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 453 | init_attr.cap.max_recv_sge = 1; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 454 | init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; |
| 455 | init_attr.qp_type = IB_QPT_RC; |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 456 | if (ib_conn->pi_support) { |
Sagi Grimberg | ff3dd52 | 2014-10-01 14:02:10 +0300 | [diff] [blame] | 457 | init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1; |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 458 | init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 459 | iser_conn->max_cmds = |
| 460 | ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS); |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 461 | } else { |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 462 | if (ib_dev->attrs.max_qp_wr > ISER_QP_MAX_REQ_DTOS) { |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 463 | init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS + 1; |
| 464 | iser_conn->max_cmds = |
| 465 | ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS); |
| 466 | } else { |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 467 | init_attr.cap.max_send_wr = ib_dev->attrs.max_qp_wr; |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 468 | iser_conn->max_cmds = |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 469 | ISER_GET_MAX_XMIT_CMDS(ib_dev->attrs.max_qp_wr); |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 470 | iser_dbg("device %s supports max_send_wr %d\n", |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 471 | device->ib_device->name, ib_dev->attrs.max_qp_wr); |
Minh Tran | f4641ef | 2014-12-07 16:09:52 +0200 | [diff] [blame] | 472 | } |
Alex Tabachnik | 6b5a8fb | 2014-03-05 19:43:47 +0200 | [diff] [blame] | 473 | } |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 474 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 475 | ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 476 | if (ret) |
Dan Carpenter | 9fda1ac | 2010-05-06 16:22:21 +0300 | [diff] [blame] | 477 | goto out_err; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 478 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 479 | ib_conn->qp = ib_conn->cma_id->qp; |
Shlomo Pongratz | 986db0d | 2013-07-28 12:35:37 +0300 | [diff] [blame] | 480 | iser_info("setting conn %p cma_id %p qp %p\n", |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 481 | ib_conn, ib_conn->cma_id, |
| 482 | ib_conn->cma_id->qp); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 483 | return ret; |
| 484 | |
Dan Carpenter | 9fda1ac | 2010-05-06 16:22:21 +0300 | [diff] [blame] | 485 | out_err: |
Sagi Grimberg | 93acb7b | 2014-12-07 16:09:55 +0200 | [diff] [blame] | 486 | mutex_lock(&ig.connlist_mutex); |
| 487 | ib_conn->comp->active_qps--; |
| 488 | mutex_unlock(&ig.connlist_mutex); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 489 | iser_err("unable to alloc mem or create resource, err %d\n", ret); |
Sagi Grimberg | 93acb7b | 2014-12-07 16:09:55 +0200 | [diff] [blame] | 490 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 491 | return ret; |
| 492 | } |
| 493 | |
| 494 | /** |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 495 | * based on the resolved device node GUID see if there already allocated |
| 496 | * device for this device. If there's no such, create one. |
| 497 | */ |
| 498 | static |
| 499 | struct iser_device *iser_device_find_by_ib_device(struct rdma_cm_id *cma_id) |
| 500 | { |
Arne Redlich | 9a37827 | 2008-03-04 14:07:22 +0200 | [diff] [blame] | 501 | struct iser_device *device; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 502 | |
| 503 | mutex_lock(&ig.device_list_mutex); |
| 504 | |
Arne Redlich | 9a37827 | 2008-03-04 14:07:22 +0200 | [diff] [blame] | 505 | list_for_each_entry(device, &ig.device_list, ig_list) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 506 | /* find if there's a match using the node GUID */ |
| 507 | if (device->ib_device->node_guid == cma_id->device->node_guid) |
Arne Redlich | d33ed42 | 2008-03-04 14:11:54 +0200 | [diff] [blame] | 508 | goto inc_refcnt; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 509 | |
Arne Redlich | 9a37827 | 2008-03-04 14:07:22 +0200 | [diff] [blame] | 510 | device = kzalloc(sizeof *device, GFP_KERNEL); |
| 511 | if (device == NULL) |
| 512 | goto out; |
| 513 | |
| 514 | /* assign this device to the device */ |
| 515 | device->ib_device = cma_id->device; |
| 516 | /* init the device and link it into ig device list */ |
| 517 | if (iser_create_device_ib_res(device)) { |
| 518 | kfree(device); |
| 519 | device = NULL; |
| 520 | goto out; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 521 | } |
Arne Redlich | 9a37827 | 2008-03-04 14:07:22 +0200 | [diff] [blame] | 522 | list_add(&device->ig_list, &ig.device_list); |
| 523 | |
Arne Redlich | d33ed42 | 2008-03-04 14:11:54 +0200 | [diff] [blame] | 524 | inc_refcnt: |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 525 | device->refcount++; |
Arne Redlich | d33ed42 | 2008-03-04 14:11:54 +0200 | [diff] [blame] | 526 | out: |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 527 | mutex_unlock(&ig.device_list_mutex); |
| 528 | return device; |
| 529 | } |
| 530 | |
| 531 | /* if there's no demand for this device, release it */ |
| 532 | static void iser_device_try_release(struct iser_device *device) |
| 533 | { |
| 534 | mutex_lock(&ig.device_list_mutex); |
| 535 | device->refcount--; |
Roi Dayan | 4f36388 | 2013-05-01 13:25:25 +0000 | [diff] [blame] | 536 | iser_info("device %p refcount %d\n", device, device->refcount); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 537 | if (!device->refcount) { |
| 538 | iser_free_device_ib_res(device); |
| 539 | list_del(&device->ig_list); |
| 540 | kfree(device); |
| 541 | } |
| 542 | mutex_unlock(&ig.device_list_mutex); |
| 543 | } |
| 544 | |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 545 | /** |
| 546 | * Called with state mutex held |
| 547 | **/ |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 548 | static int iser_conn_state_comp_exch(struct iser_conn *iser_conn, |
| 549 | enum iser_conn_state comp, |
| 550 | enum iser_conn_state exch) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 551 | { |
| 552 | int ret; |
| 553 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 554 | ret = (iser_conn->state == comp); |
| 555 | if (ret) |
| 556 | iser_conn->state = exch; |
| 557 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 558 | return ret; |
| 559 | } |
| 560 | |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 561 | void iser_release_work(struct work_struct *work) |
| 562 | { |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 563 | struct iser_conn *iser_conn; |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 564 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 565 | iser_conn = container_of(work, struct iser_conn, release_work); |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 566 | |
Sagi Grimberg | c107a6c | 2014-10-01 14:02:02 +0300 | [diff] [blame] | 567 | /* Wait for conn_stop to complete */ |
| 568 | wait_for_completion(&iser_conn->stop_completion); |
| 569 | /* Wait for IB resouces cleanup to complete */ |
| 570 | wait_for_completion(&iser_conn->ib_completion); |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 571 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 572 | mutex_lock(&iser_conn->state_mutex); |
| 573 | iser_conn->state = ISER_CONN_DOWN; |
| 574 | mutex_unlock(&iser_conn->state_mutex); |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 575 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 576 | iser_conn_release(iser_conn); |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 577 | } |
| 578 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 579 | /** |
Sagi Grimberg | 96f1519 | 2014-10-01 14:02:00 +0300 | [diff] [blame] | 580 | * iser_free_ib_conn_res - release IB related resources |
| 581 | * @iser_conn: iser connection struct |
Sagi Grimberg | 6606e6a | 2015-01-18 16:51:06 +0200 | [diff] [blame] | 582 | * @destroy: indicator if we need to try to release the |
| 583 | * iser device and memory regoins pool (only iscsi |
| 584 | * shutdown and DEVICE_REMOVAL will use this). |
Sagi Grimberg | 96f1519 | 2014-10-01 14:02:00 +0300 | [diff] [blame] | 585 | * |
| 586 | * This routine is called with the iser state mutex held |
| 587 | * so the cm_id removal is out of here. It is Safe to |
| 588 | * be invoked multiple times. |
| 589 | */ |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 590 | static void iser_free_ib_conn_res(struct iser_conn *iser_conn, |
Sagi Grimberg | 6606e6a | 2015-01-18 16:51:06 +0200 | [diff] [blame] | 591 | bool destroy) |
Sagi Grimberg | 96f1519 | 2014-10-01 14:02:00 +0300 | [diff] [blame] | 592 | { |
| 593 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
| 594 | struct iser_device *device = ib_conn->device; |
| 595 | |
| 596 | iser_info("freeing conn %p cma_id %p qp %p\n", |
| 597 | iser_conn, ib_conn->cma_id, ib_conn->qp); |
| 598 | |
Sagi Grimberg | 96f1519 | 2014-10-01 14:02:00 +0300 | [diff] [blame] | 599 | if (ib_conn->qp != NULL) { |
Sagi Grimberg | bf17554 | 2014-10-01 14:02:07 +0300 | [diff] [blame] | 600 | ib_conn->comp->active_qps--; |
Sagi Grimberg | 96f1519 | 2014-10-01 14:02:00 +0300 | [diff] [blame] | 601 | rdma_destroy_qp(ib_conn->cma_id); |
| 602 | ib_conn->qp = NULL; |
| 603 | } |
| 604 | |
Sagi Grimberg | 6606e6a | 2015-01-18 16:51:06 +0200 | [diff] [blame] | 605 | if (destroy) { |
| 606 | if (iser_conn->rx_descs) |
| 607 | iser_free_rx_descriptors(iser_conn); |
| 608 | |
| 609 | if (device != NULL) { |
| 610 | iser_device_try_release(device); |
| 611 | ib_conn->device = NULL; |
| 612 | } |
Sagi Grimberg | 96f1519 | 2014-10-01 14:02:00 +0300 | [diff] [blame] | 613 | } |
| 614 | } |
| 615 | |
| 616 | /** |
Roland Dreier | 41179e2 | 2007-07-17 18:37:42 -0700 | [diff] [blame] | 617 | * Frees all conn objects and deallocs conn descriptor |
| 618 | */ |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 619 | void iser_conn_release(struct iser_conn *iser_conn) |
Roland Dreier | 41179e2 | 2007-07-17 18:37:42 -0700 | [diff] [blame] | 620 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 621 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
Roland Dreier | 41179e2 | 2007-07-17 18:37:42 -0700 | [diff] [blame] | 622 | |
Roland Dreier | 41179e2 | 2007-07-17 18:37:42 -0700 | [diff] [blame] | 623 | mutex_lock(&ig.connlist_mutex); |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 624 | list_del(&iser_conn->conn_list); |
Roland Dreier | 41179e2 | 2007-07-17 18:37:42 -0700 | [diff] [blame] | 625 | mutex_unlock(&ig.connlist_mutex); |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 626 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 627 | mutex_lock(&iser_conn->state_mutex); |
Ariel Nahum | 9a3119e | 2015-01-18 16:51:07 +0200 | [diff] [blame] | 628 | /* In case we endup here without ep_disconnect being invoked. */ |
Ariel Nahum | 5426b17 | 2014-12-07 16:09:54 +0200 | [diff] [blame] | 629 | if (iser_conn->state != ISER_CONN_DOWN) { |
Ariel Nahum | aea8f4d | 2014-10-01 14:02:06 +0300 | [diff] [blame] | 630 | iser_warn("iser conn %p state %d, expected state down.\n", |
| 631 | iser_conn, iser_conn->state); |
Ariel Nahum | 9a3119e | 2015-01-18 16:51:07 +0200 | [diff] [blame] | 632 | iscsi_destroy_endpoint(iser_conn->ep); |
Ariel Nahum | 5426b17 | 2014-12-07 16:09:54 +0200 | [diff] [blame] | 633 | iser_conn->state = ISER_CONN_DOWN; |
| 634 | } |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 635 | /* |
| 636 | * In case we never got to bind stage, we still need to |
| 637 | * release IB resources (which is safe to call more than once). |
| 638 | */ |
| 639 | iser_free_ib_conn_res(iser_conn, true); |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 640 | mutex_unlock(&iser_conn->state_mutex); |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 641 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 642 | if (ib_conn->cma_id != NULL) { |
| 643 | rdma_destroy_id(ib_conn->cma_id); |
| 644 | ib_conn->cma_id = NULL; |
Roi Dayan | 5b61ff4 | 2013-05-08 12:21:17 +0000 | [diff] [blame] | 645 | } |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 646 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 647 | kfree(iser_conn); |
Roland Dreier | 41179e2 | 2007-07-17 18:37:42 -0700 | [diff] [blame] | 648 | } |
| 649 | |
| 650 | /** |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 651 | * triggers start of the disconnect procedures and wait for them to be done |
| 652 | * Called with state mutex held |
| 653 | */ |
| 654 | int iser_conn_terminate(struct iser_conn *iser_conn) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 655 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 656 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 657 | int err = 0; |
| 658 | |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 659 | /* terminate the iser conn only if the conn state is UP */ |
| 660 | if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP, |
| 661 | ISER_CONN_TERMINATING)) |
| 662 | return 0; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 663 | |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 664 | iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state); |
| 665 | |
| 666 | /* suspend queuing of new iscsi commands */ |
| 667 | if (iser_conn->iscsi_conn) |
| 668 | iscsi_suspend_queue(iser_conn->iscsi_conn); |
| 669 | |
| 670 | /* |
| 671 | * In case we didn't already clean up the cma_id (peer initiated |
| 672 | * a disconnection), we need to Cause the CMA to change the QP |
| 673 | * state to ERROR. |
| 674 | */ |
| 675 | if (ib_conn->cma_id) { |
| 676 | err = rdma_disconnect(ib_conn->cma_id); |
| 677 | if (err) |
| 678 | iser_err("Failed to disconnect, conn: 0x%p err %d\n", |
| 679 | iser_conn, err); |
| 680 | |
Steve Wise | 4c8ba94 | 2016-02-17 08:17:12 -0800 | [diff] [blame] | 681 | /* block until all flush errors are consumed */ |
| 682 | ib_drain_sq(ib_conn->qp); |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 683 | } |
| 684 | |
| 685 | return 1; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 686 | } |
| 687 | |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 688 | /** |
| 689 | * Called with state mutex held |
| 690 | **/ |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 691 | static void iser_connect_error(struct rdma_cm_id *cma_id) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 692 | { |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 693 | struct iser_conn *iser_conn; |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 694 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 695 | iser_conn = (struct iser_conn *)cma_id->context; |
Sagi Grimberg | c4de466 | 2015-04-14 18:08:11 +0300 | [diff] [blame] | 696 | iser_conn->state = ISER_CONN_TERMINATING; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 697 | } |
| 698 | |
Sagi Grimberg | df749cd | 2015-08-06 18:33:04 +0300 | [diff] [blame] | 699 | static void |
| 700 | iser_calc_scsi_params(struct iser_conn *iser_conn, |
| 701 | unsigned int max_sectors) |
| 702 | { |
| 703 | struct iser_device *device = iser_conn->ib_conn.device; |
| 704 | unsigned short sg_tablesize, sup_sg_tablesize; |
| 705 | |
| 706 | sg_tablesize = DIV_ROUND_UP(max_sectors * 512, SIZE_4K); |
| 707 | sup_sg_tablesize = min_t(unsigned, ISCSI_ISER_MAX_SG_TABLESIZE, |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 708 | device->ib_device->attrs.max_fast_reg_page_list_len); |
Sagi Grimberg | df749cd | 2015-08-06 18:33:04 +0300 | [diff] [blame] | 709 | |
| 710 | if (sg_tablesize > sup_sg_tablesize) { |
| 711 | sg_tablesize = sup_sg_tablesize; |
| 712 | iser_conn->scsi_max_sectors = sg_tablesize * SIZE_4K / 512; |
| 713 | } else { |
| 714 | iser_conn->scsi_max_sectors = max_sectors; |
| 715 | } |
| 716 | |
| 717 | iser_conn->scsi_sg_tablesize = sg_tablesize; |
| 718 | |
| 719 | iser_dbg("iser_conn %p, sg_tablesize %u, max_sectors %u\n", |
| 720 | iser_conn, iser_conn->scsi_sg_tablesize, |
| 721 | iser_conn->scsi_max_sectors); |
| 722 | } |
| 723 | |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 724 | /** |
| 725 | * Called with state mutex held |
| 726 | **/ |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 727 | static void iser_addr_handler(struct rdma_cm_id *cma_id) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 728 | { |
| 729 | struct iser_device *device; |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 730 | struct iser_conn *iser_conn; |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 731 | struct ib_conn *ib_conn; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 732 | int ret; |
| 733 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 734 | iser_conn = (struct iser_conn *)cma_id->context; |
| 735 | if (iser_conn->state != ISER_CONN_PENDING) |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 736 | /* bailout */ |
| 737 | return; |
| 738 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 739 | ib_conn = &iser_conn->ib_conn; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 740 | device = iser_device_find_by_ib_device(cma_id); |
Arne Redlich | d33ed42 | 2008-03-04 14:11:54 +0200 | [diff] [blame] | 741 | if (!device) { |
| 742 | iser_err("device lookup/creation failed\n"); |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 743 | iser_connect_error(cma_id); |
| 744 | return; |
Arne Redlich | d33ed42 | 2008-03-04 14:11:54 +0200 | [diff] [blame] | 745 | } |
| 746 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 747 | ib_conn->device = device; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 748 | |
Alex Tabachnik | 7f73384 | 2014-03-05 19:43:46 +0200 | [diff] [blame] | 749 | /* connection T10-PI support */ |
| 750 | if (iser_pi_enable) { |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 751 | if (!(device->ib_device->attrs.device_cap_flags & |
Alex Tabachnik | 7f73384 | 2014-03-05 19:43:46 +0200 | [diff] [blame] | 752 | IB_DEVICE_SIGNATURE_HANDOVER)) { |
| 753 | iser_warn("T10-PI requested but not supported on %s, " |
| 754 | "continue without T10-PI\n", |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 755 | ib_conn->device->ib_device->name); |
| 756 | ib_conn->pi_support = false; |
Alex Tabachnik | 7f73384 | 2014-03-05 19:43:46 +0200 | [diff] [blame] | 757 | } else { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 758 | ib_conn->pi_support = true; |
Alex Tabachnik | 7f73384 | 2014-03-05 19:43:46 +0200 | [diff] [blame] | 759 | } |
| 760 | } |
| 761 | |
Sagi Grimberg | df749cd | 2015-08-06 18:33:04 +0300 | [diff] [blame] | 762 | iser_calc_scsi_params(iser_conn, iser_max_sectors); |
| 763 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 764 | ret = rdma_resolve_route(cma_id, 1000); |
| 765 | if (ret) { |
| 766 | iser_err("resolve route failed: %d\n", ret); |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 767 | iser_connect_error(cma_id); |
| 768 | return; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 769 | } |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 770 | } |
| 771 | |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 772 | /** |
| 773 | * Called with state mutex held |
| 774 | **/ |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 775 | static void iser_route_handler(struct rdma_cm_id *cma_id) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 776 | { |
| 777 | struct rdma_conn_param conn_param; |
| 778 | int ret; |
Or Gerlitz | 8d8399d | 2013-05-01 13:25:27 +0000 | [diff] [blame] | 779 | struct iser_cm_hdr req_hdr; |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 780 | struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 781 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
| 782 | struct iser_device *device = ib_conn->device; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 783 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 784 | if (iser_conn->state != ISER_CONN_PENDING) |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 785 | /* bailout */ |
| 786 | return; |
| 787 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 788 | ret = iser_create_ib_conn_res(ib_conn); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 789 | if (ret) |
| 790 | goto failure; |
| 791 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 792 | memset(&conn_param, 0, sizeof conn_param); |
Or Gerlitz | 4a061b2 | 2015-12-18 10:59:46 +0200 | [diff] [blame] | 793 | conn_param.responder_resources = device->ib_device->attrs.max_qp_rd_atom; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 794 | conn_param.initiator_depth = 1; |
| 795 | conn_param.retry_count = 7; |
| 796 | conn_param.rnr_retry_count = 6; |
| 797 | |
Or Gerlitz | 8d8399d | 2013-05-01 13:25:27 +0000 | [diff] [blame] | 798 | memset(&req_hdr, 0, sizeof(req_hdr)); |
Jenny Derzhavetz | 59caaed | 2015-12-24 12:20:48 +0200 | [diff] [blame] | 799 | req_hdr.flags = ISER_ZBVA_NOT_SUP; |
| 800 | if (!device->remote_inv_sup) |
| 801 | req_hdr.flags |= ISER_SEND_W_INV_NOT_SUP; |
Sagi Grimberg | d3cf81f | 2015-12-09 14:12:03 +0200 | [diff] [blame] | 802 | conn_param.private_data = (void *)&req_hdr; |
| 803 | conn_param.private_data_len = sizeof(struct iser_cm_hdr); |
Or Gerlitz | 8d8399d | 2013-05-01 13:25:27 +0000 | [diff] [blame] | 804 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 805 | ret = rdma_connect(cma_id, &conn_param); |
| 806 | if (ret) { |
| 807 | iser_err("failure connecting: %d\n", ret); |
| 808 | goto failure; |
| 809 | } |
| 810 | |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 811 | return; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 812 | failure: |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 813 | iser_connect_error(cma_id); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 814 | } |
| 815 | |
Jenny Derzhavetz | 59caaed | 2015-12-24 12:20:48 +0200 | [diff] [blame] | 816 | static void iser_connected_handler(struct rdma_cm_id *cma_id, |
| 817 | const void *private_data) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 818 | { |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 819 | struct iser_conn *iser_conn; |
Or Gerlitz | 4f9208a | 2014-04-01 16:28:40 +0300 | [diff] [blame] | 820 | struct ib_qp_attr attr; |
| 821 | struct ib_qp_init_attr init_attr; |
| 822 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 823 | iser_conn = (struct iser_conn *)cma_id->context; |
| 824 | if (iser_conn->state != ISER_CONN_PENDING) |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 825 | /* bailout */ |
| 826 | return; |
| 827 | |
Or Gerlitz | 4f9208a | 2014-04-01 16:28:40 +0300 | [diff] [blame] | 828 | (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); |
| 829 | iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 830 | |
Jenny Derzhavetz | 59caaed | 2015-12-24 12:20:48 +0200 | [diff] [blame] | 831 | if (private_data) { |
| 832 | u8 flags = *(u8 *)private_data; |
| 833 | |
| 834 | iser_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP); |
| 835 | } |
| 836 | |
| 837 | iser_info("conn %p: negotiated %s invalidation\n", |
| 838 | iser_conn, iser_conn->snd_w_inv ? "remote" : "local"); |
| 839 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 840 | iser_conn->state = ISER_CONN_UP; |
| 841 | complete(&iser_conn->up_completion); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 842 | } |
| 843 | |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 844 | static void iser_disconnected_handler(struct rdma_cm_id *cma_id) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 845 | { |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 846 | struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 847 | |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 848 | if (iser_conn_terminate(iser_conn)) { |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 849 | if (iser_conn->iscsi_conn) |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 850 | iscsi_conn_failure(iser_conn->iscsi_conn, |
| 851 | ISCSI_ERR_CONN_FAILED); |
Roi Dayan | 7d9eacf | 2014-02-04 16:54:54 +0200 | [diff] [blame] | 852 | else |
| 853 | iser_err("iscsi_iser connection isn't bound\n"); |
| 854 | } |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 855 | } |
| 856 | |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 857 | static void iser_cleanup_handler(struct rdma_cm_id *cma_id, |
Sagi Grimberg | 6606e6a | 2015-01-18 16:51:06 +0200 | [diff] [blame] | 858 | bool destroy) |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 859 | { |
| 860 | struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context; |
| 861 | |
| 862 | /* |
| 863 | * We are not guaranteed that we visited disconnected_handler |
| 864 | * by now, call it here to be safe that we handle CM drep |
| 865 | * and flush errors. |
| 866 | */ |
| 867 | iser_disconnected_handler(cma_id); |
Sagi Grimberg | 6606e6a | 2015-01-18 16:51:06 +0200 | [diff] [blame] | 868 | iser_free_ib_conn_res(iser_conn, destroy); |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 869 | complete(&iser_conn->ib_completion); |
| 870 | }; |
| 871 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 872 | static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) |
| 873 | { |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 874 | struct iser_conn *iser_conn; |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 875 | int ret = 0; |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 876 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 877 | iser_conn = (struct iser_conn *)cma_id->context; |
Sagi Grimberg | 871e00a | 2015-05-18 13:40:30 +0300 | [diff] [blame] | 878 | iser_info("%s (%d): status %d conn %p id %p\n", |
| 879 | rdma_event_msg(event->event), event->event, |
| 880 | event->status, cma_id->context, cma_id); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 881 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 882 | mutex_lock(&iser_conn->state_mutex); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 883 | switch (event->event) { |
| 884 | case RDMA_CM_EVENT_ADDR_RESOLVED: |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 885 | iser_addr_handler(cma_id); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 886 | break; |
| 887 | case RDMA_CM_EVENT_ROUTE_RESOLVED: |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 888 | iser_route_handler(cma_id); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 889 | break; |
| 890 | case RDMA_CM_EVENT_ESTABLISHED: |
Jenny Derzhavetz | 59caaed | 2015-12-24 12:20:48 +0200 | [diff] [blame] | 891 | iser_connected_handler(cma_id, event->param.conn.private_data); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 892 | break; |
Steve Wise | 97540bb | 2016-10-26 12:36:47 -0700 | [diff] [blame] | 893 | case RDMA_CM_EVENT_REJECTED: |
| 894 | iser_info("Connection rejected: %s\n", |
| 895 | rdma_reject_msg(cma_id, event->status)); |
| 896 | /* FALLTHROUGH */ |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 897 | case RDMA_CM_EVENT_ADDR_ERROR: |
| 898 | case RDMA_CM_EVENT_ROUTE_ERROR: |
| 899 | case RDMA_CM_EVENT_CONNECT_ERROR: |
| 900 | case RDMA_CM_EVENT_UNREACHABLE: |
Ariel Nahum | b73c3ad | 2014-05-22 11:00:18 +0300 | [diff] [blame] | 901 | iser_connect_error(cma_id); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 902 | break; |
| 903 | case RDMA_CM_EVENT_DISCONNECTED: |
Or Gerlitz | 2f5de15 | 2008-07-22 14:16:21 -0700 | [diff] [blame] | 904 | case RDMA_CM_EVENT_ADDR_CHANGE: |
Ariel Nahum | 5426b17 | 2014-12-07 16:09:54 +0200 | [diff] [blame] | 905 | case RDMA_CM_EVENT_TIMEWAIT_EXIT: |
| 906 | iser_cleanup_handler(cma_id, false); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 907 | break; |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 908 | case RDMA_CM_EVENT_DEVICE_REMOVAL: |
| 909 | /* |
| 910 | * we *must* destroy the device as we cannot rely |
| 911 | * on iscsid to be around to initiate error handling. |
Ariel Nahum | 5426b17 | 2014-12-07 16:09:54 +0200 | [diff] [blame] | 912 | * also if we are not in state DOWN implicitly destroy |
| 913 | * the cma_id. |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 914 | */ |
| 915 | iser_cleanup_handler(cma_id, true); |
Ariel Nahum | 5426b17 | 2014-12-07 16:09:54 +0200 | [diff] [blame] | 916 | if (iser_conn->state != ISER_CONN_DOWN) { |
| 917 | iser_conn->ib_conn.cma_id = NULL; |
| 918 | ret = 1; |
| 919 | } |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 920 | break; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 921 | default: |
Sagi Grimberg | 871e00a | 2015-05-18 13:40:30 +0300 | [diff] [blame] | 922 | iser_err("Unexpected RDMA CM event: %s (%d)\n", |
| 923 | rdma_event_msg(event->event), event->event); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 924 | break; |
| 925 | } |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 926 | mutex_unlock(&iser_conn->state_mutex); |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 927 | |
| 928 | return ret; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 929 | } |
| 930 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 931 | void iser_conn_init(struct iser_conn *iser_conn) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 932 | { |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 933 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
| 934 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 935 | iser_conn->state = ISER_CONN_INIT; |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 936 | init_completion(&iser_conn->stop_completion); |
Sagi Grimberg | c47a3c9 | 2014-10-01 14:02:01 +0300 | [diff] [blame] | 937 | init_completion(&iser_conn->ib_completion); |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 938 | init_completion(&iser_conn->up_completion); |
| 939 | INIT_LIST_HEAD(&iser_conn->conn_list); |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 940 | mutex_init(&iser_conn->state_mutex); |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 941 | |
| 942 | ib_conn->post_recv_buf_count = 0; |
| 943 | ib_conn->reg_cqe.done = iser_reg_comp; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 944 | } |
| 945 | |
| 946 | /** |
| 947 | * starts the process of connecting to the target |
Thadeu Lima de Souza Cascardo | 94e2bd6 | 2009-10-16 15:20:49 +0200 | [diff] [blame] | 948 | * sleeps until the connection is established or rejected |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 949 | */ |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 950 | int iser_connect(struct iser_conn *iser_conn, |
Roi Dayan | 96ed02d | 2014-07-31 13:27:44 +0300 | [diff] [blame] | 951 | struct sockaddr *src_addr, |
| 952 | struct sockaddr *dst_addr, |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 953 | int non_blocking) |
| 954 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 955 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 956 | int err = 0; |
| 957 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 958 | mutex_lock(&iser_conn->state_mutex); |
Ariel Nahum | 504130c | 2014-07-31 13:27:49 +0300 | [diff] [blame] | 959 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 960 | sprintf(iser_conn->name, "%pISp", dst_addr); |
Roi Dayan | 96ed02d | 2014-07-31 13:27:44 +0300 | [diff] [blame] | 961 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 962 | iser_info("connecting to: %s\n", iser_conn->name); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 963 | |
| 964 | /* the device is known only --after-- address resolution */ |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 965 | ib_conn->device = NULL; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 966 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 967 | iser_conn->state = ISER_CONN_PENDING; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 968 | |
Guy Shapiro | fa20105 | 2015-10-22 15:20:10 +0300 | [diff] [blame] | 969 | ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler, |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 970 | (void *)iser_conn, |
| 971 | RDMA_PS_TCP, IB_QPT_RC); |
| 972 | if (IS_ERR(ib_conn->cma_id)) { |
| 973 | err = PTR_ERR(ib_conn->cma_id); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 974 | iser_err("rdma_create_id failed: %d\n", err); |
| 975 | goto id_failure; |
| 976 | } |
| 977 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 978 | err = rdma_resolve_addr(ib_conn->cma_id, src_addr, dst_addr, 1000); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 979 | if (err) { |
| 980 | iser_err("rdma_resolve_addr failed: %d\n", err); |
| 981 | goto addr_failure; |
| 982 | } |
| 983 | |
| 984 | if (!non_blocking) { |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 985 | wait_for_completion_interruptible(&iser_conn->up_completion); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 986 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 987 | if (iser_conn->state != ISER_CONN_UP) { |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 988 | err = -EIO; |
| 989 | goto connect_failure; |
| 990 | } |
| 991 | } |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 992 | mutex_unlock(&iser_conn->state_mutex); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 993 | |
| 994 | mutex_lock(&ig.connlist_mutex); |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 995 | list_add(&iser_conn->conn_list, &ig.connlist); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 996 | mutex_unlock(&ig.connlist_mutex); |
| 997 | return 0; |
| 998 | |
| 999 | id_failure: |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1000 | ib_conn->cma_id = NULL; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1001 | addr_failure: |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1002 | iser_conn->state = ISER_CONN_DOWN; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1003 | connect_failure: |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1004 | mutex_unlock(&iser_conn->state_mutex); |
| 1005 | iser_conn_release(iser_conn); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1006 | return err; |
| 1007 | } |
| 1008 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1009 | int iser_post_recvl(struct iser_conn *iser_conn) |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1010 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1011 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
Sagi Grimberg | 0f512b3 | 2015-11-04 10:50:31 +0200 | [diff] [blame] | 1012 | struct iser_login_desc *desc = &iser_conn->login_desc; |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1013 | struct ib_recv_wr wr, *wr_failed; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1014 | int ib_ret; |
| 1015 | |
Sagi Grimberg | 0f512b3 | 2015-11-04 10:50:31 +0200 | [diff] [blame] | 1016 | desc->sge.addr = desc->rsp_dma; |
| 1017 | desc->sge.length = ISER_RX_LOGIN_SIZE; |
| 1018 | desc->sge.lkey = ib_conn->device->pd->local_dma_lkey; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1019 | |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1020 | desc->cqe.done = iser_login_rsp; |
| 1021 | wr.wr_cqe = &desc->cqe; |
| 1022 | wr.sg_list = &desc->sge; |
| 1023 | wr.num_sge = 1; |
| 1024 | wr.next = NULL; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1025 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1026 | ib_conn->post_recv_buf_count++; |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1027 | ib_ret = ib_post_recv(ib_conn->qp, &wr, &wr_failed); |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1028 | if (ib_ret) { |
| 1029 | iser_err("ib_post_recv failed ret=%d\n", ib_ret); |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1030 | ib_conn->post_recv_buf_count--; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1031 | } |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1032 | |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1033 | return ib_ret; |
| 1034 | } |
| 1035 | |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1036 | int iser_post_recvm(struct iser_conn *iser_conn, int count) |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1037 | { |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1038 | struct ib_conn *ib_conn = &iser_conn->ib_conn; |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1039 | unsigned int my_rx_head = iser_conn->rx_desc_head; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1040 | struct iser_rx_desc *rx_desc; |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1041 | struct ib_recv_wr *wr, *wr_failed; |
| 1042 | int i, ib_ret; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1043 | |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1044 | for (wr = ib_conn->rx_wr, i = 0; i < count; i++, wr++) { |
| 1045 | rx_desc = &iser_conn->rx_descs[my_rx_head]; |
| 1046 | rx_desc->cqe.done = iser_task_rsp; |
| 1047 | wr->wr_cqe = &rx_desc->cqe; |
| 1048 | wr->sg_list = &rx_desc->rx_sg; |
| 1049 | wr->num_sge = 1; |
| 1050 | wr->next = wr + 1; |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1051 | my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1052 | } |
| 1053 | |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1054 | wr--; |
| 1055 | wr->next = NULL; /* mark end of work requests list */ |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1056 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1057 | ib_conn->post_recv_buf_count += count; |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1058 | ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &wr_failed); |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1059 | if (ib_ret) { |
| 1060 | iser_err("ib_post_recv failed ret=%d\n", ib_ret); |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1061 | ib_conn->post_recv_buf_count -= count; |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1062 | } else |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1063 | iser_conn->rx_desc_head = my_rx_head; |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1064 | |
Or Gerlitz | bcc60c3 | 2010-02-08 13:17:42 +0000 | [diff] [blame] | 1065 | return ib_ret; |
| 1066 | } |
| 1067 | |
| 1068 | |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1069 | /** |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1070 | * iser_start_send - Initiate a Send DTO operation |
| 1071 | * |
| 1072 | * returns 0 on success, -1 on failure |
| 1073 | */ |
Sagi Grimberg | 6df5a12 | 2014-10-01 14:02:12 +0300 | [diff] [blame] | 1074 | int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, |
| 1075 | bool signal) |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1076 | { |
Sagi Grimberg | 7332bed | 2015-08-06 18:33:06 +0300 | [diff] [blame] | 1077 | struct ib_send_wr *bad_wr, *wr = iser_tx_next_wr(tx_desc); |
| 1078 | int ib_ret; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1079 | |
Sagi Grimberg | a4ee353 | 2014-10-01 14:01:58 +0300 | [diff] [blame] | 1080 | ib_dma_sync_single_for_device(ib_conn->device->ib_device, |
Sagi Grimberg | 5716af6 | 2014-10-01 14:01:57 +0300 | [diff] [blame] | 1081 | tx_desc->dma_addr, ISER_HEADERS_LEN, |
| 1082 | DMA_TO_DEVICE); |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1083 | |
Sagi Grimberg | 7332bed | 2015-08-06 18:33:06 +0300 | [diff] [blame] | 1084 | wr->next = NULL; |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1085 | wr->wr_cqe = &tx_desc->cqe; |
Sagi Grimberg | 7332bed | 2015-08-06 18:33:06 +0300 | [diff] [blame] | 1086 | wr->sg_list = tx_desc->tx_sg; |
| 1087 | wr->num_sge = tx_desc->num_sge; |
| 1088 | wr->opcode = IB_WR_SEND; |
| 1089 | wr->send_flags = signal ? IB_SEND_SIGNALED : 0; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1090 | |
Christoph Hellwig | e622f2f | 2015-10-08 09:16:33 +0100 | [diff] [blame] | 1091 | ib_ret = ib_post_send(ib_conn->qp, &tx_desc->wrs[0].send, &bad_wr); |
Sagi Grimberg | ff3dd52 | 2014-10-01 14:02:10 +0300 | [diff] [blame] | 1092 | if (ib_ret) |
Sagi Grimberg | 7332bed | 2015-08-06 18:33:06 +0300 | [diff] [blame] | 1093 | iser_err("ib_post_send failed, ret:%d opcode:%d\n", |
| 1094 | ib_ret, bad_wr->opcode); |
Sagi Grimberg | ff3dd52 | 2014-10-01 14:02:10 +0300 | [diff] [blame] | 1095 | |
Or Gerlitz | f19624a | 2010-02-08 13:19:56 +0000 | [diff] [blame] | 1096 | return ib_ret; |
Or Gerlitz | 1cfa0a7 | 2006-05-11 10:02:46 +0300 | [diff] [blame] | 1097 | } |
| 1098 | |
Sagi Grimberg | 0a7a08a | 2014-03-05 19:43:50 +0200 | [diff] [blame] | 1099 | u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task, |
| 1100 | enum iser_data_dir cmd_dir, sector_t *sector) |
| 1101 | { |
Sagi Grimberg | b130ede | 2015-04-14 18:08:19 +0300 | [diff] [blame] | 1102 | struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir]; |
Sagi Grimberg | 5190cc2 | 2015-08-06 18:32:54 +0300 | [diff] [blame] | 1103 | struct iser_fr_desc *desc = reg->mem_h; |
Sagi Grimberg | 0a7a08a | 2014-03-05 19:43:50 +0200 | [diff] [blame] | 1104 | unsigned long sector_size = iser_task->sc->device->sector_size; |
| 1105 | struct ib_mr_status mr_status; |
| 1106 | int ret; |
| 1107 | |
Sagi Grimberg | d711d81 | 2015-08-06 18:32:53 +0300 | [diff] [blame] | 1108 | if (desc && desc->pi_ctx->sig_protected) { |
| 1109 | desc->pi_ctx->sig_protected = 0; |
Sagi Grimberg | 0a7a08a | 2014-03-05 19:43:50 +0200 | [diff] [blame] | 1110 | ret = ib_check_mr_status(desc->pi_ctx->sig_mr, |
| 1111 | IB_MR_CHECK_SIG_STATUS, &mr_status); |
| 1112 | if (ret) { |
| 1113 | pr_err("ib_check_mr_status failed, ret %d\n", ret); |
| 1114 | goto err; |
| 1115 | } |
| 1116 | |
| 1117 | if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { |
| 1118 | sector_t sector_off = mr_status.sig_err.sig_err_offset; |
| 1119 | |
Arnd Bergmann | 2c63d10 | 2015-11-20 17:41:36 +0100 | [diff] [blame] | 1120 | sector_div(sector_off, sector_size + 8); |
Sagi Grimberg | 0a7a08a | 2014-03-05 19:43:50 +0200 | [diff] [blame] | 1121 | *sector = scsi_get_lba(iser_task->sc) + sector_off; |
| 1122 | |
Randy Dunlap | 39c978c | 2014-03-28 14:03:40 -0700 | [diff] [blame] | 1123 | pr_err("PI error found type %d at sector %llx " |
Sagi Grimberg | 0a7a08a | 2014-03-05 19:43:50 +0200 | [diff] [blame] | 1124 | "expected %x vs actual %x\n", |
Randy Dunlap | 39c978c | 2014-03-28 14:03:40 -0700 | [diff] [blame] | 1125 | mr_status.sig_err.err_type, |
| 1126 | (unsigned long long)*sector, |
Sagi Grimberg | 0a7a08a | 2014-03-05 19:43:50 +0200 | [diff] [blame] | 1127 | mr_status.sig_err.expected, |
| 1128 | mr_status.sig_err.actual); |
| 1129 | |
| 1130 | switch (mr_status.sig_err.err_type) { |
| 1131 | case IB_SIG_BAD_GUARD: |
| 1132 | return 0x1; |
| 1133 | case IB_SIG_BAD_REFTAG: |
| 1134 | return 0x3; |
| 1135 | case IB_SIG_BAD_APPTAG: |
| 1136 | return 0x2; |
| 1137 | } |
| 1138 | } |
| 1139 | } |
| 1140 | |
| 1141 | return 0; |
| 1142 | err: |
| 1143 | /* Not alot we can do here, return ambiguous guard error */ |
| 1144 | return 0x1; |
| 1145 | } |
Christoph Hellwig | cfeb91b | 2015-12-11 11:54:28 -0800 | [diff] [blame] | 1146 | |
| 1147 | void iser_err_comp(struct ib_wc *wc, const char *type) |
| 1148 | { |
| 1149 | if (wc->status != IB_WC_WR_FLUSH_ERR) { |
| 1150 | struct iser_conn *iser_conn = to_iser_conn(wc->qp->qp_context); |
| 1151 | |
| 1152 | iser_err("%s failure: %s (%d) vend_err %x\n", type, |
| 1153 | ib_wc_status_msg(wc->status), wc->status, |
| 1154 | wc->vendor_err); |
| 1155 | |
| 1156 | if (iser_conn->iscsi_conn) |
| 1157 | iscsi_conn_failure(iser_conn->iscsi_conn, |
| 1158 | ISCSI_ERR_CONN_FAILED); |
| 1159 | } else { |
| 1160 | iser_dbg("%s failure: %s (%d)\n", type, |
| 1161 | ib_wc_status_msg(wc->status), wc->status); |
| 1162 | } |
| 1163 | } |