| /* |
| * CXL Flash Device Driver |
| * |
| * Written by: Manoj N. Kumar <manoj@linux.vnet.ibm.com>, IBM Corporation |
| * Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation |
| * |
| * Copyright (C) 2015 IBM Corporation |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version |
| * 2 of the License, or (at your option) any later version. |
| */ |
| |
| #include <linux/delay.h> |
| #include <linux/file.h> |
| #include <linux/syscalls.h> |
| #include <misc/cxl.h> |
| #include <asm/unaligned.h> |
| |
| #include <scsi/scsi.h> |
| #include <scsi/scsi_host.h> |
| #include <scsi/scsi_cmnd.h> |
| #include <scsi/scsi_eh.h> |
| #include <uapi/scsi/cxlflash_ioctl.h> |
| |
| #include "sislite.h" |
| #include "common.h" |
| #include "vlun.h" |
| #include "superpipe.h" |
| |
| struct cxlflash_global global; |
| |
| /** |
| * marshal_rele_to_resize() - translate release to resize structure |
| * @rele: Source structure from which to translate/copy. |
| * @resize: Destination structure for the translate/copy. |
| */ |
| static void marshal_rele_to_resize(struct dk_cxlflash_release *release, |
| struct dk_cxlflash_resize *resize) |
| { |
| resize->hdr = release->hdr; |
| resize->context_id = release->context_id; |
| resize->rsrc_handle = release->rsrc_handle; |
| } |
| |
| /** |
| * marshal_det_to_rele() - translate detach to release structure |
| * @detach: Destination structure for the translate/copy. |
| * @rele: Source structure from which to translate/copy. |
| */ |
| static void marshal_det_to_rele(struct dk_cxlflash_detach *detach, |
| struct dk_cxlflash_release *release) |
| { |
| release->hdr = detach->hdr; |
| release->context_id = detach->context_id; |
| } |
| |
| /** |
| * cxlflash_free_errpage() - frees resources associated with global error page |
| */ |
| void cxlflash_free_errpage(void) |
| { |
| |
| mutex_lock(&global.mutex); |
| if (global.err_page) { |
| __free_page(global.err_page); |
| global.err_page = NULL; |
| } |
| mutex_unlock(&global.mutex); |
| } |
| |
| /** |
| * cxlflash_stop_term_user_contexts() - stops/terminates known user contexts |
| * @cfg: Internal structure associated with the host. |
| * |
| * When the host needs to go down, all users must be quiesced and their |
| * memory freed. This is accomplished by putting the contexts in error |
| * state which will notify the user and let them 'drive' the tear-down. |
| * Meanwhile, this routine camps until all user contexts have been removed. |
| */ |
| void cxlflash_stop_term_user_contexts(struct cxlflash_cfg *cfg) |
| { |
| struct device *dev = &cfg->dev->dev; |
| int i, found; |
| |
| cxlflash_mark_contexts_error(cfg); |
| |
| while (true) { |
| found = false; |
| |
| for (i = 0; i < MAX_CONTEXT; i++) |
| if (cfg->ctx_tbl[i]) { |
| found = true; |
| break; |
| } |
| |
| if (!found && list_empty(&cfg->ctx_err_recovery)) |
| return; |
| |
| dev_dbg(dev, "%s: Wait for user contexts to quiesce...\n", |
| __func__); |
| wake_up_all(&cfg->limbo_waitq); |
| ssleep(1); |
| } |
| } |
| |
| /** |
| * find_error_context() - locates a context by cookie on the error recovery list |
| * @cfg: Internal structure associated with the host. |
| * @rctxid: Desired context by id. |
| * @file: Desired context by file. |
| * |
| * Return: Found context on success, NULL on failure |
| */ |
| static struct ctx_info *find_error_context(struct cxlflash_cfg *cfg, u64 rctxid, |
| struct file *file) |
| { |
| struct ctx_info *ctxi; |
| |
| list_for_each_entry(ctxi, &cfg->ctx_err_recovery, list) |
| if ((ctxi->ctxid == rctxid) || (ctxi->file == file)) |
| return ctxi; |
| |
| return NULL; |
| } |
| |
| /** |
| * get_context() - obtains a validated and locked context reference |
| * @cfg: Internal structure associated with the host. |
| * @rctxid: Desired context (raw, un-decoded format). |
| * @arg: LUN information or file associated with request. |
| * @ctx_ctrl: Control information to 'steer' desired lookup. |
| * |
| * NOTE: despite the name pid, in linux, current->pid actually refers |
| * to the lightweight process id (tid) and can change if the process is |
| * multi threaded. The tgid remains constant for the process and only changes |
| * when the process of fork. For all intents and purposes, think of tgid |
| * as a pid in the traditional sense. |
| * |
| * Return: Validated context on success, NULL on failure |
| */ |
| struct ctx_info *get_context(struct cxlflash_cfg *cfg, u64 rctxid, |
| void *arg, enum ctx_ctrl ctx_ctrl) |
| { |
| struct device *dev = &cfg->dev->dev; |
| struct ctx_info *ctxi = NULL; |
| struct lun_access *lun_access = NULL; |
| struct file *file = NULL; |
| struct llun_info *lli = arg; |
| u64 ctxid = DECODE_CTXID(rctxid); |
| int rc; |
| pid_t pid = current->tgid, ctxpid = 0; |
| |
| if (ctx_ctrl & CTX_CTRL_FILE) { |
| lli = NULL; |
| file = (struct file *)arg; |
| } |
| |
| if (ctx_ctrl & CTX_CTRL_CLONE) |
| pid = current->parent->tgid; |
| |
| if (likely(ctxid < MAX_CONTEXT)) { |
| while (true) { |
| rc = mutex_lock_interruptible(&cfg->ctx_tbl_list_mutex); |
| if (rc) |
| goto out; |
| |
| ctxi = cfg->ctx_tbl[ctxid]; |
| if (ctxi) |
| if ((file && (ctxi->file != file)) || |
| (!file && (ctxi->ctxid != rctxid))) |
| ctxi = NULL; |
| |
| if ((ctx_ctrl & CTX_CTRL_ERR) || |
| (!ctxi && (ctx_ctrl & CTX_CTRL_ERR_FALLBACK))) |
| ctxi = find_error_context(cfg, rctxid, file); |
| if (!ctxi) { |
| mutex_unlock(&cfg->ctx_tbl_list_mutex); |
| goto out; |
| } |
| |
| /* |
| * Need to acquire ownership of the context while still |
| * under the table/list lock to serialize with a remove |
| * thread. Use the 'try' to avoid stalling the |
| * table/list lock for a single context. |
| * |
| * Note that the lock order is: |
| * |
| * cfg->ctx_tbl_list_mutex -> ctxi->mutex |
| * |
| * Therefore release ctx_tbl_list_mutex before retrying. |
| */ |
| rc = mutex_trylock(&ctxi->mutex); |
| mutex_unlock(&cfg->ctx_tbl_list_mutex); |
| if (rc) |
| break; /* got the context's lock! */ |
| } |
| |
| if (ctxi->unavail) |
| goto denied; |
| |
| ctxpid = ctxi->pid; |
| if (likely(!(ctx_ctrl & CTX_CTRL_NOPID))) |
| if (pid != ctxpid) |
| goto denied; |
| |
| if (lli) { |
| list_for_each_entry(lun_access, &ctxi->luns, list) |
| if (lun_access->lli == lli) |
| goto out; |
| goto denied; |
| } |
| } |
| |
| out: |
| dev_dbg(dev, "%s: rctxid=%016llX ctxinfo=%p ctxpid=%u pid=%u " |
| "ctx_ctrl=%u\n", __func__, rctxid, ctxi, ctxpid, pid, |
| ctx_ctrl); |
| |
| return ctxi; |
| |
| denied: |
| mutex_unlock(&ctxi->mutex); |
| ctxi = NULL; |
| goto out; |
| } |
| |
| /** |
| * put_context() - release a context that was retrieved from get_context() |
| * @ctxi: Context to release. |
| * |
| * For now, releasing the context equates to unlocking it's mutex. |
| */ |
| void put_context(struct ctx_info *ctxi) |
| { |
| mutex_unlock(&ctxi->mutex); |
| } |
| |
| /** |
| * afu_attach() - attach a context to the AFU |
| * @cfg: Internal structure associated with the host. |
| * @ctxi: Context to attach. |
| * |
| * Upon setting the context capabilities, they must be confirmed with |
| * a read back operation as the context might have been closed since |
| * the mailbox was unlocked. When this occurs, registration is failed. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int afu_attach(struct cxlflash_cfg *cfg, struct ctx_info *ctxi) |
| { |
| struct device *dev = &cfg->dev->dev; |
| struct afu *afu = cfg->afu; |
| struct sisl_ctrl_map *ctrl_map = ctxi->ctrl_map; |
| int rc = 0; |
| u64 val; |
| |
| /* Unlock cap and restrict user to read/write cmds in translated mode */ |
| readq_be(&ctrl_map->mbox_r); |
| val = (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD); |
| writeq_be(val, &ctrl_map->ctx_cap); |
| val = readq_be(&ctrl_map->ctx_cap); |
| if (val != (SISL_CTX_CAP_READ_CMD | SISL_CTX_CAP_WRITE_CMD)) { |
| dev_err(dev, "%s: ctx may be closed val=%016llX\n", |
| __func__, val); |
| rc = -EAGAIN; |
| goto out; |
| } |
| |
| /* Set up MMIO registers pointing to the RHT */ |
| writeq_be((u64)ctxi->rht_start, &ctrl_map->rht_start); |
| val = SISL_RHT_CNT_ID((u64)MAX_RHT_PER_CONTEXT, (u64)(afu->ctx_hndl)); |
| writeq_be(val, &ctrl_map->rht_cnt_id); |
| out: |
| dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); |
| return rc; |
| } |
| |
| /** |
| * read_cap16() - issues a SCSI READ_CAP16 command |
| * @sdev: SCSI device associated with LUN. |
| * @lli: LUN destined for capacity request. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int read_cap16(struct scsi_device *sdev, struct llun_info *lli) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct glun_info *gli = lli->parent; |
| u8 *cmd_buf = NULL; |
| u8 *scsi_cmd = NULL; |
| u8 *sense_buf = NULL; |
| int rc = 0; |
| int result = 0; |
| int retry_cnt = 0; |
| u32 tout = (MC_DISCOVERY_TIMEOUT * HZ); |
| |
| retry: |
| cmd_buf = kzalloc(CMD_BUFSIZE, GFP_KERNEL); |
| scsi_cmd = kzalloc(MAX_COMMAND_SIZE, GFP_KERNEL); |
| sense_buf = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL); |
| if (unlikely(!cmd_buf || !scsi_cmd || !sense_buf)) { |
| rc = -ENOMEM; |
| goto out; |
| } |
| |
| scsi_cmd[0] = SERVICE_ACTION_IN_16; /* read cap(16) */ |
| scsi_cmd[1] = SAI_READ_CAPACITY_16; /* service action */ |
| put_unaligned_be32(CMD_BUFSIZE, &scsi_cmd[10]); |
| |
| dev_dbg(dev, "%s: %ssending cmd(0x%x)\n", __func__, |
| retry_cnt ? "re" : "", scsi_cmd[0]); |
| |
| result = scsi_execute(sdev, scsi_cmd, DMA_FROM_DEVICE, cmd_buf, |
| CMD_BUFSIZE, sense_buf, tout, 5, 0, NULL); |
| |
| if (driver_byte(result) == DRIVER_SENSE) { |
| result &= ~(0xFF<<24); /* DRIVER_SENSE is not an error */ |
| if (result & SAM_STAT_CHECK_CONDITION) { |
| struct scsi_sense_hdr sshdr; |
| |
| scsi_normalize_sense(sense_buf, SCSI_SENSE_BUFFERSIZE, |
| &sshdr); |
| switch (sshdr.sense_key) { |
| case NO_SENSE: |
| case RECOVERED_ERROR: |
| /* fall through */ |
| case NOT_READY: |
| result &= ~SAM_STAT_CHECK_CONDITION; |
| break; |
| case UNIT_ATTENTION: |
| switch (sshdr.asc) { |
| case 0x29: /* Power on Reset or Device Reset */ |
| /* fall through */ |
| case 0x2A: /* Device capacity changed */ |
| case 0x3F: /* Report LUNs changed */ |
| /* Retry the command once more */ |
| if (retry_cnt++ < 1) { |
| kfree(cmd_buf); |
| kfree(scsi_cmd); |
| kfree(sense_buf); |
| goto retry; |
| } |
| } |
| break; |
| default: |
| break; |
| } |
| } |
| } |
| |
| if (result) { |
| dev_err(dev, "%s: command failed, result=0x%x\n", |
| __func__, result); |
| rc = -EIO; |
| goto out; |
| } |
| |
| /* |
| * Read cap was successful, grab values from the buffer; |
| * note that we don't need to worry about unaligned access |
| * as the buffer is allocated on an aligned boundary. |
| */ |
| mutex_lock(&gli->mutex); |
| gli->max_lba = be64_to_cpu(*((u64 *)&cmd_buf[0])); |
| gli->blk_len = be32_to_cpu(*((u32 *)&cmd_buf[8])); |
| mutex_unlock(&gli->mutex); |
| |
| out: |
| kfree(cmd_buf); |
| kfree(scsi_cmd); |
| kfree(sense_buf); |
| |
| dev_dbg(dev, "%s: maxlba=%lld blklen=%d rc=%d\n", |
| __func__, gli->max_lba, gli->blk_len, rc); |
| return rc; |
| } |
| |
| /** |
| * get_rhte() - obtains validated resource handle table entry reference |
| * @ctxi: Context owning the resource handle. |
| * @rhndl: Resource handle associated with entry. |
| * @lli: LUN associated with request. |
| * |
| * Return: Validated RHTE on success, NULL on failure |
| */ |
| struct sisl_rht_entry *get_rhte(struct ctx_info *ctxi, res_hndl_t rhndl, |
| struct llun_info *lli) |
| { |
| struct sisl_rht_entry *rhte = NULL; |
| |
| if (unlikely(!ctxi->rht_start)) { |
| pr_debug("%s: Context does not have allocated RHT!\n", |
| __func__); |
| goto out; |
| } |
| |
| if (unlikely(rhndl >= MAX_RHT_PER_CONTEXT)) { |
| pr_debug("%s: Bad resource handle! (%d)\n", __func__, rhndl); |
| goto out; |
| } |
| |
| if (unlikely(ctxi->rht_lun[rhndl] != lli)) { |
| pr_debug("%s: Bad resource handle LUN! (%d)\n", |
| __func__, rhndl); |
| goto out; |
| } |
| |
| rhte = &ctxi->rht_start[rhndl]; |
| if (unlikely(rhte->nmask == 0)) { |
| pr_debug("%s: Unopened resource handle! (%d)\n", |
| __func__, rhndl); |
| rhte = NULL; |
| goto out; |
| } |
| |
| out: |
| return rhte; |
| } |
| |
| /** |
| * rhte_checkout() - obtains free/empty resource handle table entry |
| * @ctxi: Context owning the resource handle. |
| * @lli: LUN associated with request. |
| * |
| * Return: Free RHTE on success, NULL on failure |
| */ |
| struct sisl_rht_entry *rhte_checkout(struct ctx_info *ctxi, |
| struct llun_info *lli) |
| { |
| struct sisl_rht_entry *rhte = NULL; |
| int i; |
| |
| /* Find a free RHT entry */ |
| for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) |
| if (ctxi->rht_start[i].nmask == 0) { |
| rhte = &ctxi->rht_start[i]; |
| ctxi->rht_out++; |
| break; |
| } |
| |
| if (likely(rhte)) |
| ctxi->rht_lun[i] = lli; |
| |
| pr_debug("%s: returning rhte=%p (%d)\n", __func__, rhte, i); |
| return rhte; |
| } |
| |
| /** |
| * rhte_checkin() - releases a resource handle table entry |
| * @ctxi: Context owning the resource handle. |
| * @rhte: RHTE to release. |
| */ |
| void rhte_checkin(struct ctx_info *ctxi, |
| struct sisl_rht_entry *rhte) |
| { |
| u32 rsrc_handle = rhte - ctxi->rht_start; |
| |
| rhte->nmask = 0; |
| rhte->fp = 0; |
| ctxi->rht_out--; |
| ctxi->rht_lun[rsrc_handle] = NULL; |
| ctxi->rht_needs_ws[rsrc_handle] = false; |
| } |
| |
| /** |
| * rhte_format1() - populates a RHTE for format 1 |
| * @rhte: RHTE to populate. |
| * @lun_id: LUN ID of LUN associated with RHTE. |
| * @perm: Desired permissions for RHTE. |
| * @port_sel: Port selection mask |
| */ |
| static void rht_format1(struct sisl_rht_entry *rhte, u64 lun_id, u32 perm, |
| u32 port_sel) |
| { |
| /* |
| * Populate the Format 1 RHT entry for direct access (physical |
| * LUN) using the synchronization sequence defined in the |
| * SISLite specification. |
| */ |
| struct sisl_rht_entry_f1 dummy = { 0 }; |
| struct sisl_rht_entry_f1 *rhte_f1 = (struct sisl_rht_entry_f1 *)rhte; |
| |
| memset(rhte_f1, 0, sizeof(*rhte_f1)); |
| rhte_f1->fp = SISL_RHT_FP(1U, 0); |
| dma_wmb(); /* Make setting of format bit visible */ |
| |
| rhte_f1->lun_id = lun_id; |
| dma_wmb(); /* Make setting of LUN id visible */ |
| |
| /* |
| * Use a dummy RHT Format 1 entry to build the second dword |
| * of the entry that must be populated in a single write when |
| * enabled (valid bit set to TRUE). |
| */ |
| dummy.valid = 0x80; |
| dummy.fp = SISL_RHT_FP(1U, perm); |
| dummy.port_sel = port_sel; |
| rhte_f1->dw = dummy.dw; |
| |
| dma_wmb(); /* Make remaining RHT entry fields visible */ |
| } |
| |
| /** |
| * cxlflash_lun_attach() - attaches a user to a LUN and manages the LUN's mode |
| * @gli: LUN to attach. |
| * @mode: Desired mode of the LUN. |
| * @locked: Mutex status on current thread. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| int cxlflash_lun_attach(struct glun_info *gli, enum lun_mode mode, bool locked) |
| { |
| int rc = 0; |
| |
| if (!locked) |
| mutex_lock(&gli->mutex); |
| |
| if (gli->mode == MODE_NONE) |
| gli->mode = mode; |
| else if (gli->mode != mode) { |
| pr_debug("%s: LUN operating in mode %d, requested mode %d\n", |
| __func__, gli->mode, mode); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| gli->users++; |
| WARN_ON(gli->users <= 0); |
| out: |
| pr_debug("%s: Returning rc=%d gli->mode=%u gli->users=%u\n", |
| __func__, rc, gli->mode, gli->users); |
| if (!locked) |
| mutex_unlock(&gli->mutex); |
| return rc; |
| } |
| |
| /** |
| * cxlflash_lun_detach() - detaches a user from a LUN and resets the LUN's mode |
| * @gli: LUN to detach. |
| * |
| * When resetting the mode, terminate block allocation resources as they |
| * are no longer required (service is safe to call even when block allocation |
| * resources were not present - such as when transitioning from physical mode). |
| * These resources will be reallocated when needed (subsequent transition to |
| * virtual mode). |
| */ |
| void cxlflash_lun_detach(struct glun_info *gli) |
| { |
| mutex_lock(&gli->mutex); |
| WARN_ON(gli->mode == MODE_NONE); |
| if (--gli->users == 0) { |
| gli->mode = MODE_NONE; |
| cxlflash_ba_terminate(&gli->blka.ba_lun); |
| } |
| pr_debug("%s: gli->users=%u\n", __func__, gli->users); |
| WARN_ON(gli->users < 0); |
| mutex_unlock(&gli->mutex); |
| } |
| |
| /** |
| * _cxlflash_disk_release() - releases the specified resource entry |
| * @sdev: SCSI device associated with LUN. |
| * @ctxi: Context owning resources. |
| * @release: Release ioctl data structure. |
| * |
| * For LUNs in virtual mode, the virtual LUN associated with the specified |
| * resource handle is resized to 0 prior to releasing the RHTE. Note that the |
| * AFU sync should _not_ be performed when the context is sitting on the error |
| * recovery list. A context on the error recovery list is not known to the AFU |
| * due to reset. When the context is recovered, it will be reattached and made |
| * known again to the AFU. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| int _cxlflash_disk_release(struct scsi_device *sdev, |
| struct ctx_info *ctxi, |
| struct dk_cxlflash_release *release) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct llun_info *lli = sdev->hostdata; |
| struct glun_info *gli = lli->parent; |
| struct afu *afu = cfg->afu; |
| bool put_ctx = false; |
| |
| struct dk_cxlflash_resize size; |
| res_hndl_t rhndl = release->rsrc_handle; |
| |
| int rc = 0; |
| u64 ctxid = DECODE_CTXID(release->context_id), |
| rctxid = release->context_id; |
| |
| struct sisl_rht_entry *rhte; |
| struct sisl_rht_entry_f1 *rhte_f1; |
| |
| dev_dbg(dev, "%s: ctxid=%llu rhndl=0x%llx gli->mode=%u gli->users=%u\n", |
| __func__, ctxid, release->rsrc_handle, gli->mode, gli->users); |
| |
| if (!ctxi) { |
| ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%llu)\n", |
| __func__, ctxid); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| put_ctx = true; |
| } |
| |
| rhte = get_rhte(ctxi, rhndl, lli); |
| if (unlikely(!rhte)) { |
| dev_dbg(dev, "%s: Bad resource handle! (%d)\n", |
| __func__, rhndl); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| /* |
| * Resize to 0 for virtual LUNS by setting the size |
| * to 0. This will clear LXT_START and LXT_CNT fields |
| * in the RHT entry and properly sync with the AFU. |
| * |
| * Afterwards we clear the remaining fields. |
| */ |
| switch (gli->mode) { |
| case MODE_VIRTUAL: |
| marshal_rele_to_resize(release, &size); |
| size.req_size = 0; |
| rc = _cxlflash_vlun_resize(sdev, ctxi, &size); |
| if (rc) { |
| dev_dbg(dev, "%s: resize failed rc %d\n", __func__, rc); |
| goto out; |
| } |
| |
| break; |
| case MODE_PHYSICAL: |
| /* |
| * Clear the Format 1 RHT entry for direct access |
| * (physical LUN) using the synchronization sequence |
| * defined in the SISLite specification. |
| */ |
| rhte_f1 = (struct sisl_rht_entry_f1 *)rhte; |
| |
| rhte_f1->valid = 0; |
| dma_wmb(); /* Make revocation of RHT entry visible */ |
| |
| rhte_f1->lun_id = 0; |
| dma_wmb(); /* Make clearing of LUN id visible */ |
| |
| rhte_f1->dw = 0; |
| dma_wmb(); /* Make RHT entry bottom-half clearing visible */ |
| |
| if (!ctxi->err_recovery_active) |
| cxlflash_afu_sync(afu, ctxid, rhndl, AFU_HW_SYNC); |
| break; |
| default: |
| WARN(1, "Unsupported LUN mode!"); |
| goto out; |
| } |
| |
| rhte_checkin(ctxi, rhte); |
| cxlflash_lun_detach(gli); |
| |
| out: |
| if (put_ctx) |
| put_context(ctxi); |
| dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); |
| return rc; |
| } |
| |
| int cxlflash_disk_release(struct scsi_device *sdev, |
| struct dk_cxlflash_release *release) |
| { |
| return _cxlflash_disk_release(sdev, NULL, release); |
| } |
| |
| /** |
| * destroy_context() - releases a context |
| * @cfg: Internal structure associated with the host. |
| * @ctxi: Context to release. |
| * |
| * Note that the rht_lun member of the context was cut from a single |
| * allocation when the context was created and therefore does not need |
| * to be explicitly freed. Also note that we conditionally check for the |
| * existence of the context control map before clearing the RHT registers |
| * and context capabilities because it is possible to destroy a context |
| * while the context is in the error state (previous mapping was removed |
| * [so we don't have to worry about clearing] and context is waiting for |
| * a new mapping). |
| */ |
| static void destroy_context(struct cxlflash_cfg *cfg, |
| struct ctx_info *ctxi) |
| { |
| struct afu *afu = cfg->afu; |
| |
| WARN_ON(!list_empty(&ctxi->luns)); |
| |
| /* Clear RHT registers and drop all capabilities for this context */ |
| if (afu->afu_map && ctxi->ctrl_map) { |
| writeq_be(0, &ctxi->ctrl_map->rht_start); |
| writeq_be(0, &ctxi->ctrl_map->rht_cnt_id); |
| writeq_be(0, &ctxi->ctrl_map->ctx_cap); |
| } |
| |
| /* Free memory associated with context */ |
| free_page((ulong)ctxi->rht_start); |
| kfree(ctxi->rht_needs_ws); |
| kfree(ctxi->rht_lun); |
| kfree(ctxi); |
| atomic_dec_if_positive(&cfg->num_user_contexts); |
| } |
| |
| /** |
| * create_context() - allocates and initializes a context |
| * @cfg: Internal structure associated with the host. |
| * @ctx: Previously obtained CXL context reference. |
| * @ctxid: Previously obtained process element associated with CXL context. |
| * @adap_fd: Previously obtained adapter fd associated with CXL context. |
| * @file: Previously obtained file associated with CXL context. |
| * @perms: User-specified permissions. |
| * |
| * The context's mutex is locked when an allocated context is returned. |
| * |
| * Return: Allocated context on success, NULL on failure |
| */ |
| static struct ctx_info *create_context(struct cxlflash_cfg *cfg, |
| struct cxl_context *ctx, int ctxid, |
| int adap_fd, struct file *file, |
| u32 perms) |
| { |
| struct device *dev = &cfg->dev->dev; |
| struct afu *afu = cfg->afu; |
| struct ctx_info *ctxi = NULL; |
| struct llun_info **lli = NULL; |
| bool *ws = NULL; |
| struct sisl_rht_entry *rhte; |
| |
| ctxi = kzalloc(sizeof(*ctxi), GFP_KERNEL); |
| lli = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*lli)), GFP_KERNEL); |
| ws = kzalloc((MAX_RHT_PER_CONTEXT * sizeof(*ws)), GFP_KERNEL); |
| if (unlikely(!ctxi || !lli || !ws)) { |
| dev_err(dev, "%s: Unable to allocate context!\n", __func__); |
| goto err; |
| } |
| |
| rhte = (struct sisl_rht_entry *)get_zeroed_page(GFP_KERNEL); |
| if (unlikely(!rhte)) { |
| dev_err(dev, "%s: Unable to allocate RHT!\n", __func__); |
| goto err; |
| } |
| |
| ctxi->rht_lun = lli; |
| ctxi->rht_needs_ws = ws; |
| ctxi->rht_start = rhte; |
| ctxi->rht_perms = perms; |
| |
| ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl; |
| ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid); |
| ctxi->lfd = adap_fd; |
| ctxi->pid = current->tgid; /* tgid = pid */ |
| ctxi->ctx = ctx; |
| ctxi->file = file; |
| mutex_init(&ctxi->mutex); |
| INIT_LIST_HEAD(&ctxi->luns); |
| INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */ |
| |
| atomic_inc(&cfg->num_user_contexts); |
| mutex_lock(&ctxi->mutex); |
| out: |
| return ctxi; |
| |
| err: |
| kfree(ws); |
| kfree(lli); |
| kfree(ctxi); |
| ctxi = NULL; |
| goto out; |
| } |
| |
| /** |
| * _cxlflash_disk_detach() - detaches a LUN from a context |
| * @sdev: SCSI device associated with LUN. |
| * @ctxi: Context owning resources. |
| * @detach: Detach ioctl data structure. |
| * |
| * As part of the detach, all per-context resources associated with the LUN |
| * are cleaned up. When detaching the last LUN for a context, the context |
| * itself is cleaned up and released. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int _cxlflash_disk_detach(struct scsi_device *sdev, |
| struct ctx_info *ctxi, |
| struct dk_cxlflash_detach *detach) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct llun_info *lli = sdev->hostdata; |
| struct lun_access *lun_access, *t; |
| struct dk_cxlflash_release rel; |
| bool put_ctx = false; |
| |
| int i; |
| int rc = 0; |
| int lfd; |
| u64 ctxid = DECODE_CTXID(detach->context_id), |
| rctxid = detach->context_id; |
| |
| dev_dbg(dev, "%s: ctxid=%llu\n", __func__, ctxid); |
| |
| if (!ctxi) { |
| ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%llu)\n", |
| __func__, ctxid); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| put_ctx = true; |
| } |
| |
| /* Cleanup outstanding resources tied to this LUN */ |
| if (ctxi->rht_out) { |
| marshal_det_to_rele(detach, &rel); |
| for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) { |
| if (ctxi->rht_lun[i] == lli) { |
| rel.rsrc_handle = i; |
| _cxlflash_disk_release(sdev, ctxi, &rel); |
| } |
| |
| /* No need to loop further if we're done */ |
| if (ctxi->rht_out == 0) |
| break; |
| } |
| } |
| |
| /* Take our LUN out of context, free the node */ |
| list_for_each_entry_safe(lun_access, t, &ctxi->luns, list) |
| if (lun_access->lli == lli) { |
| list_del(&lun_access->list); |
| kfree(lun_access); |
| lun_access = NULL; |
| break; |
| } |
| |
| /* Tear down context following last LUN cleanup */ |
| if (list_empty(&ctxi->luns)) { |
| ctxi->unavail = true; |
| mutex_unlock(&ctxi->mutex); |
| mutex_lock(&cfg->ctx_tbl_list_mutex); |
| mutex_lock(&ctxi->mutex); |
| |
| /* Might not have been in error list so conditionally remove */ |
| if (!list_empty(&ctxi->list)) |
| list_del(&ctxi->list); |
| cfg->ctx_tbl[ctxid] = NULL; |
| mutex_unlock(&cfg->ctx_tbl_list_mutex); |
| mutex_unlock(&ctxi->mutex); |
| |
| lfd = ctxi->lfd; |
| destroy_context(cfg, ctxi); |
| ctxi = NULL; |
| put_ctx = false; |
| |
| /* |
| * As a last step, clean up external resources when not |
| * already on an external cleanup thread, i.e.: close(adap_fd). |
| * |
| * NOTE: this will free up the context from the CXL services, |
| * allowing it to dole out the same context_id on a future |
| * (or even currently in-flight) disk_attach operation. |
| */ |
| if (lfd != -1) |
| sys_close(lfd); |
| } |
| |
| out: |
| if (put_ctx) |
| put_context(ctxi); |
| dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); |
| return rc; |
| } |
| |
| static int cxlflash_disk_detach(struct scsi_device *sdev, |
| struct dk_cxlflash_detach *detach) |
| { |
| return _cxlflash_disk_detach(sdev, NULL, detach); |
| } |
| |
| /** |
| * cxlflash_cxl_release() - release handler for adapter file descriptor |
| * @inode: File-system inode associated with fd. |
| * @file: File installed with adapter file descriptor. |
| * |
| * This routine is the release handler for the fops registered with |
| * the CXL services on an initial attach for a context. It is called |
| * when a close is performed on the adapter file descriptor returned |
| * to the user. Programmatically, the user is not required to perform |
| * the close, as it is handled internally via the detach ioctl when |
| * a context is being removed. Note that nothing prevents the user |
| * from performing a close, but the user should be aware that doing |
| * so is considered catastrophic and subsequent usage of the superpipe |
| * API with previously saved off tokens will fail. |
| * |
| * When initiated from an external close (either by the user or via |
| * a process tear down), the routine derives the context reference |
| * and calls detach for each LUN associated with the context. The |
| * final detach operation will cause the context itself to be freed. |
| * Note that the saved off lfd is reset prior to calling detach to |
| * signify that the final detach should not perform a close. |
| * |
| * When initiated from a detach operation as part of the tear down |
| * of a context, the context is first completely freed and then the |
| * close is performed. This routine will fail to derive the context |
| * reference (due to the context having already been freed) and then |
| * call into the CXL release entry point. |
| * |
| * Thus, with exception to when the CXL process element (context id) |
| * lookup fails (a case that should theoretically never occur), every |
| * call into this routine results in a complete freeing of a context. |
| * |
| * As part of the detach, all per-context resources associated with the LUN |
| * are cleaned up. When detaching the last LUN for a context, the context |
| * itself is cleaned up and released. |
| * |
| * Return: 0 on success |
| */ |
| static int cxlflash_cxl_release(struct inode *inode, struct file *file) |
| { |
| struct cxl_context *ctx = cxl_fops_get_context(file); |
| struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg, |
| cxl_fops); |
| struct device *dev = &cfg->dev->dev; |
| struct ctx_info *ctxi = NULL; |
| struct dk_cxlflash_detach detach = { { 0 }, 0 }; |
| struct lun_access *lun_access, *t; |
| enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE; |
| int ctxid; |
| |
| ctxid = cxl_process_element(ctx); |
| if (unlikely(ctxid < 0)) { |
| dev_err(dev, "%s: Context %p was closed! (%d)\n", |
| __func__, ctx, ctxid); |
| goto out; |
| } |
| |
| ctxi = get_context(cfg, ctxid, file, ctrl); |
| if (unlikely(!ctxi)) { |
| ctxi = get_context(cfg, ctxid, file, ctrl | CTX_CTRL_CLONE); |
| if (!ctxi) { |
| dev_dbg(dev, "%s: Context %d already free!\n", |
| __func__, ctxid); |
| goto out_release; |
| } |
| |
| dev_dbg(dev, "%s: Another process owns context %d!\n", |
| __func__, ctxid); |
| put_context(ctxi); |
| goto out; |
| } |
| |
| dev_dbg(dev, "%s: close(%d) for context %d\n", |
| __func__, ctxi->lfd, ctxid); |
| |
| /* Reset the file descriptor to indicate we're on a close() thread */ |
| ctxi->lfd = -1; |
| detach.context_id = ctxi->ctxid; |
| list_for_each_entry_safe(lun_access, t, &ctxi->luns, list) |
| _cxlflash_disk_detach(lun_access->sdev, ctxi, &detach); |
| out_release: |
| cxl_fd_release(inode, file); |
| out: |
| dev_dbg(dev, "%s: returning\n", __func__); |
| return 0; |
| } |
| |
| /** |
| * unmap_context() - clears a previously established mapping |
| * @ctxi: Context owning the mapping. |
| * |
| * This routine is used to switch between the error notification page |
| * (dummy page of all 1's) and the real mapping (established by the CXL |
| * fault handler). |
| */ |
| static void unmap_context(struct ctx_info *ctxi) |
| { |
| unmap_mapping_range(ctxi->file->f_mapping, 0, 0, 1); |
| } |
| |
| /** |
| * get_err_page() - obtains and allocates the error notification page |
| * |
| * Return: error notification page on success, NULL on failure |
| */ |
| static struct page *get_err_page(void) |
| { |
| struct page *err_page = global.err_page; |
| |
| if (unlikely(!err_page)) { |
| err_page = alloc_page(GFP_KERNEL); |
| if (unlikely(!err_page)) { |
| pr_err("%s: Unable to allocate err_page!\n", __func__); |
| goto out; |
| } |
| |
| memset(page_address(err_page), -1, PAGE_SIZE); |
| |
| /* Serialize update w/ other threads to avoid a leak */ |
| mutex_lock(&global.mutex); |
| if (likely(!global.err_page)) |
| global.err_page = err_page; |
| else { |
| __free_page(err_page); |
| err_page = global.err_page; |
| } |
| mutex_unlock(&global.mutex); |
| } |
| |
| out: |
| pr_debug("%s: returning err_page=%p\n", __func__, err_page); |
| return err_page; |
| } |
| |
| /** |
| * cxlflash_mmap_fault() - mmap fault handler for adapter file descriptor |
| * @vma: VM area associated with mapping. |
| * @vmf: VM fault associated with current fault. |
| * |
| * To support error notification via MMIO, faults are 'caught' by this routine |
| * that was inserted before passing back the adapter file descriptor on attach. |
| * When a fault occurs, this routine evaluates if error recovery is active and |
| * if so, installs the error page to 'notify' the user about the error state. |
| * During normal operation, the fault is simply handled by the original fault |
| * handler that was installed by CXL services as part of initializing the |
| * adapter file descriptor. The VMA's page protection bits are toggled to |
| * indicate cached/not-cached depending on the memory backing the fault. |
| * |
| * Return: 0 on success, VM_FAULT_SIGBUS on failure |
| */ |
| static int cxlflash_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) |
| { |
| struct file *file = vma->vm_file; |
| struct cxl_context *ctx = cxl_fops_get_context(file); |
| struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg, |
| cxl_fops); |
| struct device *dev = &cfg->dev->dev; |
| struct ctx_info *ctxi = NULL; |
| struct page *err_page = NULL; |
| enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE; |
| int rc = 0; |
| int ctxid; |
| |
| ctxid = cxl_process_element(ctx); |
| if (unlikely(ctxid < 0)) { |
| dev_err(dev, "%s: Context %p was closed! (%d)\n", |
| __func__, ctx, ctxid); |
| goto err; |
| } |
| |
| ctxi = get_context(cfg, ctxid, file, ctrl); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%d)\n", __func__, ctxid); |
| goto err; |
| } |
| |
| dev_dbg(dev, "%s: fault(%d) for context %d\n", |
| __func__, ctxi->lfd, ctxid); |
| |
| if (likely(!ctxi->err_recovery_active)) { |
| vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
| rc = ctxi->cxl_mmap_vmops->fault(vma, vmf); |
| } else { |
| dev_dbg(dev, "%s: err recovery active, use err_page!\n", |
| __func__); |
| |
| err_page = get_err_page(); |
| if (unlikely(!err_page)) { |
| dev_err(dev, "%s: Could not obtain error page!\n", |
| __func__); |
| rc = VM_FAULT_RETRY; |
| goto out; |
| } |
| |
| get_page(err_page); |
| vmf->page = err_page; |
| vma->vm_page_prot = pgprot_cached(vma->vm_page_prot); |
| } |
| |
| out: |
| if (likely(ctxi)) |
| put_context(ctxi); |
| dev_dbg(dev, "%s: returning rc=%d\n", __func__, rc); |
| return rc; |
| |
| err: |
| rc = VM_FAULT_SIGBUS; |
| goto out; |
| } |
| |
| /* |
| * Local MMAP vmops to 'catch' faults |
| */ |
| static const struct vm_operations_struct cxlflash_mmap_vmops = { |
| .fault = cxlflash_mmap_fault, |
| }; |
| |
| /** |
| * cxlflash_cxl_mmap() - mmap handler for adapter file descriptor |
| * @file: File installed with adapter file descriptor. |
| * @vma: VM area associated with mapping. |
| * |
| * Installs local mmap vmops to 'catch' faults for error notification support. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma) |
| { |
| struct cxl_context *ctx = cxl_fops_get_context(file); |
| struct cxlflash_cfg *cfg = container_of(file->f_op, struct cxlflash_cfg, |
| cxl_fops); |
| struct device *dev = &cfg->dev->dev; |
| struct ctx_info *ctxi = NULL; |
| enum ctx_ctrl ctrl = CTX_CTRL_ERR_FALLBACK | CTX_CTRL_FILE; |
| int ctxid; |
| int rc = 0; |
| |
| ctxid = cxl_process_element(ctx); |
| if (unlikely(ctxid < 0)) { |
| dev_err(dev, "%s: Context %p was closed! (%d)\n", |
| __func__, ctx, ctxid); |
| rc = -EIO; |
| goto out; |
| } |
| |
| ctxi = get_context(cfg, ctxid, file, ctrl); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%d)\n", __func__, ctxid); |
| rc = -EIO; |
| goto out; |
| } |
| |
| dev_dbg(dev, "%s: mmap(%d) for context %d\n", |
| __func__, ctxi->lfd, ctxid); |
| |
| rc = cxl_fd_mmap(file, vma); |
| if (likely(!rc)) { |
| /* Insert ourself in the mmap fault handler path */ |
| ctxi->cxl_mmap_vmops = vma->vm_ops; |
| vma->vm_ops = &cxlflash_mmap_vmops; |
| } |
| |
| out: |
| if (likely(ctxi)) |
| put_context(ctxi); |
| return rc; |
| } |
| |
| /* |
| * Local fops for adapter file descriptor |
| */ |
| static const struct file_operations cxlflash_cxl_fops = { |
| .owner = THIS_MODULE, |
| .mmap = cxlflash_cxl_mmap, |
| .release = cxlflash_cxl_release, |
| }; |
| |
| /** |
| * cxlflash_mark_contexts_error() - move contexts to error state and list |
| * @cfg: Internal structure associated with the host. |
| * |
| * A context is only moved over to the error list when there are no outstanding |
| * references to it. This ensures that a running operation has completed. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| int cxlflash_mark_contexts_error(struct cxlflash_cfg *cfg) |
| { |
| int i, rc = 0; |
| struct ctx_info *ctxi = NULL; |
| |
| mutex_lock(&cfg->ctx_tbl_list_mutex); |
| |
| for (i = 0; i < MAX_CONTEXT; i++) { |
| ctxi = cfg->ctx_tbl[i]; |
| if (ctxi) { |
| mutex_lock(&ctxi->mutex); |
| cfg->ctx_tbl[i] = NULL; |
| list_add(&ctxi->list, &cfg->ctx_err_recovery); |
| ctxi->err_recovery_active = true; |
| ctxi->ctrl_map = NULL; |
| unmap_context(ctxi); |
| mutex_unlock(&ctxi->mutex); |
| } |
| } |
| |
| mutex_unlock(&cfg->ctx_tbl_list_mutex); |
| return rc; |
| } |
| |
| /* |
| * Dummy NULL fops |
| */ |
| static const struct file_operations null_fops = { |
| .owner = THIS_MODULE, |
| }; |
| |
| /** |
| * cxlflash_disk_attach() - attach a LUN to a context |
| * @sdev: SCSI device associated with LUN. |
| * @attach: Attach ioctl data structure. |
| * |
| * Creates a context and attaches LUN to it. A LUN can only be attached |
| * one time to a context (subsequent attaches for the same context/LUN pair |
| * are not supported). Additional LUNs can be attached to a context by |
| * specifying the 'reuse' flag defined in the cxlflash_ioctl.h header. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int cxlflash_disk_attach(struct scsi_device *sdev, |
| struct dk_cxlflash_attach *attach) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct afu *afu = cfg->afu; |
| struct llun_info *lli = sdev->hostdata; |
| struct glun_info *gli = lli->parent; |
| struct cxl_ioctl_start_work *work; |
| struct ctx_info *ctxi = NULL; |
| struct lun_access *lun_access = NULL; |
| int rc = 0; |
| u32 perms; |
| int ctxid = -1; |
| u64 rctxid = 0UL; |
| struct file *file; |
| |
| struct cxl_context *ctx; |
| |
| int fd = -1; |
| |
| /* On first attach set fileops */ |
| if (atomic_read(&cfg->num_user_contexts) == 0) |
| cfg->cxl_fops = cxlflash_cxl_fops; |
| |
| if (attach->num_interrupts > 4) { |
| dev_dbg(dev, "%s: Cannot support this many interrupts %llu\n", |
| __func__, attach->num_interrupts); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| if (gli->max_lba == 0) { |
| dev_dbg(dev, "%s: No capacity info for this LUN (%016llX)\n", |
| __func__, lli->lun_id[sdev->channel]); |
| rc = read_cap16(sdev, lli); |
| if (rc) { |
| dev_err(dev, "%s: Invalid device! (%d)\n", |
| __func__, rc); |
| rc = -ENODEV; |
| goto out; |
| } |
| dev_dbg(dev, "%s: LBA = %016llX\n", __func__, gli->max_lba); |
| dev_dbg(dev, "%s: BLK_LEN = %08X\n", __func__, gli->blk_len); |
| } |
| |
| if (attach->hdr.flags & DK_CXLFLASH_ATTACH_REUSE_CONTEXT) { |
| rctxid = attach->context_id; |
| ctxi = get_context(cfg, rctxid, NULL, 0); |
| if (!ctxi) { |
| dev_dbg(dev, "%s: Bad context! (%016llX)\n", |
| __func__, rctxid); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| list_for_each_entry(lun_access, &ctxi->luns, list) |
| if (lun_access->lli == lli) { |
| dev_dbg(dev, "%s: Already attached!\n", |
| __func__); |
| rc = -EINVAL; |
| goto out; |
| } |
| } |
| |
| lun_access = kzalloc(sizeof(*lun_access), GFP_KERNEL); |
| if (unlikely(!lun_access)) { |
| dev_err(dev, "%s: Unable to allocate lun_access!\n", __func__); |
| rc = -ENOMEM; |
| goto out; |
| } |
| |
| lun_access->lli = lli; |
| lun_access->sdev = sdev; |
| |
| /* Non-NULL context indicates reuse */ |
| if (ctxi) { |
| dev_dbg(dev, "%s: Reusing context for LUN! (%016llX)\n", |
| __func__, rctxid); |
| list_add(&lun_access->list, &ctxi->luns); |
| fd = ctxi->lfd; |
| goto out_attach; |
| } |
| |
| ctx = cxl_dev_context_init(cfg->dev); |
| if (unlikely(IS_ERR_OR_NULL(ctx))) { |
| dev_err(dev, "%s: Could not initialize context %p\n", |
| __func__, ctx); |
| rc = -ENODEV; |
| goto err0; |
| } |
| |
| ctxid = cxl_process_element(ctx); |
| if (unlikely((ctxid > MAX_CONTEXT) || (ctxid < 0))) { |
| dev_err(dev, "%s: ctxid (%d) invalid!\n", __func__, ctxid); |
| rc = -EPERM; |
| goto err1; |
| } |
| |
| file = cxl_get_fd(ctx, &cfg->cxl_fops, &fd); |
| if (unlikely(fd < 0)) { |
| rc = -ENODEV; |
| dev_err(dev, "%s: Could not get file descriptor\n", __func__); |
| goto err1; |
| } |
| |
| /* Translate read/write O_* flags from fcntl.h to AFU permission bits */ |
| perms = SISL_RHT_PERM(attach->hdr.flags + 1); |
| |
| ctxi = create_context(cfg, ctx, ctxid, fd, file, perms); |
| if (unlikely(!ctxi)) { |
| dev_err(dev, "%s: Failed to create context! (%d)\n", |
| __func__, ctxid); |
| goto err2; |
| } |
| |
| work = &ctxi->work; |
| work->num_interrupts = attach->num_interrupts; |
| work->flags = CXL_START_WORK_NUM_IRQS; |
| |
| rc = cxl_start_work(ctx, work); |
| if (unlikely(rc)) { |
| dev_dbg(dev, "%s: Could not start context rc=%d\n", |
| __func__, rc); |
| goto err3; |
| } |
| |
| rc = afu_attach(cfg, ctxi); |
| if (unlikely(rc)) { |
| dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc); |
| goto err4; |
| } |
| |
| /* |
| * No error paths after this point. Once the fd is installed it's |
| * visible to user space and can't be undone safely on this thread. |
| * There is no need to worry about a deadlock here because no one |
| * knows about us yet; we can be the only one holding our mutex. |
| */ |
| list_add(&lun_access->list, &ctxi->luns); |
| mutex_unlock(&ctxi->mutex); |
| mutex_lock(&cfg->ctx_tbl_list_mutex); |
| mutex_lock(&ctxi->mutex); |
| cfg->ctx_tbl[ctxid] = ctxi; |
| mutex_unlock(&cfg->ctx_tbl_list_mutex); |
| fd_install(fd, file); |
| |
| out_attach: |
| attach->hdr.return_flags = 0; |
| attach->context_id = ctxi->ctxid; |
| attach->block_size = gli->blk_len; |
| attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea); |
| attach->last_lba = gli->max_lba; |
| attach->max_xfer = (sdev->host->max_sectors * 512) / gli->blk_len; |
| |
| out: |
| attach->adap_fd = fd; |
| |
| if (ctxi) |
| put_context(ctxi); |
| |
| dev_dbg(dev, "%s: returning ctxid=%d fd=%d bs=%lld rc=%d llba=%lld\n", |
| __func__, ctxid, fd, attach->block_size, rc, attach->last_lba); |
| return rc; |
| |
| err4: |
| cxl_stop_context(ctx); |
| err3: |
| put_context(ctxi); |
| destroy_context(cfg, ctxi); |
| ctxi = NULL; |
| err2: |
| /* |
| * Here, we're overriding the fops with a dummy all-NULL fops because |
| * fput() calls the release fop, which will cause us to mistakenly |
| * call into the CXL code. Rather than try to add yet more complexity |
| * to that routine (cxlflash_cxl_release) we should try to fix the |
| * issue here. |
| */ |
| file->f_op = &null_fops; |
| fput(file); |
| put_unused_fd(fd); |
| fd = -1; |
| err1: |
| cxl_release_context(ctx); |
| err0: |
| kfree(lun_access); |
| goto out; |
| } |
| |
| /** |
| * recover_context() - recovers a context in error |
| * @cfg: Internal structure associated with the host. |
| * @ctxi: Context to release. |
| * |
| * Restablishes the state for a context-in-error. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi) |
| { |
| struct device *dev = &cfg->dev->dev; |
| int rc = 0; |
| int old_fd, fd = -1; |
| int ctxid = -1; |
| struct file *file; |
| struct cxl_context *ctx; |
| struct afu *afu = cfg->afu; |
| |
| ctx = cxl_dev_context_init(cfg->dev); |
| if (unlikely(IS_ERR_OR_NULL(ctx))) { |
| dev_err(dev, "%s: Could not initialize context %p\n", |
| __func__, ctx); |
| rc = -ENODEV; |
| goto out; |
| } |
| |
| ctxid = cxl_process_element(ctx); |
| if (unlikely((ctxid > MAX_CONTEXT) || (ctxid < 0))) { |
| dev_err(dev, "%s: ctxid (%d) invalid!\n", __func__, ctxid); |
| rc = -EPERM; |
| goto err1; |
| } |
| |
| file = cxl_get_fd(ctx, &cfg->cxl_fops, &fd); |
| if (unlikely(fd < 0)) { |
| rc = -ENODEV; |
| dev_err(dev, "%s: Could not get file descriptor\n", __func__); |
| goto err1; |
| } |
| |
| rc = cxl_start_work(ctx, &ctxi->work); |
| if (unlikely(rc)) { |
| dev_dbg(dev, "%s: Could not start context rc=%d\n", |
| __func__, rc); |
| goto err2; |
| } |
| |
| /* Update with new MMIO area based on updated context id */ |
| ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl; |
| |
| rc = afu_attach(cfg, ctxi); |
| if (rc) { |
| dev_err(dev, "%s: Could not attach AFU rc %d\n", __func__, rc); |
| goto err3; |
| } |
| |
| /* |
| * No error paths after this point. Once the fd is installed it's |
| * visible to user space and can't be undone safely on this thread. |
| */ |
| old_fd = ctxi->lfd; |
| ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid); |
| ctxi->lfd = fd; |
| ctxi->ctx = ctx; |
| ctxi->file = file; |
| |
| /* |
| * Put context back in table (note the reinit of the context list); |
| * we must first drop the context's mutex and then acquire it in |
| * order with the table/list mutex to avoid a deadlock - safe to do |
| * here because no one can find us at this moment in time. |
| */ |
| mutex_unlock(&ctxi->mutex); |
| mutex_lock(&cfg->ctx_tbl_list_mutex); |
| mutex_lock(&ctxi->mutex); |
| list_del_init(&ctxi->list); |
| cfg->ctx_tbl[ctxid] = ctxi; |
| mutex_unlock(&cfg->ctx_tbl_list_mutex); |
| fd_install(fd, file); |
| |
| /* Release the original adapter fd and associated CXL resources */ |
| sys_close(old_fd); |
| out: |
| dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n", |
| __func__, ctxid, fd, rc); |
| return rc; |
| |
| err3: |
| cxl_stop_context(ctx); |
| err2: |
| fput(file); |
| put_unused_fd(fd); |
| err1: |
| cxl_release_context(ctx); |
| goto out; |
| } |
| |
| /** |
| * check_state() - checks and responds to the current adapter state |
| * @cfg: Internal structure associated with the host. |
| * |
| * This routine can block and should only be used on process context. |
| * Note that when waking up from waiting in limbo, the state is unknown |
| * and must be checked again before proceeding. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int check_state(struct cxlflash_cfg *cfg) |
| { |
| struct device *dev = &cfg->dev->dev; |
| int rc = 0; |
| |
| retry: |
| switch (cfg->state) { |
| case STATE_LIMBO: |
| dev_dbg(dev, "%s: Limbo, going to wait...\n", __func__); |
| rc = wait_event_interruptible(cfg->limbo_waitq, |
| cfg->state != STATE_LIMBO); |
| if (unlikely(rc)) |
| break; |
| goto retry; |
| case STATE_FAILTERM: |
| dev_dbg(dev, "%s: Failed/Terminating!\n", __func__); |
| rc = -ENODEV; |
| break; |
| default: |
| break; |
| } |
| |
| return rc; |
| } |
| |
| /** |
| * cxlflash_afu_recover() - initiates AFU recovery |
| * @sdev: SCSI device associated with LUN. |
| * @recover: Recover ioctl data structure. |
| * |
| * Only a single recovery is allowed at a time to avoid exhausting CXL |
| * resources (leading to recovery failure) in the event that we're up |
| * against the maximum number of contexts limit. For similar reasons, |
| * a context recovery is retried if there are multiple recoveries taking |
| * place at the same time and the failure was due to CXL services being |
| * unable to keep up. |
| * |
| * Because a user can detect an error condition before the kernel, it is |
| * quite possible for this routine to act as the kernel's EEH detection |
| * source (MMIO read of mbox_r). Because of this, there is a window of |
| * time where an EEH might have been detected but not yet 'serviced' |
| * (callback invoked, causing the device to enter limbo state). To avoid |
| * looping in this routine during that window, a 1 second sleep is in place |
| * between the time the MMIO failure is detected and the time a wait on the |
| * limbo wait queue is attempted via check_state(). |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int cxlflash_afu_recover(struct scsi_device *sdev, |
| struct dk_cxlflash_recover_afu *recover) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct llun_info *lli = sdev->hostdata; |
| struct afu *afu = cfg->afu; |
| struct ctx_info *ctxi = NULL; |
| struct mutex *mutex = &cfg->ctx_recovery_mutex; |
| u64 ctxid = DECODE_CTXID(recover->context_id), |
| rctxid = recover->context_id; |
| long reg; |
| int lretry = 20; /* up to 2 seconds */ |
| int rc = 0; |
| |
| atomic_inc(&cfg->recovery_threads); |
| rc = mutex_lock_interruptible(mutex); |
| if (rc) |
| goto out; |
| |
| dev_dbg(dev, "%s: reason 0x%016llX rctxid=%016llX\n", |
| __func__, recover->reason, rctxid); |
| |
| retry: |
| /* Ensure that this process is attached to the context */ |
| ctxi = get_context(cfg, rctxid, lli, CTX_CTRL_ERR_FALLBACK); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%llu)\n", __func__, ctxid); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| if (ctxi->err_recovery_active) { |
| retry_recover: |
| rc = recover_context(cfg, ctxi); |
| if (unlikely(rc)) { |
| dev_err(dev, "%s: Recovery failed for context %llu (rc=%d)\n", |
| __func__, ctxid, rc); |
| if ((rc == -ENODEV) && |
| ((atomic_read(&cfg->recovery_threads) > 1) || |
| (lretry--))) { |
| dev_dbg(dev, "%s: Going to try again!\n", |
| __func__); |
| mutex_unlock(mutex); |
| msleep(100); |
| rc = mutex_lock_interruptible(mutex); |
| if (rc) |
| goto out; |
| goto retry_recover; |
| } |
| |
| goto out; |
| } |
| |
| ctxi->err_recovery_active = false; |
| recover->context_id = ctxi->ctxid; |
| recover->adap_fd = ctxi->lfd; |
| recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea); |
| recover->hdr.return_flags |= |
| DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET; |
| goto out; |
| } |
| |
| /* Test if in error state */ |
| reg = readq_be(&afu->ctrl_map->mbox_r); |
| if (reg == -1) { |
| dev_dbg(dev, "%s: MMIO read fail! Wait for recovery...\n", |
| __func__); |
| mutex_unlock(&ctxi->mutex); |
| ctxi = NULL; |
| ssleep(1); |
| rc = check_state(cfg); |
| if (unlikely(rc)) |
| goto out; |
| goto retry; |
| } |
| |
| dev_dbg(dev, "%s: MMIO working, no recovery required!\n", __func__); |
| out: |
| if (likely(ctxi)) |
| put_context(ctxi); |
| mutex_unlock(mutex); |
| atomic_dec_if_positive(&cfg->recovery_threads); |
| return rc; |
| } |
| |
| /** |
| * process_sense() - evaluates and processes sense data |
| * @sdev: SCSI device associated with LUN. |
| * @verify: Verify ioctl data structure. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int process_sense(struct scsi_device *sdev, |
| struct dk_cxlflash_verify *verify) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct llun_info *lli = sdev->hostdata; |
| struct glun_info *gli = lli->parent; |
| u64 prev_lba = gli->max_lba; |
| struct scsi_sense_hdr sshdr = { 0 }; |
| int rc = 0; |
| |
| rc = scsi_normalize_sense((const u8 *)&verify->sense_data, |
| DK_CXLFLASH_VERIFY_SENSE_LEN, &sshdr); |
| if (!rc) { |
| dev_err(dev, "%s: Failed to normalize sense data!\n", __func__); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| switch (sshdr.sense_key) { |
| case NO_SENSE: |
| case RECOVERED_ERROR: |
| /* fall through */ |
| case NOT_READY: |
| break; |
| case UNIT_ATTENTION: |
| switch (sshdr.asc) { |
| case 0x29: /* Power on Reset or Device Reset */ |
| /* fall through */ |
| case 0x2A: /* Device settings/capacity changed */ |
| rc = read_cap16(sdev, lli); |
| if (rc) { |
| rc = -ENODEV; |
| break; |
| } |
| if (prev_lba != gli->max_lba) |
| dev_dbg(dev, "%s: Capacity changed old=%lld " |
| "new=%lld\n", __func__, prev_lba, |
| gli->max_lba); |
| break; |
| case 0x3F: /* Report LUNs changed, Rescan. */ |
| scsi_scan_host(cfg->host); |
| break; |
| default: |
| rc = -EIO; |
| break; |
| } |
| break; |
| default: |
| rc = -EIO; |
| break; |
| } |
| out: |
| dev_dbg(dev, "%s: sense_key %x asc %x ascq %x rc %d\n", __func__, |
| sshdr.sense_key, sshdr.asc, sshdr.ascq, rc); |
| return rc; |
| } |
| |
| /** |
| * cxlflash_disk_verify() - verifies a LUN is the same and handle size changes |
| * @sdev: SCSI device associated with LUN. |
| * @verify: Verify ioctl data structure. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int cxlflash_disk_verify(struct scsi_device *sdev, |
| struct dk_cxlflash_verify *verify) |
| { |
| int rc = 0; |
| struct ctx_info *ctxi = NULL; |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct llun_info *lli = sdev->hostdata; |
| struct glun_info *gli = lli->parent; |
| struct sisl_rht_entry *rhte = NULL; |
| res_hndl_t rhndl = verify->rsrc_handle; |
| u64 ctxid = DECODE_CTXID(verify->context_id), |
| rctxid = verify->context_id; |
| u64 last_lba = 0; |
| |
| dev_dbg(dev, "%s: ctxid=%llu rhndl=%016llX, hint=%016llX, " |
| "flags=%016llX\n", __func__, ctxid, verify->rsrc_handle, |
| verify->hint, verify->hdr.flags); |
| |
| ctxi = get_context(cfg, rctxid, lli, 0); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%llu)\n", __func__, ctxid); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| rhte = get_rhte(ctxi, rhndl, lli); |
| if (unlikely(!rhte)) { |
| dev_dbg(dev, "%s: Bad resource handle! (%d)\n", |
| __func__, rhndl); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| /* |
| * Look at the hint/sense to see if it requires us to redrive |
| * inquiry (i.e. the Unit attention is due to the WWN changing). |
| */ |
| if (verify->hint & DK_CXLFLASH_VERIFY_HINT_SENSE) { |
| rc = process_sense(sdev, verify); |
| if (unlikely(rc)) { |
| dev_err(dev, "%s: Failed to validate sense data (%d)\n", |
| __func__, rc); |
| goto out; |
| } |
| } |
| |
| switch (gli->mode) { |
| case MODE_PHYSICAL: |
| last_lba = gli->max_lba; |
| break; |
| case MODE_VIRTUAL: |
| /* Cast lxt_cnt to u64 for multiply to be treated as 64bit op */ |
| last_lba = ((u64)rhte->lxt_cnt * MC_CHUNK_SIZE * gli->blk_len); |
| last_lba /= CXLFLASH_BLOCK_SIZE; |
| last_lba--; |
| break; |
| default: |
| WARN(1, "Unsupported LUN mode!"); |
| } |
| |
| verify->last_lba = last_lba; |
| |
| out: |
| if (likely(ctxi)) |
| put_context(ctxi); |
| dev_dbg(dev, "%s: returning rc=%d llba=%llX\n", |
| __func__, rc, verify->last_lba); |
| return rc; |
| } |
| |
| /** |
| * decode_ioctl() - translates an encoded ioctl to an easily identifiable string |
| * @cmd: The ioctl command to decode. |
| * |
| * Return: A string identifying the decoded ioctl. |
| */ |
| static char *decode_ioctl(int cmd) |
| { |
| switch (cmd) { |
| case DK_CXLFLASH_ATTACH: |
| return __stringify_1(DK_CXLFLASH_ATTACH); |
| case DK_CXLFLASH_USER_DIRECT: |
| return __stringify_1(DK_CXLFLASH_USER_DIRECT); |
| case DK_CXLFLASH_USER_VIRTUAL: |
| return __stringify_1(DK_CXLFLASH_USER_VIRTUAL); |
| case DK_CXLFLASH_VLUN_RESIZE: |
| return __stringify_1(DK_CXLFLASH_VLUN_RESIZE); |
| case DK_CXLFLASH_RELEASE: |
| return __stringify_1(DK_CXLFLASH_RELEASE); |
| case DK_CXLFLASH_DETACH: |
| return __stringify_1(DK_CXLFLASH_DETACH); |
| case DK_CXLFLASH_VERIFY: |
| return __stringify_1(DK_CXLFLASH_VERIFY); |
| case DK_CXLFLASH_VLUN_CLONE: |
| return __stringify_1(DK_CXLFLASH_VLUN_CLONE); |
| case DK_CXLFLASH_RECOVER_AFU: |
| return __stringify_1(DK_CXLFLASH_RECOVER_AFU); |
| case DK_CXLFLASH_MANAGE_LUN: |
| return __stringify_1(DK_CXLFLASH_MANAGE_LUN); |
| } |
| |
| return "UNKNOWN"; |
| } |
| |
| /** |
| * cxlflash_disk_direct_open() - opens a direct (physical) disk |
| * @sdev: SCSI device associated with LUN. |
| * @arg: UDirect ioctl data structure. |
| * |
| * On successful return, the user is informed of the resource handle |
| * to be used to identify the direct lun and the size (in blocks) of |
| * the direct lun in last LBA format. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int cxlflash_disk_direct_open(struct scsi_device *sdev, void *arg) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct afu *afu = cfg->afu; |
| struct llun_info *lli = sdev->hostdata; |
| struct glun_info *gli = lli->parent; |
| |
| struct dk_cxlflash_udirect *pphys = (struct dk_cxlflash_udirect *)arg; |
| |
| u64 ctxid = DECODE_CTXID(pphys->context_id), |
| rctxid = pphys->context_id; |
| u64 lun_size = 0; |
| u64 last_lba = 0; |
| u64 rsrc_handle = -1; |
| u32 port = CHAN2PORT(sdev->channel); |
| |
| int rc = 0; |
| |
| struct ctx_info *ctxi = NULL; |
| struct sisl_rht_entry *rhte = NULL; |
| |
| pr_debug("%s: ctxid=%llu ls=0x%llx\n", __func__, ctxid, lun_size); |
| |
| rc = cxlflash_lun_attach(gli, MODE_PHYSICAL, false); |
| if (unlikely(rc)) { |
| dev_dbg(dev, "%s: Failed to attach to LUN! (PHYSICAL)\n", |
| __func__); |
| goto out; |
| } |
| |
| ctxi = get_context(cfg, rctxid, lli, 0); |
| if (unlikely(!ctxi)) { |
| dev_dbg(dev, "%s: Bad context! (%llu)\n", __func__, ctxid); |
| rc = -EINVAL; |
| goto err1; |
| } |
| |
| rhte = rhte_checkout(ctxi, lli); |
| if (unlikely(!rhte)) { |
| dev_dbg(dev, "%s: too many opens for this context\n", __func__); |
| rc = -EMFILE; /* too many opens */ |
| goto err1; |
| } |
| |
| rsrc_handle = (rhte - ctxi->rht_start); |
| |
| rht_format1(rhte, lli->lun_id[sdev->channel], ctxi->rht_perms, port); |
| cxlflash_afu_sync(afu, ctxid, rsrc_handle, AFU_LW_SYNC); |
| |
| last_lba = gli->max_lba; |
| pphys->hdr.return_flags = 0; |
| pphys->last_lba = last_lba; |
| pphys->rsrc_handle = rsrc_handle; |
| |
| out: |
| if (likely(ctxi)) |
| put_context(ctxi); |
| dev_dbg(dev, "%s: returning handle 0x%llx rc=%d llba %lld\n", |
| __func__, rsrc_handle, rc, last_lba); |
| return rc; |
| |
| err1: |
| cxlflash_lun_detach(gli); |
| goto out; |
| } |
| |
| /** |
| * ioctl_common() - common IOCTL handler for driver |
| * @sdev: SCSI device associated with LUN. |
| * @cmd: IOCTL command. |
| * |
| * Handles common fencing operations that are valid for multiple ioctls. Always |
| * allow through ioctls that are cleanup oriented in nature, even when operating |
| * in a failed/terminating state. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| static int ioctl_common(struct scsi_device *sdev, int cmd) |
| { |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct llun_info *lli = sdev->hostdata; |
| int rc = 0; |
| |
| if (unlikely(!lli)) { |
| dev_dbg(dev, "%s: Unknown LUN\n", __func__); |
| rc = -EINVAL; |
| goto out; |
| } |
| |
| rc = check_state(cfg); |
| if (unlikely(rc) && (cfg->state == STATE_FAILTERM)) { |
| switch (cmd) { |
| case DK_CXLFLASH_VLUN_RESIZE: |
| case DK_CXLFLASH_RELEASE: |
| case DK_CXLFLASH_DETACH: |
| dev_dbg(dev, "%s: Command override! (%d)\n", |
| __func__, rc); |
| rc = 0; |
| break; |
| } |
| } |
| out: |
| return rc; |
| } |
| |
| /** |
| * cxlflash_ioctl() - IOCTL handler for driver |
| * @sdev: SCSI device associated with LUN. |
| * @cmd: IOCTL command. |
| * @arg: Userspace ioctl data structure. |
| * |
| * Return: 0 on success, -errno on failure |
| */ |
| int cxlflash_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) |
| { |
| typedef int (*sioctl) (struct scsi_device *, void *); |
| |
| struct cxlflash_cfg *cfg = (struct cxlflash_cfg *)sdev->host->hostdata; |
| struct device *dev = &cfg->dev->dev; |
| struct afu *afu = cfg->afu; |
| struct dk_cxlflash_hdr *hdr; |
| char buf[sizeof(union cxlflash_ioctls)]; |
| size_t size = 0; |
| bool known_ioctl = false; |
| int idx; |
| int rc = 0; |
| struct Scsi_Host *shost = sdev->host; |
| sioctl do_ioctl = NULL; |
| |
| static const struct { |
| size_t size; |
| sioctl ioctl; |
| } ioctl_tbl[] = { /* NOTE: order matters here */ |
| {sizeof(struct dk_cxlflash_attach), (sioctl)cxlflash_disk_attach}, |
| {sizeof(struct dk_cxlflash_udirect), cxlflash_disk_direct_open}, |
| {sizeof(struct dk_cxlflash_release), (sioctl)cxlflash_disk_release}, |
| {sizeof(struct dk_cxlflash_detach), (sioctl)cxlflash_disk_detach}, |
| {sizeof(struct dk_cxlflash_verify), (sioctl)cxlflash_disk_verify}, |
| {sizeof(struct dk_cxlflash_recover_afu), (sioctl)cxlflash_afu_recover}, |
| {sizeof(struct dk_cxlflash_manage_lun), (sioctl)cxlflash_manage_lun}, |
| {sizeof(struct dk_cxlflash_uvirtual), cxlflash_disk_virtual_open}, |
| {sizeof(struct dk_cxlflash_resize), (sioctl)cxlflash_vlun_resize}, |
| {sizeof(struct dk_cxlflash_clone), (sioctl)cxlflash_disk_clone}, |
| }; |
| |
| /* Restrict command set to physical support only for internal LUN */ |
| if (afu->internal_lun) |
| switch (cmd) { |
| case DK_CXLFLASH_RELEASE: |
| case DK_CXLFLASH_USER_VIRTUAL: |
| case DK_CXLFLASH_VLUN_RESIZE: |
| case DK_CXLFLASH_VLUN_CLONE: |
| dev_dbg(dev, "%s: %s not supported for lun_mode=%d\n", |
| __func__, decode_ioctl(cmd), afu->internal_lun); |
| rc = -EINVAL; |
| goto cxlflash_ioctl_exit; |
| } |
| |
| switch (cmd) { |
| case DK_CXLFLASH_ATTACH: |
| case DK_CXLFLASH_USER_DIRECT: |
| case DK_CXLFLASH_RELEASE: |
| case DK_CXLFLASH_DETACH: |
| case DK_CXLFLASH_VERIFY: |
| case DK_CXLFLASH_RECOVER_AFU: |
| case DK_CXLFLASH_USER_VIRTUAL: |
| case DK_CXLFLASH_VLUN_RESIZE: |
| case DK_CXLFLASH_VLUN_CLONE: |
| dev_dbg(dev, "%s: %s (%08X) on dev(%d/%d/%d/%llu)\n", |
| __func__, decode_ioctl(cmd), cmd, shost->host_no, |
| sdev->channel, sdev->id, sdev->lun); |
| rc = ioctl_common(sdev, cmd); |
| if (unlikely(rc)) |
| goto cxlflash_ioctl_exit; |
| |
| /* fall through */ |
| |
| case DK_CXLFLASH_MANAGE_LUN: |
| known_ioctl = true; |
| idx = _IOC_NR(cmd) - _IOC_NR(DK_CXLFLASH_ATTACH); |
| size = ioctl_tbl[idx].size; |
| do_ioctl = ioctl_tbl[idx].ioctl; |
| |
| if (likely(do_ioctl)) |
| break; |
| |
| /* fall through */ |
| default: |
| rc = -EINVAL; |
| goto cxlflash_ioctl_exit; |
| } |
| |
| if (unlikely(copy_from_user(&buf, arg, size))) { |
| dev_err(dev, "%s: copy_from_user() fail! " |
| "size=%lu cmd=%d (%s) arg=%p\n", |
| __func__, size, cmd, decode_ioctl(cmd), arg); |
| rc = -EFAULT; |
| goto cxlflash_ioctl_exit; |
| } |
| |
| hdr = (struct dk_cxlflash_hdr *)&buf; |
| if (hdr->version != DK_CXLFLASH_VERSION_0) { |
| dev_dbg(dev, "%s: Version %u not supported for %s\n", |
| __func__, hdr->version, decode_ioctl(cmd)); |
| rc = -EINVAL; |
| goto cxlflash_ioctl_exit; |
| } |
| |
| if (hdr->rsvd[0] || hdr->rsvd[1] || hdr->rsvd[2] || hdr->return_flags) { |
| dev_dbg(dev, "%s: Reserved/rflags populated!\n", __func__); |
| rc = -EINVAL; |
| goto cxlflash_ioctl_exit; |
| } |
| |
| rc = do_ioctl(sdev, (void *)&buf); |
| if (likely(!rc)) |
| if (unlikely(copy_to_user(arg, &buf, size))) { |
| dev_err(dev, "%s: copy_to_user() fail! " |
| "size=%lu cmd=%d (%s) arg=%p\n", |
| __func__, size, cmd, decode_ioctl(cmd), arg); |
| rc = -EFAULT; |
| } |
| |
| /* fall through to exit */ |
| |
| cxlflash_ioctl_exit: |
| if (unlikely(rc && known_ioctl)) |
| dev_err(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) " |
| "returned rc %d\n", __func__, |
| decode_ioctl(cmd), cmd, shost->host_no, |
| sdev->channel, sdev->id, sdev->lun, rc); |
| else |
| dev_dbg(dev, "%s: ioctl %s (%08X) on dev(%d/%d/%d/%llu) " |
| "returned rc %d\n", __func__, decode_ioctl(cmd), |
| cmd, shost->host_no, sdev->channel, sdev->id, |
| sdev->lun, rc); |
| return rc; |
| } |