1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channel Host Bus Adapters.                                *
4  * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term *
5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  ********************************************************************/
23 
24 #include <linux/nvme.h>
25 #include <linux/nvme-fc-driver.h>
26 #include <linux/nvme-fc.h>
27 
28 #define LPFC_NVME_DEFAULT_SEGS		(64 + 1)	/* 256K IOs */
29 
30 #define LPFC_NVME_ERSP_LEN		0x20
31 
32 #define LPFC_NVME_WAIT_TMO              10
33 #define LPFC_NVME_EXPEDITE_XRICNT	8
34 #define LPFC_NVME_FB_SHIFT		9
35 #define LPFC_NVME_MAX_FB		(1 << 20)	/* 1M */
36 
37 #define lpfc_ndlp_get_nrport(ndlp)				\
38 	((!ndlp->nrport || (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT))\
39 	? NULL : ndlp->nrport)
40 
41 struct lpfc_nvme_qhandle {
42 	uint32_t index;		/* WQ index to use */
43 	uint32_t qidx;		/* queue index passed to create */
44 	uint32_t cpu_id;	/* current cpu id at time of create */
45 };
46 
47 /* Declare nvme-based local and remote port definitions. */
48 struct lpfc_nvme_lport {
49 	struct lpfc_vport *vport;
50 	struct completion *lport_unreg_cmp;
51 	/* Add stats counters here */
52 	atomic_t fc4NvmeLsRequests;
53 	atomic_t fc4NvmeLsCmpls;
54 	atomic_t xmt_fcp_noxri;
55 	atomic_t xmt_fcp_bad_ndlp;
56 	atomic_t xmt_fcp_qdepth;
57 	atomic_t xmt_fcp_wqerr;
58 	atomic_t xmt_fcp_err;
59 	atomic_t xmt_fcp_abort;
60 	atomic_t xmt_ls_abort;
61 	atomic_t xmt_ls_err;
62 	atomic_t cmpl_fcp_xb;
63 	atomic_t cmpl_fcp_err;
64 	atomic_t cmpl_ls_xb;
65 	atomic_t cmpl_ls_err;
66 };
67 
68 struct lpfc_nvme_rport {
69 	struct lpfc_nvme_lport *lport;
70 	struct nvme_fc_remote_port *remoteport;
71 	struct lpfc_nodelist *ndlp;
72 	struct completion rport_unreg_done;
73 };
74 
75 struct lpfc_nvme_fcpreq_priv {
76 	struct lpfc_io_buf *nvme_buf;
77 };
78 
79 /*
80  * set NVME LS request timeouts to 30s. It is larger than the 2*R_A_TOV
81  * set by the spec, which appears to have issues with some devices.
82  */
83 #define LPFC_NVME_LS_TIMEOUT		30
84 
85 
86 #define LPFC_NVMET_DEFAULT_SEGS		(64 + 1)	/* 256K IOs */
87 #define LPFC_NVMET_RQE_MIN_POST		128
88 #define LPFC_NVMET_RQE_DEF_POST		512
89 #define LPFC_NVMET_RQE_DEF_COUNT	2048
90 #define LPFC_NVMET_SUCCESS_LEN		12
91 
92 #define LPFC_NVMET_MRQ_AUTO		0
93 #define LPFC_NVMET_MRQ_MAX		16
94 
95 #define LPFC_NVMET_WAIT_TMO		(5 * MSEC_PER_SEC)
96 
97 /* Used for NVME Target */
98 #define LPFC_NVMET_INV_HOST_ACTIVE      1
99 
100 struct lpfc_nvmet_tgtport {
101 	struct lpfc_hba *phba;
102 	struct completion *tport_unreg_cmp;
103 	atomic_t state;		/* tracks nvmet hosthandle invalidation */
104 
105 	/* Stats counters - lpfc_nvmet_unsol_ls_buffer */
106 	atomic_t rcv_ls_req_in;
107 	atomic_t rcv_ls_req_out;
108 	atomic_t rcv_ls_req_drop;
109 	atomic_t xmt_ls_abort;
110 	atomic_t xmt_ls_abort_cmpl;
111 
112 	/* Stats counters - lpfc_nvmet_xmt_ls_rsp */
113 	atomic_t xmt_ls_rsp;
114 	atomic_t xmt_ls_drop;
115 
116 	/* Stats counters - lpfc_nvmet_xmt_ls_rsp_cmp */
117 	atomic_t xmt_ls_rsp_error;
118 	atomic_t xmt_ls_rsp_aborted;
119 	atomic_t xmt_ls_rsp_xb_set;
120 	atomic_t xmt_ls_rsp_cmpl;
121 
122 	/* Stats counters - lpfc_nvmet_unsol_fcp_buffer */
123 	atomic_t rcv_fcp_cmd_in;
124 	atomic_t rcv_fcp_cmd_out;
125 	atomic_t rcv_fcp_cmd_drop;
126 	atomic_t rcv_fcp_cmd_defer;
127 	atomic_t xmt_fcp_release;
128 
129 	/* Stats counters - lpfc_nvmet_xmt_fcp_op */
130 	atomic_t xmt_fcp_drop;
131 	atomic_t xmt_fcp_read_rsp;
132 	atomic_t xmt_fcp_read;
133 	atomic_t xmt_fcp_write;
134 	atomic_t xmt_fcp_rsp;
135 
136 	/* Stats counters - lpfc_nvmet_xmt_fcp_op_cmp */
137 	atomic_t xmt_fcp_rsp_xb_set;
138 	atomic_t xmt_fcp_rsp_cmpl;
139 	atomic_t xmt_fcp_rsp_error;
140 	atomic_t xmt_fcp_rsp_aborted;
141 	atomic_t xmt_fcp_rsp_drop;
142 
143 	/* Stats counters - lpfc_nvmet_xmt_fcp_abort */
144 	atomic_t xmt_fcp_xri_abort_cqe;
145 	atomic_t xmt_fcp_abort;
146 	atomic_t xmt_fcp_abort_cmpl;
147 	atomic_t xmt_abort_sol;
148 	atomic_t xmt_abort_unsol;
149 	atomic_t xmt_abort_rsp;
150 	atomic_t xmt_abort_rsp_error;
151 
152 	/* Stats counters - defer IO */
153 	atomic_t defer_ctx;
154 	atomic_t defer_fod;
155 	atomic_t defer_wqfull;
156 };
157 
158 struct lpfc_nvmet_ctx_info {
159 	struct list_head nvmet_ctx_list;
160 	spinlock_t	nvmet_ctx_list_lock; /* lock per CPU */
161 	struct lpfc_nvmet_ctx_info *nvmet_ctx_next_cpu;
162 	struct lpfc_nvmet_ctx_info *nvmet_ctx_start_cpu;
163 	uint16_t	nvmet_ctx_list_cnt;
164 	char pad[16];  /* pad to a cache-line */
165 };
166 
167 /* This retrieves the context info associated with the specified cpu / mrq */
168 #define lpfc_get_ctx_list(phba, cpu, mrq)  \
169 	(phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq))
170 
171 /* Values for state field of struct lpfc_async_xchg_ctx */
172 #define LPFC_NVME_STE_LS_RCV		1
173 #define LPFC_NVME_STE_LS_ABORT		2
174 #define LPFC_NVME_STE_LS_RSP		3
175 #define LPFC_NVME_STE_RCV		4
176 #define LPFC_NVME_STE_DATA		5
177 #define LPFC_NVME_STE_ABORT		6
178 #define LPFC_NVME_STE_DONE		7
179 #define LPFC_NVME_STE_FREE		0xff
180 
181 /* Values for flag field of struct lpfc_async_xchg_ctx */
182 #define LPFC_NVME_IO_INP		0x1  /* IO is in progress on exchange */
183 #define LPFC_NVME_ABORT_OP		0x2  /* Abort WQE issued on exchange */
184 #define LPFC_NVME_XBUSY			0x4  /* XB bit set on IO cmpl */
185 #define LPFC_NVME_CTX_RLS		0x8  /* ctx free requested */
186 #define LPFC_NVME_ABTS_RCV		0x10  /* ABTS received on exchange */
187 #define LPFC_NVME_CTX_REUSE_WQ		0x20  /* ctx reused via WQ */
188 #define LPFC_NVME_DEFER_WQFULL		0x40  /* Waiting on a free WQE */
189 #define LPFC_NVME_TNOTIFY		0x80  /* notify transport of abts */
190 
191 struct lpfc_async_xchg_ctx {
192 	union {
193 		struct nvmefc_tgt_fcp_req fcp_req;
194 	} hdlrctx;
195 	struct list_head list;
196 	struct lpfc_hba *phba;
197 	struct lpfc_nodelist *ndlp;
198 	struct nvmefc_ls_req *ls_req;
199 	struct nvmefc_ls_rsp ls_rsp;
200 	struct lpfc_iocbq *wqeq;
201 	struct lpfc_iocbq *abort_wqeq;
202 	spinlock_t ctxlock; /* protect flag access */
203 	uint32_t sid;
204 	uint32_t offset;
205 	uint16_t oxid;
206 	uint16_t size;
207 	uint16_t entry_cnt;
208 	uint16_t cpu;
209 	uint16_t idx;
210 	uint16_t state;
211 	uint16_t flag;
212 	void *payload;
213 	struct rqb_dmabuf *rqb_buffer;
214 	struct lpfc_nvmet_ctxbuf *ctxbuf;
215 	struct lpfc_sli4_hdw_queue *hdwq;
216 
217 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
218 	uint64_t ts_isr_cmd;
219 	uint64_t ts_cmd_nvme;
220 	uint64_t ts_nvme_data;
221 	uint64_t ts_data_wqput;
222 	uint64_t ts_isr_data;
223 	uint64_t ts_data_nvme;
224 	uint64_t ts_nvme_status;
225 	uint64_t ts_status_wqput;
226 	uint64_t ts_isr_status;
227 	uint64_t ts_status_nvme;
228 #endif
229 };
230 
231 
232 /* routines found in lpfc_nvme.c */
233 int __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
234 		struct nvmefc_ls_req *pnvme_lsreq,
235 		void (*gen_req_cmp)(struct lpfc_hba *phba,
236 				struct lpfc_iocbq *cmdwqe,
237 				struct lpfc_wcqe_complete *wcqe));
238 void __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba,  struct lpfc_vport *vport,
239 		struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe);
240 int __lpfc_nvme_ls_abort(struct lpfc_vport *vport,
241 		struct lpfc_nodelist *ndlp, struct nvmefc_ls_req *pnvme_lsreq);
242 
243 /* routines found in lpfc_nvmet.c */
244 int lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba,
245 			struct lpfc_async_xchg_ctx *ctxp, uint32_t sid,
246 			uint16_t xri);
247 int __lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg,
248 			struct nvmefc_ls_rsp *ls_rsp,
249 			void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba,
250 				struct lpfc_iocbq *cmdwqe,
251 				struct lpfc_wcqe_complete *wcqe));
252 void __lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba,
253 		struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe);
254