1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
3 #include <linux/libnvdimm.h>
4 #include <asm/unaligned.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/ndctl.h>
8 #include <linux/async.h>
9 #include <linux/slab.h>
10 #include "cxlmem.h"
11 #include "cxl.h"
12
13 /*
14 * Ordered workqueue for cxl nvdimm device arrival and departure
15 * to coordinate bus rescans when a bridge arrives and trigger remove
16 * operations when the bridge is removed.
17 */
18 static struct workqueue_struct *cxl_pmem_wq;
19
20 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
21
clear_exclusive(void * cxlm)22 static void clear_exclusive(void *cxlm)
23 {
24 clear_exclusive_cxl_commands(cxlm, exclusive_cmds);
25 }
26
unregister_nvdimm(void * nvdimm)27 static void unregister_nvdimm(void *nvdimm)
28 {
29 nvdimm_delete(nvdimm);
30 }
31
cxl_nvdimm_probe(struct device * dev)32 static int cxl_nvdimm_probe(struct device *dev)
33 {
34 struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
35 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
36 unsigned long flags = 0, cmd_mask = 0;
37 struct cxl_mem *cxlm = cxlmd->cxlm;
38 struct cxl_nvdimm_bridge *cxl_nvb;
39 struct nvdimm *nvdimm;
40 int rc;
41
42 cxl_nvb = cxl_find_nvdimm_bridge(cxl_nvd);
43 if (!cxl_nvb)
44 return -ENXIO;
45
46 device_lock(&cxl_nvb->dev);
47 if (!cxl_nvb->nvdimm_bus) {
48 rc = -ENXIO;
49 goto out;
50 }
51
52 set_exclusive_cxl_commands(cxlm, exclusive_cmds);
53 rc = devm_add_action_or_reset(dev, clear_exclusive, cxlm);
54 if (rc)
55 goto out;
56
57 set_bit(NDD_LABELING, &flags);
58 set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask);
59 set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask);
60 set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask);
61 nvdimm = nvdimm_create(cxl_nvb->nvdimm_bus, cxl_nvd, NULL, flags,
62 cmd_mask, 0, NULL);
63 if (!nvdimm) {
64 rc = -ENOMEM;
65 goto out;
66 }
67
68 dev_set_drvdata(dev, nvdimm);
69 rc = devm_add_action_or_reset(dev, unregister_nvdimm, nvdimm);
70 out:
71 device_unlock(&cxl_nvb->dev);
72 put_device(&cxl_nvb->dev);
73
74 return rc;
75 }
76
77 static struct cxl_driver cxl_nvdimm_driver = {
78 .name = "cxl_nvdimm",
79 .probe = cxl_nvdimm_probe,
80 .id = CXL_DEVICE_NVDIMM,
81 };
82
cxl_pmem_get_config_size(struct cxl_mem * cxlm,struct nd_cmd_get_config_size * cmd,unsigned int buf_len)83 static int cxl_pmem_get_config_size(struct cxl_mem *cxlm,
84 struct nd_cmd_get_config_size *cmd,
85 unsigned int buf_len)
86 {
87 if (sizeof(*cmd) > buf_len)
88 return -EINVAL;
89
90 *cmd = (struct nd_cmd_get_config_size) {
91 .config_size = cxlm->lsa_size,
92 .max_xfer = cxlm->payload_size,
93 };
94
95 return 0;
96 }
97
cxl_pmem_get_config_data(struct cxl_mem * cxlm,struct nd_cmd_get_config_data_hdr * cmd,unsigned int buf_len)98 static int cxl_pmem_get_config_data(struct cxl_mem *cxlm,
99 struct nd_cmd_get_config_data_hdr *cmd,
100 unsigned int buf_len)
101 {
102 struct cxl_mbox_get_lsa get_lsa;
103 int rc;
104
105 if (sizeof(*cmd) > buf_len)
106 return -EINVAL;
107 if (struct_size(cmd, out_buf, cmd->in_length) > buf_len)
108 return -EINVAL;
109
110 get_lsa = (struct cxl_mbox_get_lsa) {
111 .offset = cmd->in_offset,
112 .length = cmd->in_length,
113 };
114
115 rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LSA, &get_lsa,
116 sizeof(get_lsa), cmd->out_buf,
117 cmd->in_length);
118 cmd->status = 0;
119
120 return rc;
121 }
122
cxl_pmem_set_config_data(struct cxl_mem * cxlm,struct nd_cmd_set_config_hdr * cmd,unsigned int buf_len)123 static int cxl_pmem_set_config_data(struct cxl_mem *cxlm,
124 struct nd_cmd_set_config_hdr *cmd,
125 unsigned int buf_len)
126 {
127 struct cxl_mbox_set_lsa *set_lsa;
128 int rc;
129
130 if (sizeof(*cmd) > buf_len)
131 return -EINVAL;
132
133 /* 4-byte status follows the input data in the payload */
134 if (struct_size(cmd, in_buf, cmd->in_length) + 4 > buf_len)
135 return -EINVAL;
136
137 set_lsa =
138 kvzalloc(struct_size(set_lsa, data, cmd->in_length), GFP_KERNEL);
139 if (!set_lsa)
140 return -ENOMEM;
141
142 *set_lsa = (struct cxl_mbox_set_lsa) {
143 .offset = cmd->in_offset,
144 };
145 memcpy(set_lsa->data, cmd->in_buf, cmd->in_length);
146
147 rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_SET_LSA, set_lsa,
148 struct_size(set_lsa, data, cmd->in_length),
149 NULL, 0);
150
151 /*
152 * Set "firmware" status (4-packed bytes at the end of the input
153 * payload.
154 */
155 put_unaligned(0, (u32 *) &cmd->in_buf[cmd->in_length]);
156 kvfree(set_lsa);
157
158 return rc;
159 }
160
cxl_pmem_nvdimm_ctl(struct nvdimm * nvdimm,unsigned int cmd,void * buf,unsigned int buf_len)161 static int cxl_pmem_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd,
162 void *buf, unsigned int buf_len)
163 {
164 struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm);
165 unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm);
166 struct cxl_memdev *cxlmd = cxl_nvd->cxlmd;
167 struct cxl_mem *cxlm = cxlmd->cxlm;
168
169 if (!test_bit(cmd, &cmd_mask))
170 return -ENOTTY;
171
172 switch (cmd) {
173 case ND_CMD_GET_CONFIG_SIZE:
174 return cxl_pmem_get_config_size(cxlm, buf, buf_len);
175 case ND_CMD_GET_CONFIG_DATA:
176 return cxl_pmem_get_config_data(cxlm, buf, buf_len);
177 case ND_CMD_SET_CONFIG_DATA:
178 return cxl_pmem_set_config_data(cxlm, buf, buf_len);
179 default:
180 return -ENOTTY;
181 }
182 }
183
cxl_pmem_ctl(struct nvdimm_bus_descriptor * nd_desc,struct nvdimm * nvdimm,unsigned int cmd,void * buf,unsigned int buf_len,int * cmd_rc)184 static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
185 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
186 unsigned int buf_len, int *cmd_rc)
187 {
188 /*
189 * No firmware response to translate, let the transport error
190 * code take precedence.
191 */
192 *cmd_rc = 0;
193
194 if (!nvdimm)
195 return -ENOTTY;
196 return cxl_pmem_nvdimm_ctl(nvdimm, cmd, buf, buf_len);
197 }
198
online_nvdimm_bus(struct cxl_nvdimm_bridge * cxl_nvb)199 static bool online_nvdimm_bus(struct cxl_nvdimm_bridge *cxl_nvb)
200 {
201 if (cxl_nvb->nvdimm_bus)
202 return true;
203 cxl_nvb->nvdimm_bus =
204 nvdimm_bus_register(&cxl_nvb->dev, &cxl_nvb->nd_desc);
205 return cxl_nvb->nvdimm_bus != NULL;
206 }
207
cxl_nvdimm_release_driver(struct device * dev,void * data)208 static int cxl_nvdimm_release_driver(struct device *dev, void *data)
209 {
210 if (!is_cxl_nvdimm(dev))
211 return 0;
212 device_release_driver(dev);
213 return 0;
214 }
215
offline_nvdimm_bus(struct nvdimm_bus * nvdimm_bus)216 static void offline_nvdimm_bus(struct nvdimm_bus *nvdimm_bus)
217 {
218 if (!nvdimm_bus)
219 return;
220
221 /*
222 * Set the state of cxl_nvdimm devices to unbound / idle before
223 * nvdimm_bus_unregister() rips the nvdimm objects out from
224 * underneath them.
225 */
226 bus_for_each_dev(&cxl_bus_type, NULL, NULL, cxl_nvdimm_release_driver);
227 nvdimm_bus_unregister(nvdimm_bus);
228 }
229
cxl_nvb_update_state(struct work_struct * work)230 static void cxl_nvb_update_state(struct work_struct *work)
231 {
232 struct cxl_nvdimm_bridge *cxl_nvb =
233 container_of(work, typeof(*cxl_nvb), state_work);
234 struct nvdimm_bus *victim_bus = NULL;
235 bool release = false, rescan = false;
236
237 device_lock(&cxl_nvb->dev);
238 switch (cxl_nvb->state) {
239 case CXL_NVB_ONLINE:
240 if (!online_nvdimm_bus(cxl_nvb)) {
241 dev_err(&cxl_nvb->dev,
242 "failed to establish nvdimm bus\n");
243 release = true;
244 } else
245 rescan = true;
246 break;
247 case CXL_NVB_OFFLINE:
248 case CXL_NVB_DEAD:
249 victim_bus = cxl_nvb->nvdimm_bus;
250 cxl_nvb->nvdimm_bus = NULL;
251 break;
252 default:
253 break;
254 }
255 device_unlock(&cxl_nvb->dev);
256
257 if (release)
258 device_release_driver(&cxl_nvb->dev);
259 if (rescan) {
260 int rc = bus_rescan_devices(&cxl_bus_type);
261
262 dev_dbg(&cxl_nvb->dev, "rescan: %d\n", rc);
263 }
264 offline_nvdimm_bus(victim_bus);
265
266 put_device(&cxl_nvb->dev);
267 }
268
cxl_nvdimm_bridge_remove(struct device * dev)269 static void cxl_nvdimm_bridge_remove(struct device *dev)
270 {
271 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
272
273 if (cxl_nvb->state == CXL_NVB_ONLINE)
274 cxl_nvb->state = CXL_NVB_OFFLINE;
275 if (queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
276 get_device(&cxl_nvb->dev);
277 }
278
cxl_nvdimm_bridge_probe(struct device * dev)279 static int cxl_nvdimm_bridge_probe(struct device *dev)
280 {
281 struct cxl_nvdimm_bridge *cxl_nvb = to_cxl_nvdimm_bridge(dev);
282
283 if (cxl_nvb->state == CXL_NVB_DEAD)
284 return -ENXIO;
285
286 if (cxl_nvb->state == CXL_NVB_NEW) {
287 cxl_nvb->nd_desc = (struct nvdimm_bus_descriptor) {
288 .provider_name = "CXL",
289 .module = THIS_MODULE,
290 .ndctl = cxl_pmem_ctl,
291 };
292
293 INIT_WORK(&cxl_nvb->state_work, cxl_nvb_update_state);
294 }
295
296 cxl_nvb->state = CXL_NVB_ONLINE;
297 if (queue_work(cxl_pmem_wq, &cxl_nvb->state_work))
298 get_device(&cxl_nvb->dev);
299
300 return 0;
301 }
302
303 static struct cxl_driver cxl_nvdimm_bridge_driver = {
304 .name = "cxl_nvdimm_bridge",
305 .probe = cxl_nvdimm_bridge_probe,
306 .remove = cxl_nvdimm_bridge_remove,
307 .id = CXL_DEVICE_NVDIMM_BRIDGE,
308 };
309
cxl_pmem_init(void)310 static __init int cxl_pmem_init(void)
311 {
312 int rc;
313
314 set_bit(CXL_MEM_COMMAND_ID_SET_PARTITION_INFO, exclusive_cmds);
315 set_bit(CXL_MEM_COMMAND_ID_SET_SHUTDOWN_STATE, exclusive_cmds);
316 set_bit(CXL_MEM_COMMAND_ID_SET_LSA, exclusive_cmds);
317
318 cxl_pmem_wq = alloc_ordered_workqueue("cxl_pmem", 0);
319 if (!cxl_pmem_wq)
320 return -ENXIO;
321
322 rc = cxl_driver_register(&cxl_nvdimm_bridge_driver);
323 if (rc)
324 goto err_bridge;
325
326 rc = cxl_driver_register(&cxl_nvdimm_driver);
327 if (rc)
328 goto err_nvdimm;
329
330 return 0;
331
332 err_nvdimm:
333 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
334 err_bridge:
335 destroy_workqueue(cxl_pmem_wq);
336 return rc;
337 }
338
cxl_pmem_exit(void)339 static __exit void cxl_pmem_exit(void)
340 {
341 cxl_driver_unregister(&cxl_nvdimm_driver);
342 cxl_driver_unregister(&cxl_nvdimm_bridge_driver);
343 destroy_workqueue(cxl_pmem_wq);
344 }
345
346 MODULE_LICENSE("GPL v2");
347 module_init(cxl_pmem_init);
348 module_exit(cxl_pmem_exit);
349 MODULE_IMPORT_NS(CXL);
350 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM_BRIDGE);
351 MODULE_ALIAS_CXL(CXL_DEVICE_NVDIMM);
352