Lines Matching refs:wq
19 static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
66 static void free_hw_descs(struct idxd_wq *wq) in free_hw_descs() argument
70 for (i = 0; i < wq->num_descs; i++) in free_hw_descs()
71 kfree(wq->hw_descs[i]); in free_hw_descs()
73 kfree(wq->hw_descs); in free_hw_descs()
76 static int alloc_hw_descs(struct idxd_wq *wq, int num) in alloc_hw_descs() argument
78 struct device *dev = &wq->idxd->pdev->dev; in alloc_hw_descs()
82 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *), in alloc_hw_descs()
84 if (!wq->hw_descs) in alloc_hw_descs()
88 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]), in alloc_hw_descs()
90 if (!wq->hw_descs[i]) { in alloc_hw_descs()
91 free_hw_descs(wq); in alloc_hw_descs()
99 static void free_descs(struct idxd_wq *wq) in free_descs() argument
103 for (i = 0; i < wq->num_descs; i++) in free_descs()
104 kfree(wq->descs[i]); in free_descs()
106 kfree(wq->descs); in free_descs()
109 static int alloc_descs(struct idxd_wq *wq, int num) in alloc_descs() argument
111 struct device *dev = &wq->idxd->pdev->dev; in alloc_descs()
115 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *), in alloc_descs()
117 if (!wq->descs) in alloc_descs()
121 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]), in alloc_descs()
123 if (!wq->descs[i]) { in alloc_descs()
124 free_descs(wq); in alloc_descs()
133 int idxd_wq_alloc_resources(struct idxd_wq *wq) in idxd_wq_alloc_resources() argument
135 struct idxd_device *idxd = wq->idxd; in idxd_wq_alloc_resources()
139 if (wq->type != IDXD_WQT_KERNEL) in idxd_wq_alloc_resources()
142 num_descs = wq_dedicated(wq) ? wq->size : wq->threshold; in idxd_wq_alloc_resources()
143 wq->num_descs = num_descs; in idxd_wq_alloc_resources()
145 rc = alloc_hw_descs(wq, num_descs); in idxd_wq_alloc_resources()
149 wq->compls_size = num_descs * idxd->data->compl_size; in idxd_wq_alloc_resources()
150 wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL); in idxd_wq_alloc_resources()
151 if (!wq->compls) { in idxd_wq_alloc_resources()
156 rc = alloc_descs(wq, num_descs); in idxd_wq_alloc_resources()
160 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL, in idxd_wq_alloc_resources()
166 struct idxd_desc *desc = wq->descs[i]; in idxd_wq_alloc_resources()
168 desc->hw = wq->hw_descs[i]; in idxd_wq_alloc_resources()
170 desc->completion = &wq->compls[i]; in idxd_wq_alloc_resources()
172 desc->iax_completion = &wq->iax_compls[i]; in idxd_wq_alloc_resources()
173 desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i; in idxd_wq_alloc_resources()
175 desc->wq = wq; in idxd_wq_alloc_resources()
182 free_descs(wq); in idxd_wq_alloc_resources()
184 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); in idxd_wq_alloc_resources()
186 free_hw_descs(wq); in idxd_wq_alloc_resources()
190 void idxd_wq_free_resources(struct idxd_wq *wq) in idxd_wq_free_resources() argument
192 struct device *dev = &wq->idxd->pdev->dev; in idxd_wq_free_resources()
194 if (wq->type != IDXD_WQT_KERNEL) in idxd_wq_free_resources()
197 free_hw_descs(wq); in idxd_wq_free_resources()
198 free_descs(wq); in idxd_wq_free_resources()
199 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); in idxd_wq_free_resources()
200 sbitmap_queue_free(&wq->sbq); in idxd_wq_free_resources()
203 int idxd_wq_enable(struct idxd_wq *wq) in idxd_wq_enable() argument
205 struct idxd_device *idxd = wq->idxd; in idxd_wq_enable()
209 if (wq->state == IDXD_WQ_ENABLED) { in idxd_wq_enable()
210 dev_dbg(dev, "WQ %d already enabled\n", wq->id); in idxd_wq_enable()
214 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status); in idxd_wq_enable()
222 wq->state = IDXD_WQ_ENABLED; in idxd_wq_enable()
223 dev_dbg(dev, "WQ %d enabled\n", wq->id); in idxd_wq_enable()
227 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config) in idxd_wq_disable() argument
229 struct idxd_device *idxd = wq->idxd; in idxd_wq_disable()
233 dev_dbg(dev, "Disabling WQ %d\n", wq->id); in idxd_wq_disable()
235 if (wq->state != IDXD_WQ_ENABLED) { in idxd_wq_disable()
236 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); in idxd_wq_disable()
240 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); in idxd_wq_disable()
249 idxd_wq_disable_cleanup(wq); in idxd_wq_disable()
250 wq->state = IDXD_WQ_DISABLED; in idxd_wq_disable()
251 dev_dbg(dev, "WQ %d disabled\n", wq->id); in idxd_wq_disable()
255 void idxd_wq_drain(struct idxd_wq *wq) in idxd_wq_drain() argument
257 struct idxd_device *idxd = wq->idxd; in idxd_wq_drain()
261 if (wq->state != IDXD_WQ_ENABLED) { in idxd_wq_drain()
262 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); in idxd_wq_drain()
266 dev_dbg(dev, "Draining WQ %d\n", wq->id); in idxd_wq_drain()
267 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); in idxd_wq_drain()
271 void idxd_wq_reset(struct idxd_wq *wq) in idxd_wq_reset() argument
273 struct idxd_device *idxd = wq->idxd; in idxd_wq_reset()
277 if (wq->state != IDXD_WQ_ENABLED) { in idxd_wq_reset()
278 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state); in idxd_wq_reset()
282 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); in idxd_wq_reset()
284 idxd_wq_disable_cleanup(wq); in idxd_wq_reset()
285 wq->state = IDXD_WQ_DISABLED; in idxd_wq_reset()
288 int idxd_wq_map_portal(struct idxd_wq *wq) in idxd_wq_map_portal() argument
290 struct idxd_device *idxd = wq->idxd; in idxd_wq_map_portal()
296 start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED); in idxd_wq_map_portal()
298 wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE); in idxd_wq_map_portal()
299 if (!wq->portal) in idxd_wq_map_portal()
305 void idxd_wq_unmap_portal(struct idxd_wq *wq) in idxd_wq_unmap_portal() argument
307 struct device *dev = &wq->idxd->pdev->dev; in idxd_wq_unmap_portal()
309 devm_iounmap(dev, wq->portal); in idxd_wq_unmap_portal()
310 wq->portal = NULL; in idxd_wq_unmap_portal()
311 wq->portal_offset = 0; in idxd_wq_unmap_portal()
319 struct idxd_wq *wq = idxd->wqs[i]; in idxd_wqs_unmap_portal() local
321 if (wq->portal) in idxd_wqs_unmap_portal()
322 idxd_wq_unmap_portal(wq); in idxd_wqs_unmap_portal()
326 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) in idxd_wq_set_pasid() argument
328 struct idxd_device *idxd = wq->idxd; in idxd_wq_set_pasid()
333 rc = idxd_wq_disable(wq, false); in idxd_wq_set_pasid()
337 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); in idxd_wq_set_pasid()
345 rc = idxd_wq_enable(wq); in idxd_wq_set_pasid()
352 int idxd_wq_disable_pasid(struct idxd_wq *wq) in idxd_wq_disable_pasid() argument
354 struct idxd_device *idxd = wq->idxd; in idxd_wq_disable_pasid()
359 rc = idxd_wq_disable(wq, false); in idxd_wq_disable_pasid()
363 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); in idxd_wq_disable_pasid()
371 rc = idxd_wq_enable(wq); in idxd_wq_disable_pasid()
378 static void idxd_wq_disable_cleanup(struct idxd_wq *wq) in idxd_wq_disable_cleanup() argument
380 struct idxd_device *idxd = wq->idxd; in idxd_wq_disable_cleanup()
382 lockdep_assert_held(&wq->wq_lock); in idxd_wq_disable_cleanup()
383 memset(wq->wqcfg, 0, idxd->wqcfg_size); in idxd_wq_disable_cleanup()
384 wq->type = IDXD_WQT_NONE; in idxd_wq_disable_cleanup()
385 wq->size = 0; in idxd_wq_disable_cleanup()
386 wq->group = NULL; in idxd_wq_disable_cleanup()
387 wq->threshold = 0; in idxd_wq_disable_cleanup()
388 wq->priority = 0; in idxd_wq_disable_cleanup()
389 wq->ats_dis = 0; in idxd_wq_disable_cleanup()
390 clear_bit(WQ_FLAG_DEDICATED, &wq->flags); in idxd_wq_disable_cleanup()
391 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); in idxd_wq_disable_cleanup()
392 memset(wq->name, 0, WQ_NAME_SIZE); in idxd_wq_disable_cleanup()
397 struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active); in idxd_wq_ref_release() local
399 complete(&wq->wq_dead); in idxd_wq_ref_release()
402 int idxd_wq_init_percpu_ref(struct idxd_wq *wq) in idxd_wq_init_percpu_ref() argument
406 memset(&wq->wq_active, 0, sizeof(wq->wq_active)); in idxd_wq_init_percpu_ref()
407 rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, 0, GFP_KERNEL); in idxd_wq_init_percpu_ref()
410 reinit_completion(&wq->wq_dead); in idxd_wq_init_percpu_ref()
414 void idxd_wq_quiesce(struct idxd_wq *wq) in idxd_wq_quiesce() argument
416 percpu_ref_kill(&wq->wq_active); in idxd_wq_quiesce()
417 wait_for_completion(&wq->wq_dead); in idxd_wq_quiesce()
698 struct idxd_wq *wq = idxd->wqs[i]; in idxd_device_wqs_clear_state() local
700 if (wq->state == IDXD_WQ_ENABLED) { in idxd_device_wqs_clear_state()
701 idxd_wq_disable_cleanup(wq); in idxd_device_wqs_clear_state()
702 wq->state = IDXD_WQ_DISABLED; in idxd_device_wqs_clear_state()
811 static int idxd_wq_config_write(struct idxd_wq *wq) in idxd_wq_config_write() argument
813 struct idxd_device *idxd = wq->idxd; in idxd_wq_config_write()
818 if (!wq->group) in idxd_wq_config_write()
826 wq_offset = WQCFG_OFFSET(idxd, wq->id, i); in idxd_wq_config_write()
827 wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset); in idxd_wq_config_write()
831 wq->wqcfg->wq_size = wq->size; in idxd_wq_config_write()
833 if (wq->size == 0) { in idxd_wq_config_write()
840 wq->wqcfg->wq_thresh = wq->threshold; in idxd_wq_config_write()
843 if (wq_dedicated(wq)) in idxd_wq_config_write()
844 wq->wqcfg->mode = 1; in idxd_wq_config_write()
847 wq->wqcfg->pasid_en = 1; in idxd_wq_config_write()
848 if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq)) in idxd_wq_config_write()
849 wq->wqcfg->pasid = idxd->pasid; in idxd_wq_config_write()
863 wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); in idxd_wq_config_write()
864 if (wq_dedicated(wq) && wq->wqcfg->pasid_en && in idxd_wq_config_write()
866 wq->type == IDXD_WQT_KERNEL) { in idxd_wq_config_write()
871 wq->wqcfg->priority = wq->priority; in idxd_wq_config_write()
874 test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)) in idxd_wq_config_write()
875 wq->wqcfg->bof = 1; in idxd_wq_config_write()
878 wq->wqcfg->wq_ats_disable = wq->ats_dis; in idxd_wq_config_write()
881 wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); in idxd_wq_config_write()
882 wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); in idxd_wq_config_write()
884 dev_dbg(dev, "WQ %d CFGs\n", wq->id); in idxd_wq_config_write()
886 wq_offset = WQCFG_OFFSET(idxd, wq->id, i); in idxd_wq_config_write()
887 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset); in idxd_wq_config_write()
889 wq->id, i, wq_offset, in idxd_wq_config_write()
901 struct idxd_wq *wq = idxd->wqs[i]; in idxd_wqs_config_write() local
903 rc = idxd_wq_config_write(wq); in idxd_wqs_config_write()
967 struct idxd_wq *wq; in idxd_wqs_setup() local
979 wq = idxd->wqs[i]; in idxd_wqs_setup()
980 group = wq->group; in idxd_wqs_setup()
982 if (!wq->group) in idxd_wqs_setup()
984 if (!wq->size) in idxd_wqs_setup()
987 if (wq_shared(wq) && !device_swq_supported(idxd)) { in idxd_wqs_setup()
993 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64); in idxd_wqs_setup()
1031 static int idxd_wq_load_config(struct idxd_wq *wq) in idxd_wq_load_config() argument
1033 struct idxd_device *idxd = wq->idxd; in idxd_wq_load_config()
1038 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0); in idxd_wq_load_config()
1039 memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size); in idxd_wq_load_config()
1041 wq->size = wq->wqcfg->wq_size; in idxd_wq_load_config()
1042 wq->threshold = wq->wqcfg->wq_thresh; in idxd_wq_load_config()
1045 if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en) in idxd_wq_load_config()
1048 set_bit(WQ_FLAG_DEDICATED, &wq->flags); in idxd_wq_load_config()
1050 wq->priority = wq->wqcfg->priority; in idxd_wq_load_config()
1053 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i); in idxd_wq_load_config()
1054 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]); in idxd_wq_load_config()
1071 struct idxd_wq *wq; in idxd_group_load_config() local
1091 wq = idxd->wqs[id]; in idxd_group_load_config()
1092 wq->group = group; in idxd_group_load_config()
1135 struct idxd_wq *wq = idxd->wqs[i]; in idxd_device_load_config() local
1137 rc = idxd_wq_load_config(wq); in idxd_device_load_config()
1145 int __drv_enable_wq(struct idxd_wq *wq) in __drv_enable_wq() argument
1147 struct idxd_device *idxd = wq->idxd; in __drv_enable_wq()
1151 lockdep_assert_held(&wq->wq_lock); in __drv_enable_wq()
1158 if (wq->state != IDXD_WQ_DISABLED) { in __drv_enable_wq()
1159 dev_dbg(dev, "wq %d already enabled.\n", wq->id); in __drv_enable_wq()
1165 if (!wq->group) { in __drv_enable_wq()
1166 dev_dbg(dev, "wq %d not attached to group.\n", wq->id); in __drv_enable_wq()
1171 if (strlen(wq->name) == 0) { in __drv_enable_wq()
1173 dev_dbg(dev, "wq %d name not set.\n", wq->id); in __drv_enable_wq()
1178 if (wq_shared(wq)) { in __drv_enable_wq()
1192 if (wq->threshold == 0) { in __drv_enable_wq()
1205 dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); in __drv_enable_wq()
1209 rc = idxd_wq_enable(wq); in __drv_enable_wq()
1211 dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc); in __drv_enable_wq()
1215 rc = idxd_wq_map_portal(wq); in __drv_enable_wq()
1218 dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc); in __drv_enable_wq()
1222 wq->client_count = 0; in __drv_enable_wq()
1226 rc = idxd_wq_disable(wq, false); in __drv_enable_wq()
1228 dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq))); in __drv_enable_wq()
1233 int drv_enable_wq(struct idxd_wq *wq) in drv_enable_wq() argument
1237 mutex_lock(&wq->wq_lock); in drv_enable_wq()
1238 rc = __drv_enable_wq(wq); in drv_enable_wq()
1239 mutex_unlock(&wq->wq_lock); in drv_enable_wq()
1243 void __drv_disable_wq(struct idxd_wq *wq) in __drv_disable_wq() argument
1245 struct idxd_device *idxd = wq->idxd; in __drv_disable_wq()
1248 lockdep_assert_held(&wq->wq_lock); in __drv_disable_wq()
1250 if (idxd_wq_refcount(wq)) in __drv_disable_wq()
1252 wq->id, idxd_wq_refcount(wq)); in __drv_disable_wq()
1254 idxd_wq_unmap_portal(wq); in __drv_disable_wq()
1256 idxd_wq_drain(wq); in __drv_disable_wq()
1257 idxd_wq_reset(wq); in __drv_disable_wq()
1259 wq->client_count = 0; in __drv_disable_wq()
1262 void drv_disable_wq(struct idxd_wq *wq) in drv_disable_wq() argument
1264 mutex_lock(&wq->wq_lock); in drv_disable_wq()
1265 __drv_disable_wq(wq); in drv_disable_wq()
1266 mutex_unlock(&wq->wq_lock); in drv_disable_wq()
1316 struct idxd_wq *wq = idxd->wqs[i]; in idxd_device_drv_remove() local
1317 struct device *wq_dev = wq_confdev(wq); in idxd_device_drv_remove()
1319 if (wq->state == IDXD_WQ_DISABLED) in idxd_device_drv_remove()