Lines Matching refs:mr
35 static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt) in populate_mtts() argument
38 int nsg = mr->nsg; in populate_mtts()
44 for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) { in populate_mtts()
47 nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size)) in populate_mtts()
52 static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr) in create_direct_mr() argument
59 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + roundup(MLX5_ST_SZ_BYTES(mtt) * mr->nsg, 16); in create_direct_mr()
66 MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO)); in create_direct_mr()
67 MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO)); in create_direct_mr()
71 MLX5_SET64(mkc, mkc, start_addr, mr->offset); in create_direct_mr()
72 MLX5_SET64(mkc, mkc, len, mr->end - mr->start); in create_direct_mr()
73 MLX5_SET(mkc, mkc, log_page_size, mr->log_size); in create_direct_mr()
75 get_octo_len(mr->end - mr->start, mr->log_size)); in create_direct_mr()
77 get_octo_len(mr->end - mr->start, mr->log_size)); in create_direct_mr()
78 populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt)); in create_direct_mr()
79 err = mlx5_vdpa_create_mkey(mvdev, &mr->mr, in, inlen); in create_direct_mr()
89 static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr) in destroy_direct_mr() argument
91 mlx5_vdpa_destroy_mkey(mvdev, mr->mr); in destroy_direct_mr()
94 static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr) in map_start() argument
96 return max_t(u64, map->start, mr->start); in map_start()
99 static u64 map_end(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr) in map_end() argument
101 return min_t(u64, map->last + 1, mr->end); in map_end()
104 static u64 maplen(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr) in maplen() argument
106 return map_end(map, mr) - map_start(map, mr); in maplen()
165 klm->key = cpu_to_be32(dmr->mr); in fill_indir()
182 static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr) in create_indirect_key() argument
191 start = indir_start_addr(mr); in create_indirect_key()
192 len = indir_len(mr); in create_indirect_key()
196 inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + klm_byte_size(mr->num_klms); in create_indirect_key()
210 MLX5_SET(mkc, mkc, translations_octword_size, klm_byte_size(mr->num_klms) / 16); in create_indirect_key()
211 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, mr->num_klms); in create_indirect_key()
212 fill_indir(mvdev, mr, in); in create_indirect_key()
213 err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen); in create_indirect_key()
223 static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr, in map_direct_mr() argument
240 for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1); in map_direct_mr()
241 map; map = vhost_iotlb_itree_next(map, start, mr->end - 1)) { in map_direct_mr()
242 size = maplen(map, mr); in map_direct_mr()
249 nsg = MLX5_DIV_ROUND_UP_POW2(mr->end - mr->start, log_entity_size); in map_direct_mr()
251 err = sg_alloc_table(&mr->sg_head, nsg, GFP_KERNEL); in map_direct_mr()
255 sg = mr->sg_head.sgl; in map_direct_mr()
256 for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1); in map_direct_mr()
257 map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) { in map_direct_mr()
258 paend = map->addr + maplen(map, mr); in map_direct_mr()
274 mr->log_size = log_entity_size; in map_direct_mr()
275 mr->nsg = nsg; in map_direct_mr()
276 mr->nent = dma_map_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0); in map_direct_mr()
277 if (!mr->nent) { in map_direct_mr()
282 err = create_direct_mr(mvdev, mr); in map_direct_mr()
289 dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0); in map_direct_mr()
291 sg_free_table(&mr->sg_head); in map_direct_mr()
295 static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr) in unmap_direct_mr() argument
299 destroy_direct_mr(mvdev, mr); in unmap_direct_mr()
300 dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0); in unmap_direct_mr()
301 sg_free_table(&mr->sg_head); in unmap_direct_mr()
307 struct mlx5_vdpa_mr *mr = &mvdev->mr; in add_direct_chain() local
336 mr->num_directs++; in add_direct_chain()
337 mr->num_klms++; in add_direct_chain()
341 list_splice_tail(&tmp, &mr->head); in add_direct_chain()
345 list_for_each_entry_safe(dmr, n, &mr->head, list) { in add_direct_chain()
361 struct mlx5_vdpa_mr *mr = &mvdev->mr; in create_user_mr() local
373 INIT_LIST_HEAD(&mr->head); in create_user_mr()
387 mr->num_klms += nnuls; in create_user_mr()
406 err = create_indirect_key(mvdev, mr); in create_user_mr()
410 mr->user_mr = true; in create_user_mr()
414 list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) { in create_user_mr()
422 static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr) in create_dma_mr() argument
442 err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen); in create_dma_mr()
444 mr->user_mr = false; in create_dma_mr()
450 static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr) in destroy_dma_mr() argument
452 mlx5_vdpa_destroy_mkey(mvdev, mr->mkey); in destroy_dma_mr()
481 static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr) in destroy_user_mr() argument
486 destroy_indirect_key(mvdev, mr); in destroy_user_mr()
487 list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) { in destroy_user_mr()
496 struct mlx5_vdpa_mr *mr = &mvdev->mr; in mlx5_vdpa_destroy_mr() local
498 mutex_lock(&mr->mkey_mtx); in mlx5_vdpa_destroy_mr()
499 if (!mr->initialized) in mlx5_vdpa_destroy_mr()
503 if (mr->user_mr) in mlx5_vdpa_destroy_mr()
504 destroy_user_mr(mvdev, mr); in mlx5_vdpa_destroy_mr()
506 destroy_dma_mr(mvdev, mr); in mlx5_vdpa_destroy_mr()
508 memset(mr, 0, sizeof(*mr)); in mlx5_vdpa_destroy_mr()
509 mr->initialized = false; in mlx5_vdpa_destroy_mr()
511 mutex_unlock(&mr->mkey_mtx); in mlx5_vdpa_destroy_mr()
516 struct mlx5_vdpa_mr *mr = &mvdev->mr; in _mlx5_vdpa_create_mr() local
519 if (mr->initialized) in _mlx5_vdpa_create_mr()
525 err = create_dma_mr(mvdev, mr); in _mlx5_vdpa_create_mr()
534 mr->initialized = true; in _mlx5_vdpa_create_mr()
539 destroy_user_mr(mvdev, mr); in _mlx5_vdpa_create_mr()
541 destroy_dma_mr(mvdev, mr); in _mlx5_vdpa_create_mr()
550 mutex_lock(&mvdev->mr.mkey_mtx); in mlx5_vdpa_create_mr()
552 mutex_unlock(&mvdev->mr.mkey_mtx); in mlx5_vdpa_create_mr()
559 struct mlx5_vdpa_mr *mr = &mvdev->mr; in mlx5_vdpa_handle_set_map() local
563 mutex_lock(&mr->mkey_mtx); in mlx5_vdpa_handle_set_map()
564 if (mr->initialized) { in mlx5_vdpa_handle_set_map()
570 mutex_unlock(&mr->mkey_mtx); in mlx5_vdpa_handle_set_map()