Lines Matching refs:table
204 struct hns_roce_hem_table *table, unsigned long *obj, in hns_roce_calc_hem_mhop() argument
213 if (get_hem_table_config(hr_dev, mhop, table->type)) in hns_roce_calc_hem_mhop()
223 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
225 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : in hns_roce_calc_hem_mhop()
227 table_idx = *obj / (chunk_size / table->obj_size); in hns_roce_calc_hem_mhop()
243 table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
337 struct hns_roce_hem_table *table, unsigned long obj, in calc_hem_config() argument
348 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); in calc_hem_config()
356 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in calc_hem_config()
373 table->type, mhop->hop_num); in calc_hem_config()
377 if (unlikely(index->buf >= table->num_hem)) { in calc_hem_config()
379 table->type, index->buf, table->num_hem); in calc_hem_config()
387 struct hns_roce_hem_table *table, in free_mhop_hem() argument
395 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
396 table->hem[index->buf] = NULL; in free_mhop_hem()
400 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], in free_mhop_hem()
401 table->bt_l1_dma_addr[index->l1]); in free_mhop_hem()
402 table->bt_l1[index->l1] = NULL; in free_mhop_hem()
406 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], in free_mhop_hem()
407 table->bt_l0_dma_addr[index->l0]); in free_mhop_hem()
408 table->bt_l0[index->l0] = NULL; in free_mhop_hem()
413 struct hns_roce_hem_table *table, in alloc_mhop_hem() argument
426 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || in alloc_mhop_hem()
427 check_whether_bt_num_2(table->type, mhop->hop_num)) && in alloc_mhop_hem()
428 !table->bt_l0[index->l0]) { in alloc_mhop_hem()
429 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
430 &table->bt_l0_dma_addr[index->l0], in alloc_mhop_hem()
432 if (!table->bt_l0[index->l0]) { in alloc_mhop_hem()
440 if (check_whether_bt_num_3(table->type, mhop->hop_num) && in alloc_mhop_hem()
441 !table->bt_l1[index->l1]) { in alloc_mhop_hem()
442 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
443 &table->bt_l1_dma_addr[index->l1], in alloc_mhop_hem()
445 if (!table->bt_l1[index->l1]) { in alloc_mhop_hem()
450 *(table->bt_l0[index->l0] + mhop->l1_idx) = in alloc_mhop_hem()
451 table->bt_l1_dma_addr[index->l1]; in alloc_mhop_hem()
458 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; in alloc_mhop_hem()
459 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; in alloc_mhop_hem()
460 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, in alloc_mhop_hem()
462 if (!table->hem[index->buf]) { in alloc_mhop_hem()
468 hns_roce_hem_first(table->hem[index->buf], &iter); in alloc_mhop_hem()
470 if (table->type < HEM_TYPE_MTT) { in alloc_mhop_hem()
472 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; in alloc_mhop_hem()
474 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
476 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
481 free_mhop_hem(hr_dev, table, mhop, index); in alloc_mhop_hem()
487 struct hns_roce_hem_table *table, unsigned long obj, in set_mhop_hem() argument
496 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); in set_mhop_hem()
504 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); in set_mhop_hem()
516 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); in set_mhop_hem()
525 struct hns_roce_hem_table *table, in hns_roce_table_mhop_get() argument
533 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
539 mutex_lock(&table->mutex); in hns_roce_table_mhop_get()
540 if (table->hem[index.buf]) { in hns_roce_table_mhop_get()
541 refcount_inc(&table->hem[index.buf]->refcount); in hns_roce_table_mhop_get()
545 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
552 if (table->type < HEM_TYPE_MTT) { in hns_roce_table_mhop_get()
553 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
560 refcount_set(&table->hem[index.buf]->refcount, 1); in hns_roce_table_mhop_get()
564 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
566 mutex_unlock(&table->mutex); in hns_roce_table_mhop_get()
571 struct hns_roce_hem_table *table, unsigned long obj) in hns_roce_table_get() argument
577 if (hns_roce_check_whether_mhop(hr_dev, table->type)) in hns_roce_table_get()
578 return hns_roce_table_mhop_get(hr_dev, table, obj); in hns_roce_table_get()
580 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_get()
582 mutex_lock(&table->mutex); in hns_roce_table_get()
584 if (table->hem[i]) { in hns_roce_table_get()
585 refcount_inc(&table->hem[i]->refcount); in hns_roce_table_get()
589 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
590 table->table_chunk_size >> PAGE_SHIFT, in hns_roce_table_get()
591 table->table_chunk_size, in hns_roce_table_get()
592 (table->lowmem ? GFP_KERNEL : in hns_roce_table_get()
594 if (!table->hem[i]) { in hns_roce_table_get()
600 if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) { in hns_roce_table_get()
601 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
602 table->hem[i] = NULL; in hns_roce_table_get()
608 refcount_set(&table->hem[i]->refcount, 1); in hns_roce_table_get()
610 mutex_unlock(&table->mutex); in hns_roce_table_get()
615 struct hns_roce_hem_table *table, unsigned long obj, in clear_mhop_hem() argument
626 if (check_whether_bt_num_2(table->type, hop_num)) { in clear_mhop_hem()
627 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
628 chunk_ba_num, table->num_hem)) in clear_mhop_hem()
630 } else if (check_whether_bt_num_3(table->type, hop_num)) { in clear_mhop_hem()
631 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
632 chunk_ba_num, table->num_hem)) { in clear_mhop_hem()
634 if (hns_roce_check_bt_null(table->bt_l1, index->l1, in clear_mhop_hem()
640 if (table->type < HEM_TYPE_MTT) { in clear_mhop_hem()
646 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) in clear_mhop_hem()
650 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) in clear_mhop_hem()
654 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) in clear_mhop_hem()
660 struct hns_roce_hem_table *table, in hns_roce_table_mhop_put() argument
669 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
676 mutex_lock(&table->mutex); in hns_roce_table_mhop_put()
677 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, in hns_roce_table_mhop_put()
678 &table->mutex)) in hns_roce_table_mhop_put()
681 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
682 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_put()
684 mutex_unlock(&table->mutex); in hns_roce_table_mhop_put()
688 struct hns_roce_hem_table *table, unsigned long obj) in hns_roce_table_put() argument
693 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_put()
694 hns_roce_table_mhop_put(hr_dev, table, obj, 1); in hns_roce_table_put()
698 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_put()
700 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, in hns_roce_table_put()
701 &table->mutex)) in hns_roce_table_put()
704 if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) in hns_roce_table_put()
707 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
708 table->hem[i] = NULL; in hns_roce_table_put()
710 mutex_unlock(&table->mutex); in hns_roce_table_put()
714 struct hns_roce_hem_table *table, in hns_roce_table_find() argument
729 if (!table->lowmem) in hns_roce_table_find()
732 mutex_lock(&table->mutex); in hns_roce_table_find()
734 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_find()
735 obj_per_chunk = table->table_chunk_size / table->obj_size; in hns_roce_table_find()
736 hem = table->hem[obj / obj_per_chunk]; in hns_roce_table_find()
738 dma_offset = offset = idx_offset * table->obj_size; in hns_roce_table_find()
742 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) in hns_roce_table_find()
753 hem = table->hem[hem_idx]; in hns_roce_table_find()
781 mutex_unlock(&table->mutex); in hns_roce_table_find()
786 struct hns_roce_hem_table *table, u32 type, in hns_roce_init_hem_table() argument
794 table->table_chunk_size = hr_dev->caps.chunk_sz; in hns_roce_init_hem_table()
795 obj_per_chunk = table->table_chunk_size / obj_size; in hns_roce_init_hem_table()
798 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); in hns_roce_init_hem_table()
799 if (!table->hem) in hns_roce_init_hem_table()
824 table->hem = kcalloc(num_hem, sizeof(*table->hem), in hns_roce_init_hem_table()
826 if (!table->hem) in hns_roce_init_hem_table()
833 table->bt_l1 = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
834 sizeof(*table->bt_l1), in hns_roce_init_hem_table()
836 if (!table->bt_l1) in hns_roce_init_hem_table()
839 table->bt_l1_dma_addr = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
840 sizeof(*table->bt_l1_dma_addr), in hns_roce_init_hem_table()
843 if (!table->bt_l1_dma_addr) in hns_roce_init_hem_table()
849 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), in hns_roce_init_hem_table()
851 if (!table->bt_l0) in hns_roce_init_hem_table()
854 table->bt_l0_dma_addr = kcalloc(num_bt_l0, in hns_roce_init_hem_table()
855 sizeof(*table->bt_l0_dma_addr), in hns_roce_init_hem_table()
857 if (!table->bt_l0_dma_addr) in hns_roce_init_hem_table()
862 table->type = type; in hns_roce_init_hem_table()
863 table->num_hem = num_hem; in hns_roce_init_hem_table()
864 table->obj_size = obj_size; in hns_roce_init_hem_table()
865 table->lowmem = use_lowmem; in hns_roce_init_hem_table()
866 mutex_init(&table->mutex); in hns_roce_init_hem_table()
871 kfree(table->bt_l0); in hns_roce_init_hem_table()
872 table->bt_l0 = NULL; in hns_roce_init_hem_table()
875 kfree(table->bt_l1_dma_addr); in hns_roce_init_hem_table()
876 table->bt_l1_dma_addr = NULL; in hns_roce_init_hem_table()
879 kfree(table->bt_l1); in hns_roce_init_hem_table()
880 table->bt_l1 = NULL; in hns_roce_init_hem_table()
883 kfree(table->hem); in hns_roce_init_hem_table()
884 table->hem = NULL; in hns_roce_init_hem_table()
891 struct hns_roce_hem_table *table) in hns_roce_cleanup_mhop_hem_table() argument
898 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) in hns_roce_cleanup_mhop_hem_table()
900 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : in hns_roce_cleanup_mhop_hem_table()
903 for (i = 0; i < table->num_hem; ++i) { in hns_roce_cleanup_mhop_hem_table()
904 obj = i * buf_chunk_size / table->obj_size; in hns_roce_cleanup_mhop_hem_table()
905 if (table->hem[i]) in hns_roce_cleanup_mhop_hem_table()
906 hns_roce_table_mhop_put(hr_dev, table, obj, 0); in hns_roce_cleanup_mhop_hem_table()
909 kfree(table->hem); in hns_roce_cleanup_mhop_hem_table()
910 table->hem = NULL; in hns_roce_cleanup_mhop_hem_table()
911 kfree(table->bt_l1); in hns_roce_cleanup_mhop_hem_table()
912 table->bt_l1 = NULL; in hns_roce_cleanup_mhop_hem_table()
913 kfree(table->bt_l1_dma_addr); in hns_roce_cleanup_mhop_hem_table()
914 table->bt_l1_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
915 kfree(table->bt_l0); in hns_roce_cleanup_mhop_hem_table()
916 table->bt_l0 = NULL; in hns_roce_cleanup_mhop_hem_table()
917 kfree(table->bt_l0_dma_addr); in hns_roce_cleanup_mhop_hem_table()
918 table->bt_l0_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
922 struct hns_roce_hem_table *table) in hns_roce_cleanup_hem_table() argument
927 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_cleanup_hem_table()
928 hns_roce_cleanup_mhop_hem_table(hr_dev, table); in hns_roce_cleanup_hem_table()
932 for (i = 0; i < table->num_hem; ++i) in hns_roce_cleanup_hem_table()
933 if (table->hem[i]) { in hns_roce_cleanup_hem_table()
934 if (hr_dev->hw->clear_hem(hr_dev, table, in hns_roce_cleanup_hem_table()
935 i * table->table_chunk_size / table->obj_size, 0)) in hns_roce_cleanup_hem_table()
938 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
941 kfree(table->hem); in hns_roce_cleanup_hem_table()
948 &hr_dev->srq_table.table); in hns_roce_cleanup_hem()
949 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); in hns_roce_cleanup_hem()