Lines Matching refs:wss
105 struct rvt_wss *wss = rdi->wss; in rvt_wss_exit() local
107 if (!wss) in rvt_wss_exit()
111 kfree(wss->entries); in rvt_wss_exit()
112 wss->entries = NULL; in rvt_wss_exit()
113 kfree(rdi->wss); in rvt_wss_exit()
114 rdi->wss = NULL; in rvt_wss_exit()
131 struct rvt_wss *wss; in rvt_wss_init() local
135 rdi->wss = NULL; in rvt_wss_init()
139 rdi->wss = kzalloc_node(sizeof(*rdi->wss), GFP_KERNEL, node); in rvt_wss_init()
140 if (!rdi->wss) in rvt_wss_init()
142 wss = rdi->wss; in rvt_wss_init()
166 wss->pages_mask = table_bits - 1; in rvt_wss_init()
167 wss->num_entries = table_bits / BITS_PER_LONG; in rvt_wss_init()
169 wss->threshold = (llc_bits * wss_threshold) / 100; in rvt_wss_init()
170 if (wss->threshold == 0) in rvt_wss_init()
171 wss->threshold = 1; in rvt_wss_init()
173 wss->clean_period = wss_clean_period; in rvt_wss_init()
174 atomic_set(&wss->clean_counter, wss_clean_period); in rvt_wss_init()
176 wss->entries = kcalloc_node(wss->num_entries, sizeof(*wss->entries), in rvt_wss_init()
178 if (!wss->entries) { in rvt_wss_init()
201 static void wss_advance_clean_counter(struct rvt_wss *wss) in wss_advance_clean_counter() argument
208 if (atomic_dec_and_test(&wss->clean_counter)) { in wss_advance_clean_counter()
222 atomic_set(&wss->clean_counter, wss->clean_period); in wss_advance_clean_counter()
230 entry = (atomic_inc_return(&wss->clean_entry) - 1) in wss_advance_clean_counter()
231 & (wss->num_entries - 1); in wss_advance_clean_counter()
234 bits = xchg(&wss->entries[entry], 0); in wss_advance_clean_counter()
238 atomic_sub(weight, &wss->total_count); in wss_advance_clean_counter()
245 static void wss_insert(struct rvt_wss *wss, void *address) in wss_insert() argument
247 u32 page = ((unsigned long)address >> PAGE_SHIFT) & wss->pages_mask; in wss_insert()
251 if (!test_and_set_bit(nr, &wss->entries[entry])) in wss_insert()
252 atomic_inc(&wss->total_count); in wss_insert()
254 wss_advance_clean_counter(wss); in wss_insert()
260 static inline bool wss_exceeds_threshold(struct rvt_wss *wss) in wss_exceeds_threshold() argument
262 return atomic_read(&wss->total_count) >= wss->threshold; in wss_exceeds_threshold()
2823 struct rvt_wss *wss = rdi->wss; in rvt_copy_sge() local
2835 wss_insert(wss, sge->vaddr); in rvt_copy_sge()
2837 wss_insert(wss, (sge->vaddr + PAGE_SIZE)); in rvt_copy_sge()
2839 cacheless_copy = wss_exceeds_threshold(wss); in rvt_copy_sge()
2841 wss_advance_clean_counter(wss); in rvt_copy_sge()