Lines Matching refs:rh
103 struct dm_region_hash *rh; /* FIXME: can we get rid of this ? */ member
117 static region_t dm_rh_sector_to_region(struct dm_region_hash *rh, sector_t sector) in dm_rh_sector_to_region() argument
119 return sector >> rh->region_shift; in dm_rh_sector_to_region()
122 sector_t dm_rh_region_to_sector(struct dm_region_hash *rh, region_t region) in dm_rh_region_to_sector() argument
124 return region << rh->region_shift; in dm_rh_region_to_sector()
128 region_t dm_rh_bio_to_region(struct dm_region_hash *rh, struct bio *bio) in dm_rh_bio_to_region() argument
130 return dm_rh_sector_to_region(rh, bio->bi_iter.bi_sector - in dm_rh_bio_to_region()
131 rh->target_begin); in dm_rh_bio_to_region()
137 return reg->rh->context; in dm_rh_region_context()
147 sector_t dm_rh_get_region_size(struct dm_region_hash *rh) in dm_rh_get_region_size() argument
149 return rh->region_size; in dm_rh_get_region_size()
170 struct dm_region_hash *rh; in dm_region_hash_create() local
184 rh = kzalloc(sizeof(*rh), GFP_KERNEL); in dm_region_hash_create()
185 if (!rh) { in dm_region_hash_create()
190 rh->context = context; in dm_region_hash_create()
191 rh->dispatch_bios = dispatch_bios; in dm_region_hash_create()
192 rh->wakeup_workers = wakeup_workers; in dm_region_hash_create()
193 rh->wakeup_all_recovery_waiters = wakeup_all_recovery_waiters; in dm_region_hash_create()
194 rh->target_begin = target_begin; in dm_region_hash_create()
195 rh->max_recovery = max_recovery; in dm_region_hash_create()
196 rh->log = log; in dm_region_hash_create()
197 rh->region_size = region_size; in dm_region_hash_create()
198 rh->region_shift = __ffs(region_size); in dm_region_hash_create()
199 rwlock_init(&rh->hash_lock); in dm_region_hash_create()
200 rh->mask = nr_buckets - 1; in dm_region_hash_create()
201 rh->nr_buckets = nr_buckets; in dm_region_hash_create()
203 rh->shift = RH_HASH_SHIFT; in dm_region_hash_create()
204 rh->prime = RH_HASH_MULT; in dm_region_hash_create()
206 rh->buckets = vmalloc(array_size(nr_buckets, sizeof(*rh->buckets))); in dm_region_hash_create()
207 if (!rh->buckets) { in dm_region_hash_create()
209 kfree(rh); in dm_region_hash_create()
214 INIT_LIST_HEAD(rh->buckets + i); in dm_region_hash_create()
216 spin_lock_init(&rh->region_lock); in dm_region_hash_create()
217 sema_init(&rh->recovery_count, 0); in dm_region_hash_create()
218 atomic_set(&rh->recovery_in_flight, 0); in dm_region_hash_create()
219 INIT_LIST_HEAD(&rh->clean_regions); in dm_region_hash_create()
220 INIT_LIST_HEAD(&rh->quiesced_regions); in dm_region_hash_create()
221 INIT_LIST_HEAD(&rh->recovered_regions); in dm_region_hash_create()
222 INIT_LIST_HEAD(&rh->failed_recovered_regions); in dm_region_hash_create()
223 rh->flush_failure = 0; in dm_region_hash_create()
225 ret = mempool_init_kmalloc_pool(&rh->region_pool, MIN_REGIONS, in dm_region_hash_create()
228 vfree(rh->buckets); in dm_region_hash_create()
229 kfree(rh); in dm_region_hash_create()
230 rh = ERR_PTR(-ENOMEM); in dm_region_hash_create()
233 return rh; in dm_region_hash_create()
237 void dm_region_hash_destroy(struct dm_region_hash *rh) in dm_region_hash_destroy() argument
242 BUG_ON(!list_empty(&rh->quiesced_regions)); in dm_region_hash_destroy()
243 for (h = 0; h < rh->nr_buckets; h++) { in dm_region_hash_destroy()
244 list_for_each_entry_safe(reg, nreg, rh->buckets + h, in dm_region_hash_destroy()
247 mempool_free(reg, &rh->region_pool); in dm_region_hash_destroy()
251 if (rh->log) in dm_region_hash_destroy()
252 dm_dirty_log_destroy(rh->log); in dm_region_hash_destroy()
254 mempool_exit(&rh->region_pool); in dm_region_hash_destroy()
255 vfree(rh->buckets); in dm_region_hash_destroy()
256 kfree(rh); in dm_region_hash_destroy()
260 struct dm_dirty_log *dm_rh_dirty_log(struct dm_region_hash *rh) in dm_rh_dirty_log() argument
262 return rh->log; in dm_rh_dirty_log()
266 static unsigned rh_hash(struct dm_region_hash *rh, region_t region) in rh_hash() argument
268 return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask; in rh_hash()
271 static struct dm_region *__rh_lookup(struct dm_region_hash *rh, region_t region) in __rh_lookup() argument
274 struct list_head *bucket = rh->buckets + rh_hash(rh, region); in __rh_lookup()
283 static void __rh_insert(struct dm_region_hash *rh, struct dm_region *reg) in __rh_insert() argument
285 list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key)); in __rh_insert()
288 static struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region) in __rh_alloc() argument
292 nreg = mempool_alloc(&rh->region_pool, GFP_ATOMIC); in __rh_alloc()
296 nreg->state = rh->log->type->in_sync(rh->log, region, 1) ? in __rh_alloc()
298 nreg->rh = rh; in __rh_alloc()
304 write_lock_irq(&rh->hash_lock); in __rh_alloc()
305 reg = __rh_lookup(rh, region); in __rh_alloc()
308 mempool_free(nreg, &rh->region_pool); in __rh_alloc()
310 __rh_insert(rh, nreg); in __rh_alloc()
312 spin_lock(&rh->region_lock); in __rh_alloc()
313 list_add(&nreg->list, &rh->clean_regions); in __rh_alloc()
314 spin_unlock(&rh->region_lock); in __rh_alloc()
319 write_unlock_irq(&rh->hash_lock); in __rh_alloc()
324 static struct dm_region *__rh_find(struct dm_region_hash *rh, region_t region) in __rh_find() argument
328 reg = __rh_lookup(rh, region); in __rh_find()
330 read_unlock(&rh->hash_lock); in __rh_find()
331 reg = __rh_alloc(rh, region); in __rh_find()
332 read_lock(&rh->hash_lock); in __rh_find()
338 int dm_rh_get_state(struct dm_region_hash *rh, region_t region, int may_block) in dm_rh_get_state() argument
343 read_lock(&rh->hash_lock); in dm_rh_get_state()
344 reg = __rh_lookup(rh, region); in dm_rh_get_state()
345 read_unlock(&rh->hash_lock); in dm_rh_get_state()
354 r = rh->log->type->in_sync(rh->log, region, may_block); in dm_rh_get_state()
366 struct dm_region_hash *rh = reg->rh; in complete_resync_work() local
368 rh->log->type->set_region_sync(rh->log, reg->key, success); in complete_resync_work()
379 rh->dispatch_bios(rh->context, ®->delayed_bios); in complete_resync_work()
380 if (atomic_dec_and_test(&rh->recovery_in_flight)) in complete_resync_work()
381 rh->wakeup_all_recovery_waiters(rh->context); in complete_resync_work()
382 up(&rh->recovery_count); in complete_resync_work()
395 void dm_rh_mark_nosync(struct dm_region_hash *rh, struct bio *bio) in dm_rh_mark_nosync() argument
398 struct dm_dirty_log *log = rh->log; in dm_rh_mark_nosync()
400 region_t region = dm_rh_bio_to_region(rh, bio); in dm_rh_mark_nosync()
404 rh->flush_failure = 1; in dm_rh_mark_nosync()
414 read_lock(&rh->hash_lock); in dm_rh_mark_nosync()
415 reg = __rh_find(rh, region); in dm_rh_mark_nosync()
416 read_unlock(&rh->hash_lock); in dm_rh_mark_nosync()
422 spin_lock_irqsave(&rh->region_lock, flags); in dm_rh_mark_nosync()
433 spin_unlock_irqrestore(&rh->region_lock, flags); in dm_rh_mark_nosync()
440 void dm_rh_update_states(struct dm_region_hash *rh, int errors_handled) in dm_rh_update_states() argument
451 write_lock_irq(&rh->hash_lock); in dm_rh_update_states()
452 spin_lock(&rh->region_lock); in dm_rh_update_states()
453 if (!list_empty(&rh->clean_regions)) { in dm_rh_update_states()
454 list_splice_init(&rh->clean_regions, &clean); in dm_rh_update_states()
460 if (!list_empty(&rh->recovered_regions)) { in dm_rh_update_states()
461 list_splice_init(&rh->recovered_regions, &recovered); in dm_rh_update_states()
467 if (!list_empty(&rh->failed_recovered_regions)) { in dm_rh_update_states()
468 list_splice_init(&rh->failed_recovered_regions, in dm_rh_update_states()
475 spin_unlock(&rh->region_lock); in dm_rh_update_states()
476 write_unlock_irq(&rh->hash_lock); in dm_rh_update_states()
484 rh->log->type->clear_region(rh->log, reg->key); in dm_rh_update_states()
486 mempool_free(reg, &rh->region_pool); in dm_rh_update_states()
491 mempool_free(reg, &rh->region_pool); in dm_rh_update_states()
495 rh->log->type->clear_region(rh->log, reg->key); in dm_rh_update_states()
496 mempool_free(reg, &rh->region_pool); in dm_rh_update_states()
499 rh->log->type->flush(rh->log); in dm_rh_update_states()
503 static void rh_inc(struct dm_region_hash *rh, region_t region) in rh_inc() argument
507 read_lock(&rh->hash_lock); in rh_inc()
508 reg = __rh_find(rh, region); in rh_inc()
510 spin_lock_irq(&rh->region_lock); in rh_inc()
516 spin_unlock_irq(&rh->region_lock); in rh_inc()
518 rh->log->type->mark_region(rh->log, reg->key); in rh_inc()
520 spin_unlock_irq(&rh->region_lock); in rh_inc()
523 read_unlock(&rh->hash_lock); in rh_inc()
526 void dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios) in dm_rh_inc_pending() argument
533 rh_inc(rh, dm_rh_bio_to_region(rh, bio)); in dm_rh_inc_pending()
538 void dm_rh_dec(struct dm_region_hash *rh, region_t region) in dm_rh_dec() argument
544 read_lock(&rh->hash_lock); in dm_rh_dec()
545 reg = __rh_lookup(rh, region); in dm_rh_dec()
546 read_unlock(&rh->hash_lock); in dm_rh_dec()
548 spin_lock_irqsave(&rh->region_lock, flags); in dm_rh_dec()
562 if (unlikely(rh->flush_failure)) { in dm_rh_dec()
570 list_add_tail(®->list, &rh->quiesced_regions); in dm_rh_dec()
573 list_add(®->list, &rh->clean_regions); in dm_rh_dec()
577 spin_unlock_irqrestore(&rh->region_lock, flags); in dm_rh_dec()
580 rh->wakeup_workers(rh->context); in dm_rh_dec()
587 static int __rh_recovery_prepare(struct dm_region_hash *rh) in __rh_recovery_prepare() argument
596 r = rh->log->type->get_resync_work(rh->log, ®ion); in __rh_recovery_prepare()
604 read_lock(&rh->hash_lock); in __rh_recovery_prepare()
605 reg = __rh_find(rh, region); in __rh_recovery_prepare()
606 read_unlock(&rh->hash_lock); in __rh_recovery_prepare()
608 spin_lock_irq(&rh->region_lock); in __rh_recovery_prepare()
615 list_move(®->list, &rh->quiesced_regions); in __rh_recovery_prepare()
617 spin_unlock_irq(&rh->region_lock); in __rh_recovery_prepare()
622 void dm_rh_recovery_prepare(struct dm_region_hash *rh) in dm_rh_recovery_prepare() argument
625 atomic_inc(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
627 while (!down_trylock(&rh->recovery_count)) { in dm_rh_recovery_prepare()
628 atomic_inc(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
629 if (__rh_recovery_prepare(rh) <= 0) { in dm_rh_recovery_prepare()
630 atomic_dec(&rh->recovery_in_flight); in dm_rh_recovery_prepare()
631 up(&rh->recovery_count); in dm_rh_recovery_prepare()
637 if (atomic_dec_and_test(&rh->recovery_in_flight)) in dm_rh_recovery_prepare()
638 rh->wakeup_all_recovery_waiters(rh->context); in dm_rh_recovery_prepare()
645 struct dm_region *dm_rh_recovery_start(struct dm_region_hash *rh) in dm_rh_recovery_start() argument
649 spin_lock_irq(&rh->region_lock); in dm_rh_recovery_start()
650 if (!list_empty(&rh->quiesced_regions)) { in dm_rh_recovery_start()
651 reg = list_entry(rh->quiesced_regions.next, in dm_rh_recovery_start()
655 spin_unlock_irq(&rh->region_lock); in dm_rh_recovery_start()
663 struct dm_region_hash *rh = reg->rh; in dm_rh_recovery_end() local
665 spin_lock_irq(&rh->region_lock); in dm_rh_recovery_end()
667 list_add(®->list, ®->rh->recovered_regions); in dm_rh_recovery_end()
669 list_add(®->list, ®->rh->failed_recovered_regions); in dm_rh_recovery_end()
671 spin_unlock_irq(&rh->region_lock); in dm_rh_recovery_end()
673 rh->wakeup_workers(rh->context); in dm_rh_recovery_end()
678 int dm_rh_recovery_in_flight(struct dm_region_hash *rh) in dm_rh_recovery_in_flight() argument
680 return atomic_read(&rh->recovery_in_flight); in dm_rh_recovery_in_flight()
684 int dm_rh_flush(struct dm_region_hash *rh) in dm_rh_flush() argument
686 return rh->log->type->flush(rh->log); in dm_rh_flush()
690 void dm_rh_delay(struct dm_region_hash *rh, struct bio *bio) in dm_rh_delay() argument
694 read_lock(&rh->hash_lock); in dm_rh_delay()
695 reg = __rh_find(rh, dm_rh_bio_to_region(rh, bio)); in dm_rh_delay()
697 read_unlock(&rh->hash_lock); in dm_rh_delay()
701 void dm_rh_stop_recovery(struct dm_region_hash *rh) in dm_rh_stop_recovery() argument
706 for (i = 0; i < rh->max_recovery; i++) in dm_rh_stop_recovery()
707 down(&rh->recovery_count); in dm_rh_stop_recovery()
711 void dm_rh_start_recovery(struct dm_region_hash *rh) in dm_rh_start_recovery() argument
715 for (i = 0; i < rh->max_recovery; i++) in dm_rh_start_recovery()
716 up(&rh->recovery_count); in dm_rh_start_recovery()
718 rh->wakeup_workers(rh->context); in dm_rh_start_recovery()