Lines Matching refs:dio
382 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1566 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in submit_flush_bio() argument
1572 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in submit_flush_bio()
1595 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) in do_endio_flush() argument
1597 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in do_endio_flush()
1599 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1600 submit_flush_bio(ic, dio); in do_endio_flush()
1605 static void dec_in_flight(struct dm_integrity_io *dio) in dec_in_flight() argument
1607 if (atomic_dec_and_test(&dio->in_flight)) { in dec_in_flight()
1608 struct dm_integrity_c *ic = dio->ic; in dec_in_flight()
1611 remove_range(ic, &dio->range); in dec_in_flight()
1613 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD)) in dec_in_flight()
1616 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in dec_in_flight()
1618 if (unlikely(dio->bi_status) && !bio->bi_status) in dec_in_flight()
1619 bio->bi_status = dio->bi_status; in dec_in_flight()
1620 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { in dec_in_flight()
1621 dio->range.logical_sector += dio->range.n_sectors; in dec_in_flight()
1622 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); in dec_in_flight()
1623 INIT_WORK(&dio->work, integrity_bio_wait); in dec_in_flight()
1624 queue_work(ic->offload_wq, &dio->work); in dec_in_flight()
1627 do_endio_flush(ic, dio); in dec_in_flight()
1633 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in integrity_end_io() local
1635 dm_bio_restore(&dio->bio_details, bio); in integrity_end_io()
1639 if (dio->completion) in integrity_end_io()
1640 complete(dio->completion); in integrity_end_io()
1642 dec_in_flight(dio); in integrity_end_io()
1700 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); in integrity_metadata() local
1701 struct dm_integrity_c *ic = dio->ic; in integrity_metadata()
1709 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in integrity_metadata()
1719 if (likely(dio->op != REQ_OP_DISCARD)) in integrity_metadata()
1733 if (unlikely(dio->op == REQ_OP_DISCARD)) { in integrity_metadata()
1734 sector_t bi_sector = dio->bio_details.bi_iter.bi_sector; in integrity_metadata()
1735 unsigned bi_size = dio->bio_details.bi_iter.bi_size; in integrity_metadata()
1743 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1765 sector = dio->range.logical_sector; in integrity_metadata()
1766 sectors_to_process = dio->range.n_sectors; in integrity_metadata()
1768 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { in integrity_metadata()
1785 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1786 checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE); in integrity_metadata()
1818 struct bio_integrity_payload *bip = dio->bio_details.bi_integrity; in integrity_metadata()
1823 unsigned data_to_process = dio->range.n_sectors; in integrity_metadata()
1834 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, in integrity_metadata()
1835 this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE); in integrity_metadata()
1845 dec_in_flight(dio); in integrity_metadata()
1848 dio->bi_status = errno_to_blk_status(r); in integrity_metadata()
1849 dec_in_flight(dio); in integrity_metadata()
1855 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in dm_integrity_map() local
1860 dio->ic = ic; in dm_integrity_map()
1861 dio->bi_status = 0; in dm_integrity_map()
1862 dio->op = bio_op(bio); in dm_integrity_map()
1864 if (unlikely(dio->op == REQ_OP_DISCARD)) { in dm_integrity_map()
1878 submit_flush_bio(ic, dio); in dm_integrity_map()
1882 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); in dm_integrity_map()
1883 dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA; in dm_integrity_map()
1884 if (unlikely(dio->fua)) { in dm_integrity_map()
1891 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_map()
1893 dio->range.logical_sector, bio_sectors(bio), in dm_integrity_map()
1897 …if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1… in dm_integrity_map()
1900 dio->range.logical_sector, bio_sectors(bio)); in dm_integrity_map()
1904 if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map()
1937 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) in dm_integrity_map()
1940 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in dm_integrity_map()
1941 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in dm_integrity_map()
1944 dm_integrity_map_continue(dio, true); in dm_integrity_map()
1948 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, in __journal_read_write() argument
1951 struct dm_integrity_c *ic = dio->ic; in __journal_read_write()
1955 logical_sector = dio->range.logical_sector; in __journal_read_write()
1956 n_sectors = dio->range.n_sectors; in __journal_read_write()
1967 if (likely(dio->op == REQ_OP_WRITE)) in __journal_read_write()
1973 if (unlikely(dio->op == REQ_OP_READ)) { in __journal_read_write()
2022 if (likely(dio->op == REQ_OP_WRITE)) in __journal_read_write()
2030 if (likely(dio->op == REQ_OP_WRITE)) in __journal_read_write()
2035 if (likely(dio->op == REQ_OP_WRITE)) { in __journal_read_write()
2071 if (unlikely(dio->op == REQ_OP_READ)) in __journal_read_write()
2076 if (likely(dio->op == REQ_OP_WRITE)) { in __journal_read_write()
2086 remove_range(ic, &dio->range); in __journal_read_write()
2092 dio->range.logical_sector = logical_sector; in __journal_read_write()
2093 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); in __journal_read_write()
2094 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); in __journal_read_write()
2101 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) in dm_integrity_map_continue() argument
2103 struct dm_integrity_c *ic = dio->ic; in dm_integrity_map_continue()
2104 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); in dm_integrity_map_continue()
2109 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ; in dm_integrity_map_continue()
2110 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') in dm_integrity_map_continue()
2114 INIT_WORK(&dio->work, integrity_bio_wait); in dm_integrity_map_continue()
2115 queue_work(ic->offload_wq, &dio->work); in dm_integrity_map_continue()
2127 dio->range.n_sectors = bio_sectors(bio); in dm_integrity_map_continue()
2129 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) { in dm_integrity_map_continue()
2130 if (dio->op == REQ_OP_WRITE) { in dm_integrity_map_continue()
2134 dio->range.n_sectors = min(dio->range.n_sectors, in dm_integrity_map_continue()
2136 if (unlikely(!dio->range.n_sectors)) { in dm_integrity_map_continue()
2142 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; in dm_integrity_map_continue()
2160 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); in dm_integrity_map_continue()
2174 } while ((i += ic->sectors_per_block) < dio->range.n_sectors); in dm_integrity_map_continue()
2180 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2182 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) in dm_integrity_map_continue()
2183 dio->range.n_sectors = next_sector - dio->range.logical_sector; in dm_integrity_map_continue()
2187 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { in dm_integrity_map_continue()
2188 if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) in dm_integrity_map_continue()
2191 dio->range.n_sectors = i; in dm_integrity_map_continue()
2195 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
2204 INIT_WORK(&dio->work, integrity_bio_wait); in dm_integrity_map_continue()
2205 queue_work(ic->wait_wq, &dio->work); in dm_integrity_map_continue()
2209 dio->range.n_sectors = ic->sectors_per_block; in dm_integrity_map_continue()
2210 wait_and_add_new_range(ic, &dio->range); in dm_integrity_map_continue()
2218 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2220 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2225 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) { in dm_integrity_map_continue()
2227 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); in dm_integrity_map_continue()
2229 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { in dm_integrity_map_continue()
2230 remove_range_unlocked(ic, &dio->range); in dm_integrity_map_continue()
2248 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { in dm_integrity_map_continue()
2249 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2250 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { in dm_integrity_map_continue()
2253 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); in dm_integrity_map_continue()
2262 dio->in_flight = (atomic_t)ATOMIC_INIT(2); in dm_integrity_map_continue()
2266 dio->completion = &read_comp; in dm_integrity_map_continue()
2268 dio->completion = NULL; in dm_integrity_map_continue()
2270 dm_bio_record(&dio->bio_details, bio); in dm_integrity_map_continue()
2275 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; in dm_integrity_map_continue()
2277 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { in dm_integrity_map_continue()
2278 integrity_metadata(&dio->work); in dm_integrity_map_continue()
2281 dio->in_flight = (atomic_t)ATOMIC_INIT(1); in dm_integrity_map_continue()
2282 dio->completion = NULL; in dm_integrity_map_continue()
2294 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) in dm_integrity_map_continue()
2297 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, in dm_integrity_map_continue()
2298 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) in dm_integrity_map_continue()
2303 integrity_metadata(&dio->work); in dm_integrity_map_continue()
2306 dec_in_flight(dio); in dm_integrity_map_continue()
2309 INIT_WORK(&dio->work, integrity_metadata); in dm_integrity_map_continue()
2310 queue_work(ic->metadata_wq, &dio->work); in dm_integrity_map_continue()
2316 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) in dm_integrity_map_continue()
2319 do_endio_flush(ic, dio); in dm_integrity_map_continue()
2325 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); in integrity_bio_wait() local
2327 dm_integrity_map_continue(dio, false); in integrity_bio_wait()
2781 struct dm_integrity_io *dio; in bitmap_block_work() local
2783 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in bitmap_block_work()
2785 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2786 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { in bitmap_block_work()
2787 remove_range(ic, &dio->range); in bitmap_block_work()
2788 INIT_WORK(&dio->work, integrity_bio_wait); in bitmap_block_work()
2789 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()
2791 block_bitmap_op(ic, ic->journal, dio->range.logical_sector, in bitmap_block_work()
2792 dio->range.n_sectors, BITMAP_OP_SET); in bitmap_block_work()
2805 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); in bitmap_block_work() local
2807 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, in bitmap_block_work()
2808 dio->range.n_sectors, BITMAP_OP_SET); in bitmap_block_work()
2810 remove_range(ic, &dio->range); in bitmap_block_work()
2811 INIT_WORK(&dio->work, integrity_bio_wait); in bitmap_block_work()
2812 queue_work(ic->offload_wq, &dio->work); in bitmap_block_work()