Lines Matching refs:dio
118 struct dio { struct
168 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
172 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
175 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
182 if (dio->page_errors == 0) in dio_refill_pages()
183 dio->page_errors = ret; in dio_refill_pages()
185 dio->pages[0] = page; in dio_refill_pages()
210 static inline struct page *dio_get_page(struct dio *dio, in dio_get_page() argument
216 ret = dio_refill_pages(dio, sdio); in dio_get_page()
221 return dio->pages[sdio->head]; in dio_get_page()
235 static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) in dio_complete() argument
237 loff_t offset = dio->iocb->ki_pos; in dio_complete()
250 if (dio->result) { in dio_complete()
251 transferred = dio->result; in dio_complete()
254 if ((dio->op == REQ_OP_READ) && in dio_complete()
255 ((offset + transferred) > dio->i_size)) in dio_complete()
256 transferred = dio->i_size - offset; in dio_complete()
263 ret = dio->page_errors; in dio_complete()
265 ret = dio->io_error; in dio_complete()
269 if (dio->end_io) { in dio_complete()
271 err = dio->end_io(dio->iocb, offset, ret, dio->private); in dio_complete()
289 ret > 0 && dio->op == REQ_OP_WRITE && in dio_complete()
290 dio->inode->i_mapping->nrpages) { in dio_complete()
291 err = invalidate_inode_pages2_range(dio->inode->i_mapping, in dio_complete()
295 dio_warn_stale_pagecache(dio->iocb->ki_filp); in dio_complete()
298 inode_dio_end(dio->inode); in dio_complete()
306 dio->iocb->ki_pos += transferred; in dio_complete()
308 if (ret > 0 && dio->op == REQ_OP_WRITE) in dio_complete()
309 ret = generic_write_sync(dio->iocb, ret); in dio_complete()
310 dio->iocb->ki_complete(dio->iocb, ret); in dio_complete()
313 kmem_cache_free(dio_cache, dio); in dio_complete()
319 struct dio *dio = container_of(work, struct dio, complete_work); in dio_aio_complete_work() local
321 dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE); in dio_aio_complete_work()
324 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio);
331 struct dio *dio = bio->bi_private; in dio_bio_end_aio() local
337 dio_bio_complete(dio, bio); in dio_bio_end_aio()
339 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_aio()
340 remaining = --dio->refcount; in dio_bio_end_aio()
341 if (remaining == 1 && dio->waiter) in dio_bio_end_aio()
342 wake_up_process(dio->waiter); in dio_bio_end_aio()
343 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_aio()
354 if (dio->result) in dio_bio_end_aio()
355 defer_completion = dio->defer_completion || in dio_bio_end_aio()
356 (dio->op == REQ_OP_WRITE && in dio_bio_end_aio()
357 dio->inode->i_mapping->nrpages); in dio_bio_end_aio()
359 INIT_WORK(&dio->complete_work, dio_aio_complete_work); in dio_bio_end_aio()
360 queue_work(dio->inode->i_sb->s_dio_done_wq, in dio_bio_end_aio()
361 &dio->complete_work); in dio_bio_end_aio()
363 dio_complete(dio, 0, DIO_COMPLETE_ASYNC); in dio_bio_end_aio()
377 struct dio *dio = bio->bi_private; in dio_bio_end_io() local
380 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_io()
381 bio->bi_private = dio->bio_list; in dio_bio_end_io()
382 dio->bio_list = bio; in dio_bio_end_io()
383 if (--dio->refcount == 1 && dio->waiter) in dio_bio_end_io()
384 wake_up_process(dio->waiter); in dio_bio_end_io()
385 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_io()
389 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
403 bio_set_op_attrs(bio, dio->op, dio->op_flags); in dio_bio_alloc()
404 if (dio->is_async) in dio_bio_alloc()
409 bio->bi_write_hint = dio->iocb->ki_hint; in dio_bio_alloc()
422 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
427 bio->bi_private = dio; in dio_bio_submit()
431 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_submit()
432 dio->refcount++; in dio_bio_submit()
433 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_submit()
435 if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) in dio_bio_submit()
438 dio->bio_disk = bio->bi_bdev->bd_disk; in dio_bio_submit()
441 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
453 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
456 put_page(dio->pages[sdio->head++]); in dio_cleanup()
465 static struct bio *dio_await_one(struct dio *dio) in dio_await_one() argument
470 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
478 while (dio->refcount > 1 && dio->bio_list == NULL) { in dio_await_one()
480 dio->waiter = current; in dio_await_one()
481 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
484 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
485 dio->waiter = NULL; in dio_await_one()
487 if (dio->bio_list) { in dio_await_one()
488 bio = dio->bio_list; in dio_await_one()
489 dio->bio_list = bio->bi_private; in dio_await_one()
491 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
498 static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) in dio_bio_complete() argument
501 bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty; in dio_bio_complete()
505 dio->io_error = -EAGAIN; in dio_bio_complete()
507 dio->io_error = -EIO; in dio_bio_complete()
510 if (dio->is_async && should_dirty) { in dio_bio_complete()
526 static void dio_await_completion(struct dio *dio) in dio_await_completion() argument
530 bio = dio_await_one(dio); in dio_await_completion()
532 dio_bio_complete(dio, bio); in dio_await_completion()
543 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
548 while (dio->bio_list) { in dio_bio_reap()
553 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_reap()
554 bio = dio->bio_list; in dio_bio_reap()
555 dio->bio_list = bio->bi_private; in dio_bio_reap()
556 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_reap()
557 ret2 = blk_status_to_errno(dio_bio_complete(dio, bio)); in dio_bio_reap()
590 static int dio_set_defer_completion(struct dio *dio) in dio_set_defer_completion() argument
592 struct super_block *sb = dio->inode->i_sb; in dio_set_defer_completion()
594 if (dio->defer_completion) in dio_set_defer_completion()
596 dio->defer_completion = true; in dio_set_defer_completion()
625 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
640 ret = dio->page_errors; in get_more_blocks()
662 create = dio->op == REQ_OP_WRITE; in get_more_blocks()
663 if (dio->flags & DIO_SKIP_HOLES) { in get_more_blocks()
664 i_size = i_size_read(dio->inode); in get_more_blocks()
669 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
673 dio->private = map_bh->b_private; in get_more_blocks()
676 ret = dio_set_defer_completion(dio); in get_more_blocks()
684 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
690 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
696 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
741 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
767 dio_bio_submit(dio, sdio); in dio_send_cur_page()
771 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
777 dio_bio_submit(dio, sdio); in dio_send_cur_page()
778 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
806 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
813 if (dio->op == REQ_OP_WRITE) { in submit_page_section()
835 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
854 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
856 dio_bio_submit(dio, sdio); in submit_page_section()
872 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
900 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
923 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
934 page = dio_get_page(dio, sdio); in do_direct_IO()
955 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
1001 if (dio->op == REQ_OP_WRITE) { in do_direct_IO()
1010 i_size_aligned = ALIGN(i_size_read(dio->inode), in do_direct_IO()
1021 dio->result += 1 << blkbits; in do_direct_IO()
1031 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1049 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1062 dio->result += this_chunk_bytes; in do_direct_IO()
1077 static inline int drop_refcount(struct dio *dio) in drop_refcount() argument
1093 spin_lock_irqsave(&dio->bio_lock, flags); in drop_refcount()
1094 ret2 = --dio->refcount; in drop_refcount()
1095 spin_unlock_irqrestore(&dio->bio_lock, flags); in drop_refcount()
1137 struct dio *dio; in do_blockdev_direct_IO() local
1152 dio = kmem_cache_alloc(dio_cache, GFP_KERNEL); in do_blockdev_direct_IO()
1153 if (!dio) in do_blockdev_direct_IO()
1160 memset(dio, 0, offsetof(struct dio, pages)); in do_blockdev_direct_IO()
1162 dio->flags = flags; in do_blockdev_direct_IO()
1163 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1169 dio->i_size = i_size_read(inode); in do_blockdev_direct_IO()
1170 if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { in do_blockdev_direct_IO()
1183 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1198 dio->is_async = false; in do_blockdev_direct_IO()
1200 dio->is_async = false; in do_blockdev_direct_IO()
1202 dio->is_async = true; in do_blockdev_direct_IO()
1204 dio->inode = inode; in do_blockdev_direct_IO()
1206 dio->op = REQ_OP_WRITE; in do_blockdev_direct_IO()
1207 dio->op_flags = REQ_SYNC | REQ_IDLE; in do_blockdev_direct_IO()
1209 dio->op_flags |= REQ_NOWAIT; in do_blockdev_direct_IO()
1211 dio->op = REQ_OP_READ; in do_blockdev_direct_IO()
1218 if (dio->is_async && iov_iter_rw(iter) == WRITE) { in do_blockdev_direct_IO()
1221 retval = dio_set_defer_completion(dio); in do_blockdev_direct_IO()
1222 else if (!dio->inode->i_sb->s_dio_done_wq) { in do_blockdev_direct_IO()
1228 retval = sb_init_dio_done_wq(dio->inode->i_sb); in do_blockdev_direct_IO()
1245 dio->end_io = end_io; in do_blockdev_direct_IO()
1250 dio->iocb = iocb; in do_blockdev_direct_IO()
1252 spin_lock_init(&dio->bio_lock); in do_blockdev_direct_IO()
1253 dio->refcount = 1; in do_blockdev_direct_IO()
1255 dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ; in do_blockdev_direct_IO()
1270 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1272 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1285 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1290 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1297 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1305 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1312 if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) in do_blockdev_direct_IO()
1313 inode_unlock(dio->inode); in do_blockdev_direct_IO()
1323 if (dio->is_async && retval == 0 && dio->result && in do_blockdev_direct_IO()
1324 (iov_iter_rw(iter) == READ || dio->result == count)) in do_blockdev_direct_IO()
1327 dio_await_completion(dio); in do_blockdev_direct_IO()
1329 if (drop_refcount(dio) == 0) { in do_blockdev_direct_IO()
1330 retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE); in do_blockdev_direct_IO()
1337 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) in do_blockdev_direct_IO()
1340 kmem_cache_free(dio_cache, dio); in do_blockdev_direct_IO()
1370 dio_cache = KMEM_CACHE(dio, SLAB_PANIC); in dio_init()