Lines Matching refs:dio
53 struct iomap_dio *dio, struct bio *bio, loff_t pos) in iomap_dio_submit_bio() argument
55 atomic_inc(&dio->ref); in iomap_dio_submit_bio()
57 if (dio->iocb->ki_flags & IOCB_HIPRI) { in iomap_dio_submit_bio()
58 bio_set_polled(bio, dio->iocb); in iomap_dio_submit_bio()
59 dio->submit.poll_bio = bio; in iomap_dio_submit_bio()
62 if (dio->dops && dio->dops->submit_io) in iomap_dio_submit_bio()
63 dio->dops->submit_io(iter, bio, pos); in iomap_dio_submit_bio()
68 ssize_t iomap_dio_complete(struct iomap_dio *dio) in iomap_dio_complete() argument
70 const struct iomap_dio_ops *dops = dio->dops; in iomap_dio_complete()
71 struct kiocb *iocb = dio->iocb; in iomap_dio_complete()
74 ssize_t ret = dio->error; in iomap_dio_complete()
77 ret = dops->end_io(iocb, dio->size, ret, dio->flags); in iomap_dio_complete()
80 ret = dio->size; in iomap_dio_complete()
82 if (offset + ret > dio->i_size && in iomap_dio_complete()
83 !(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_complete()
84 ret = dio->i_size - offset; in iomap_dio_complete()
100 if (!dio->error && dio->size && in iomap_dio_complete()
101 (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) { in iomap_dio_complete()
105 (offset + dio->size - 1) >> PAGE_SHIFT); in iomap_dio_complete()
115 if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) in iomap_dio_complete()
119 ret += dio->done_before; in iomap_dio_complete()
121 kfree(dio); in iomap_dio_complete()
129 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); in iomap_dio_complete_work() local
130 struct kiocb *iocb = dio->iocb; in iomap_dio_complete_work()
132 iocb->ki_complete(iocb, iomap_dio_complete(dio)); in iomap_dio_complete_work()
140 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret) in iomap_dio_set_error() argument
142 cmpxchg(&dio->error, 0, ret); in iomap_dio_set_error()
147 struct iomap_dio *dio = bio->bi_private; in iomap_dio_bio_end_io() local
148 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY); in iomap_dio_bio_end_io()
151 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status)); in iomap_dio_bio_end_io()
153 if (atomic_dec_and_test(&dio->ref)) { in iomap_dio_bio_end_io()
154 if (dio->wait_for_completion) { in iomap_dio_bio_end_io()
155 struct task_struct *waiter = dio->submit.waiter; in iomap_dio_bio_end_io()
156 WRITE_ONCE(dio->submit.waiter, NULL); in iomap_dio_bio_end_io()
158 } else if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_end_io()
159 struct inode *inode = file_inode(dio->iocb->ki_filp); in iomap_dio_bio_end_io()
161 WRITE_ONCE(dio->iocb->private, NULL); in iomap_dio_bio_end_io()
162 INIT_WORK(&dio->aio.work, iomap_dio_complete_work); in iomap_dio_bio_end_io()
163 queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work); in iomap_dio_bio_end_io()
165 WRITE_ONCE(dio->iocb->private, NULL); in iomap_dio_bio_end_io()
166 iomap_dio_complete_work(&dio->aio.work); in iomap_dio_bio_end_io()
178 static void iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio, in iomap_dio_zero() argument
188 bio->bi_private = dio; in iomap_dio_zero()
194 iomap_dio_submit_bio(iter, dio, bio, pos); in iomap_dio_zero()
202 static inline unsigned int iomap_dio_bio_opflags(struct iomap_dio *dio, in iomap_dio_bio_opflags() argument
207 if (!(dio->flags & IOMAP_DIO_WRITE)) { in iomap_dio_bio_opflags()
220 dio->flags &= ~IOMAP_DIO_WRITE_FUA; in iomap_dio_bio_opflags()
226 struct iomap_dio *dio) in iomap_dio_bio_iter() argument
232 unsigned int align = iov_iter_alignment(dio->submit.iter); in iomap_dio_bio_iter()
247 dio->flags |= IOMAP_DIO_UNWRITTEN; in iomap_dio_bio_iter()
252 dio->flags |= IOMAP_DIO_COW; in iomap_dio_bio_iter()
265 (dio->flags & IOMAP_DIO_WRITE_FUA) && in iomap_dio_bio_iter()
275 orig_count = iov_iter_count(dio->submit.iter); in iomap_dio_bio_iter()
276 iov_iter_truncate(dio->submit.iter, length); in iomap_dio_bio_iter()
278 if (!iov_iter_count(dio->submit.iter)) in iomap_dio_bio_iter()
285 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) in iomap_dio_bio_iter()
286 dio->iocb->ki_flags &= ~IOCB_HIPRI; in iomap_dio_bio_iter()
292 iomap_dio_zero(iter, dio, pos - pad, pad); in iomap_dio_bio_iter()
300 bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua); in iomap_dio_bio_iter()
302 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); in iomap_dio_bio_iter()
305 if (dio->error) { in iomap_dio_bio_iter()
306 iov_iter_revert(dio->submit.iter, copied); in iomap_dio_bio_iter()
314 bio->bi_write_hint = dio->iocb->ki_hint; in iomap_dio_bio_iter()
315 bio->bi_ioprio = dio->iocb->ki_ioprio; in iomap_dio_bio_iter()
316 bio->bi_private = dio; in iomap_dio_bio_iter()
320 ret = bio_iov_iter_get_pages(bio, dio->submit.iter); in iomap_dio_bio_iter()
333 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_bio_iter()
336 if (dio->flags & IOMAP_DIO_DIRTY) in iomap_dio_bio_iter()
340 dio->size += n; in iomap_dio_bio_iter()
343 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, in iomap_dio_bio_iter()
349 dio->iocb->ki_flags &= ~IOCB_HIPRI; in iomap_dio_bio_iter()
350 iomap_dio_submit_bio(iter, dio, bio, pos); in iomap_dio_bio_iter()
362 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) { in iomap_dio_bio_iter()
366 iomap_dio_zero(iter, dio, pos, fs_block_size - pad); in iomap_dio_bio_iter()
370 iov_iter_reexpand(dio->submit.iter, orig_count - copied); in iomap_dio_bio_iter()
377 struct iomap_dio *dio) in iomap_dio_hole_iter() argument
379 loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter); in iomap_dio_hole_iter()
381 dio->size += length; in iomap_dio_hole_iter()
388 struct iomap_dio *dio) in iomap_dio_inline_iter() argument
391 struct iov_iter *iter = dio->submit.iter; in iomap_dio_inline_iter()
400 if (dio->flags & IOMAP_DIO_WRITE) { in iomap_dio_inline_iter()
414 dio->size += copied; in iomap_dio_inline_iter()
421 struct iomap_dio *dio) in iomap_dio_iter() argument
425 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_iter()
427 return iomap_dio_hole_iter(iter, dio); in iomap_dio_iter()
429 if (!(dio->flags & IOMAP_DIO_WRITE)) in iomap_dio_iter()
430 return iomap_dio_hole_iter(iter, dio); in iomap_dio_iter()
431 return iomap_dio_bio_iter(iter, dio); in iomap_dio_iter()
433 return iomap_dio_bio_iter(iter, dio); in iomap_dio_iter()
435 return iomap_dio_inline_iter(iter, dio); in iomap_dio_iter()
444 dio->iocb->ki_filp, current->comm); in iomap_dio_iter()
489 struct iomap_dio *dio; in __iomap_dio_rw() local
494 dio = kmalloc(sizeof(*dio), GFP_KERNEL); in __iomap_dio_rw()
495 if (!dio) in __iomap_dio_rw()
498 dio->iocb = iocb; in __iomap_dio_rw()
499 atomic_set(&dio->ref, 1); in __iomap_dio_rw()
500 dio->size = 0; in __iomap_dio_rw()
501 dio->i_size = i_size_read(inode); in __iomap_dio_rw()
502 dio->dops = dops; in __iomap_dio_rw()
503 dio->error = 0; in __iomap_dio_rw()
504 dio->flags = 0; in __iomap_dio_rw()
505 dio->done_before = done_before; in __iomap_dio_rw()
507 dio->submit.iter = iter; in __iomap_dio_rw()
508 dio->submit.waiter = current; in __iomap_dio_rw()
509 dio->submit.poll_bio = NULL; in __iomap_dio_rw()
512 if (iomi.pos >= dio->i_size) in __iomap_dio_rw()
525 dio->flags |= IOMAP_DIO_DIRTY; in __iomap_dio_rw()
528 dio->flags |= IOMAP_DIO_WRITE; in __iomap_dio_rw()
540 dio->flags |= IOMAP_DIO_NEED_SYNC; in __iomap_dio_rw()
549 dio->flags |= IOMAP_DIO_WRITE_FUA; in __iomap_dio_rw()
554 if (iomi.pos >= dio->i_size || in __iomap_dio_rw()
555 iomi.pos + iomi.len > dio->i_size) in __iomap_dio_rw()
589 iomi.processed = iomap_dio_iter(&iomi, dio); in __iomap_dio_rw()
604 if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size) in __iomap_dio_rw()
605 iov_iter_revert(iter, iomi.pos - dio->i_size); in __iomap_dio_rw()
607 if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) { in __iomap_dio_rw()
619 iomap_dio_set_error(dio, ret); in __iomap_dio_rw()
625 if (dio->flags & IOMAP_DIO_WRITE_FUA) in __iomap_dio_rw()
626 dio->flags &= ~IOMAP_DIO_NEED_SYNC; in __iomap_dio_rw()
628 WRITE_ONCE(iocb->private, dio->submit.poll_bio); in __iomap_dio_rw()
645 dio->wait_for_completion = wait_for_completion; in __iomap_dio_rw()
646 if (!atomic_dec_and_test(&dio->ref)) { in __iomap_dio_rw()
652 if (!READ_ONCE(dio->submit.waiter)) in __iomap_dio_rw()
655 if (!dio->submit.poll_bio || in __iomap_dio_rw()
656 !bio_poll(dio->submit.poll_bio, NULL, 0)) in __iomap_dio_rw()
662 return dio; in __iomap_dio_rw()
665 kfree(dio); in __iomap_dio_rw()
677 struct iomap_dio *dio; in iomap_dio_rw() local
679 dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, done_before); in iomap_dio_rw()
680 if (IS_ERR_OR_NULL(dio)) in iomap_dio_rw()
681 return PTR_ERR_OR_ZERO(dio); in iomap_dio_rw()
682 return iomap_dio_complete(dio); in iomap_dio_rw()