Lines Matching refs:rings
332 struct io_rings *rings; member
1450 struct io_rings *r = ctx->rings; in io_account_cq_overflow()
1667 smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); in io_commit_cqring()
1672 struct io_rings *r = ctx->rings; in io_sqring_full()
1679 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); in __io_cqring_events()
1684 struct io_rings *rings = ctx->rings; in io_get_cqe() local
1696 return &rings->cqes[tail & mask]; in io_get_cqe()
1703 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) in io_should_trigger_evfd()
1772 WRITE_ONCE(ctx->rings->sq_flags, in __io_cqring_overflow_flush()
1773 ctx->rings->sq_flags & ~IORING_SQ_CQ_OVERFLOW); in __io_cqring_overflow_flush()
1850 WRITE_ONCE(ctx->rings->sq_flags, in io_cqring_event_overflow()
1851 ctx->rings->sq_flags | IORING_SQ_CQ_OVERFLOW); in io_cqring_event_overflow()
2460 struct io_rings *rings = ctx->rings; in io_sqring_entries() local
2463 return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; in io_sqring_entries()
7301 struct io_rings *rings = ctx->rings; in io_commit_sqring() local
7308 smp_store_release(&rings->sq.head, ctx->cached_sq_head); in io_commit_sqring()
7338 WRITE_ONCE(ctx->rings->sq_dropped, in io_get_sqe()
7339 READ_ONCE(ctx->rings->sq_dropped) + 1); in io_get_sqe()
7400 WRITE_ONCE(ctx->rings->sq_flags, in io_ring_set_wakeup_flag()
7401 ctx->rings->sq_flags | IORING_SQ_NEED_WAKEUP); in io_ring_set_wakeup_flag()
7408 WRITE_ONCE(ctx->rings->sq_flags, in io_ring_clear_wakeup_flag()
7409 ctx->rings->sq_flags & ~IORING_SQ_NEED_WAKEUP); in io_ring_clear_wakeup_flag()
7641 struct io_rings *rings = ctx->rings; in io_cqring_wait() local
7679 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events; in io_cqring_wait()
7697 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0; in io_cqring_wait()
8859 struct io_rings *rings; in rings_size() local
8862 off = struct_size(rings, cqes, cq_entries); in rings_size()
9357 if (ctx->rings) in io_ring_ctx_free()
9384 io_mem_free(ctx->rings); in io_ring_ctx_free()
9571 if (ctx->rings) in io_ring_ctx_wait_and_kill()
9911 ptr = ctx->rings; in io_uring_validate_mmap_request()
10151 struct io_rings *r = ctx->rings; in __io_uring_show_fdinfo()
10293 struct io_rings *rings; in io_allocate_scq_urings() local
10304 rings = io_mem_alloc(size); in io_allocate_scq_urings()
10305 if (!rings) in io_allocate_scq_urings()
10308 ctx->rings = rings; in io_allocate_scq_urings()
10309 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); in io_allocate_scq_urings()
10310 rings->sq_ring_mask = p->sq_entries - 1; in io_allocate_scq_urings()
10311 rings->cq_ring_mask = p->cq_entries - 1; in io_allocate_scq_urings()
10312 rings->sq_ring_entries = p->sq_entries; in io_allocate_scq_urings()
10313 rings->cq_ring_entries = p->cq_entries; in io_allocate_scq_urings()
10317 io_mem_free(ctx->rings); in io_allocate_scq_urings()
10318 ctx->rings = NULL; in io_allocate_scq_urings()
10324 io_mem_free(ctx->rings); in io_allocate_scq_urings()
10325 ctx->rings = NULL; in io_allocate_scq_urings()
10460 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings; in io_uring_create()