Lines Matching refs:cpu_slab
299 raw_cpu_inc(s->cpu_slab->stat[si]); in stat()
2300 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); in note_cmpxchg_failure()
2326 c = per_cpu_ptr(s->cpu_slab, cpu); in init_kmem_cache_cpus()
2540 local_lock_irqsave(&s->cpu_slab->lock, flags); in unfreeze_partials()
2541 partial_page = this_cpu_read(s->cpu_slab->partial); in unfreeze_partials()
2542 this_cpu_write(s->cpu_slab->partial, NULL); in unfreeze_partials()
2543 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in unfreeze_partials()
2575 local_lock_irqsave(&s->cpu_slab->lock, flags); in put_cpu_partial()
2577 oldpage = this_cpu_read(s->cpu_slab->partial); in put_cpu_partial()
2598 this_cpu_write(s->cpu_slab->partial, page); in put_cpu_partial()
2600 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in put_cpu_partial()
2622 local_lock_irqsave(&s->cpu_slab->lock, flags); in flush_slab()
2631 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in flush_slab()
2641 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in __flush_cpu_slab()
2677 c = this_cpu_ptr(s->cpu_slab); in flush_cpu_slab()
2687 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in has_cpu_slab()
2860 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in get_freelist()
2946 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
2948 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2959 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2968 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in ___slab_alloc()
2978 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2983 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
2985 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2991 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
2997 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
2999 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3003 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3010 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3021 slub_put_cpu_ptr(s->cpu_slab); in ___slab_alloc()
3023 c = slub_get_cpu_ptr(s->cpu_slab); in ___slab_alloc()
3063 local_lock_irqsave(&s->cpu_slab->lock, flags); in ___slab_alloc()
3072 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in ___slab_alloc()
3106 c = slub_get_cpu_ptr(s->cpu_slab); in __slab_alloc()
3111 slub_put_cpu_ptr(s->cpu_slab); in __slab_alloc()
3169 c = raw_cpu_ptr(s->cpu_slab); in slab_alloc_node()
3219 s->cpu_slab->freelist, s->cpu_slab->tid, in slab_alloc_node()
3454 c = raw_cpu_ptr(s->cpu_slab); in do_slab_free()
3467 s->cpu_slab->freelist, s->cpu_slab->tid, in do_slab_free()
3484 local_lock(&s->cpu_slab->lock); in do_slab_free()
3485 c = this_cpu_ptr(s->cpu_slab); in do_slab_free()
3487 local_unlock(&s->cpu_slab->lock); in do_slab_free()
3497 local_unlock(&s->cpu_slab->lock); in do_slab_free()
3678 c = slub_get_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3679 local_lock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3700 local_unlock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3711 c = this_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3714 local_lock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3723 local_unlock_irq(&s->cpu_slab->lock); in kmem_cache_alloc_bulk()
3724 slub_put_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3734 slub_put_cpu_ptr(s->cpu_slab); in kmem_cache_alloc_bulk()
3902 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), in alloc_kmem_cache_cpus()
3905 if (!s->cpu_slab) in alloc_kmem_cache_cpus()
3974 free_percpu(s->cpu_slab); in __kmem_cache_release()
5240 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, in show_slab_objects()
5477 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5491 page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
5666 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; in show_stat()
5692 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; in clear_stat()