Lines Matching refs:bm
581 static void memory_bm_position_reset(struct memory_bitmap *bm) in memory_bm_position_reset() argument
583 bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree, in memory_bm_position_reset()
585 bm->cur.node = list_entry(bm->cur.zone->leaves.next, in memory_bm_position_reset()
587 bm->cur.node_pfn = 0; in memory_bm_position_reset()
588 bm->cur.node_bit = 0; in memory_bm_position_reset()
591 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
676 static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, in memory_bm_create() argument
685 INIT_LIST_HEAD(&bm->zones); in memory_bm_create()
700 list_add_tail(&zone->list, &bm->zones); in memory_bm_create()
703 bm->p_list = ca.chain; in memory_bm_create()
704 memory_bm_position_reset(bm); in memory_bm_create()
710 bm->p_list = ca.chain; in memory_bm_create()
711 memory_bm_free(bm, PG_UNSAFE_CLEAR); in memory_bm_create()
719 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) in memory_bm_free() argument
723 list_for_each_entry(zone, &bm->zones, list) in memory_bm_free()
726 free_list_of_pages(bm->p_list, clear_nosave_free); in memory_bm_free()
728 INIT_LIST_HEAD(&bm->zones); in memory_bm_free()
740 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn, in memory_bm_find_bit() argument
747 zone = bm->cur.zone; in memory_bm_find_bit()
755 list_for_each_entry(curr, &bm->zones, list) { in memory_bm_find_bit()
776 node = bm->cur.node; in memory_bm_find_bit()
777 if (zone == bm->cur.zone && in memory_bm_find_bit()
778 ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn) in memory_bm_find_bit()
795 bm->cur.zone = zone; in memory_bm_find_bit()
796 bm->cur.node = node; in memory_bm_find_bit()
797 bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK; in memory_bm_find_bit()
806 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn) in memory_bm_set_bit() argument
812 error = memory_bm_find_bit(bm, pfn, &addr, &bit); in memory_bm_set_bit()
817 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) in mem_bm_set_bit_check() argument
823 error = memory_bm_find_bit(bm, pfn, &addr, &bit); in mem_bm_set_bit_check()
830 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn) in memory_bm_clear_bit() argument
836 error = memory_bm_find_bit(bm, pfn, &addr, &bit); in memory_bm_clear_bit()
841 static void memory_bm_clear_current(struct memory_bitmap *bm) in memory_bm_clear_current() argument
845 bit = max(bm->cur.node_bit - 1, 0); in memory_bm_clear_current()
846 clear_bit(bit, bm->cur.node->data); in memory_bm_clear_current()
849 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) in memory_bm_test_bit() argument
855 error = memory_bm_find_bit(bm, pfn, &addr, &bit); in memory_bm_test_bit()
860 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) in memory_bm_pfn_present() argument
865 return !memory_bm_find_bit(bm, pfn, &addr, &bit); in memory_bm_pfn_present()
878 static bool rtree_next_node(struct memory_bitmap *bm) in rtree_next_node() argument
880 if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) { in rtree_next_node()
881 bm->cur.node = list_entry(bm->cur.node->list.next, in rtree_next_node()
883 bm->cur.node_pfn += BM_BITS_PER_BLOCK; in rtree_next_node()
884 bm->cur.node_bit = 0; in rtree_next_node()
890 if (!list_is_last(&bm->cur.zone->list, &bm->zones)) { in rtree_next_node()
891 bm->cur.zone = list_entry(bm->cur.zone->list.next, in rtree_next_node()
893 bm->cur.node = list_entry(bm->cur.zone->leaves.next, in rtree_next_node()
895 bm->cur.node_pfn = 0; in rtree_next_node()
896 bm->cur.node_bit = 0; in rtree_next_node()
915 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm) in memory_bm_next_pfn() argument
921 pages = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn; in memory_bm_next_pfn()
922 bits = min(pages - bm->cur.node_pfn, BM_BITS_PER_BLOCK); in memory_bm_next_pfn()
923 bit = find_next_bit(bm->cur.node->data, bits, in memory_bm_next_pfn()
924 bm->cur.node_bit); in memory_bm_next_pfn()
926 pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit; in memory_bm_next_pfn()
927 bm->cur.node_bit = bit + 1; in memory_bm_next_pfn()
930 } while (rtree_next_node(bm)); in memory_bm_next_pfn()
958 static void memory_bm_recycle(struct memory_bitmap *bm) in memory_bm_recycle() argument
963 list_for_each_entry(zone, &bm->zones, list) in memory_bm_recycle()
966 p_list = bm->p_list; in memory_bm_recycle()
1076 static void mark_nosave_pages(struct memory_bitmap *bm) in mark_nosave_pages() argument
1099 mem_bm_set_bit_check(bm, pfn); in mark_nosave_pages()
1191 struct memory_bitmap *bm = free_pages_map; in clear_or_poison_free_pages() local
1198 memory_bm_position_reset(bm); in clear_or_poison_free_pages()
1199 pfn = memory_bm_next_pfn(bm); in clear_or_poison_free_pages()
1204 pfn = memory_bm_next_pfn(bm); in clear_or_poison_free_pages()
1206 memory_bm_position_reset(bm); in clear_or_poison_free_pages()
1952 static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm, in alloc_highmem_pages() argument
1965 memory_bm_set_bit(bm, page_to_pfn(page)); in alloc_highmem_pages()
1972 static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm, in alloc_highmem_pages() argument
2108 static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm) in pack_pfns() argument
2113 buf[j] = memory_bm_next_pfn(bm); in pack_pfns()
2201 static void mark_unsafe_pages(struct memory_bitmap *bm) in mark_unsafe_pages() argument
2214 duplicate_memory_bitmap(free_pages_map, bm); in mark_unsafe_pages()
2257 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm) in unpack_orig_pfns() argument
2265 if (pfn_valid(buf[j]) && memory_bm_pfn_present(bm, buf[j])) in unpack_orig_pfns()
2266 memory_bm_set_bit(bm, buf[j]); in unpack_orig_pfns()
2300 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) in count_highmem_image_pages() argument
2305 memory_bm_position_reset(bm); in count_highmem_image_pages()
2306 pfn = memory_bm_next_pfn(bm); in count_highmem_image_pages()
2311 pfn = memory_bm_next_pfn(bm); in count_highmem_image_pages()
2333 static int prepare_highmem_image(struct memory_bitmap *bm, in prepare_highmem_image() argument
2338 if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE)) in prepare_highmem_image()
2357 memory_bm_set_bit(bm, page_to_pfn(page)); in prepare_highmem_image()
2364 memory_bm_position_reset(bm); in prepare_highmem_image()
2365 safe_highmem_bm = bm; in prepare_highmem_image()
2466 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) { return 0; } in count_highmem_image_pages() argument
2468 static inline int prepare_highmem_image(struct memory_bitmap *bm, in prepare_highmem_image() argument
2499 static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm) in prepare_image() argument
2509 nr_highmem = count_highmem_image_pages(bm); in prepare_image()
2510 mark_unsafe_pages(bm); in prepare_image()
2516 duplicate_memory_bitmap(new_bm, bm); in prepare_image()
2517 memory_bm_free(bm, PG_UNSAFE_KEEP); in prepare_image()
2519 error = prepare_highmem_image(bm, &nr_highmem); in prepare_image()
2575 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) in get_buffer() argument
2579 unsigned long pfn = memory_bm_next_pfn(bm); in get_buffer()