1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/swap_cgroup.h>
3 #include <linux/vmalloc.h>
4 #include <linux/mm.h>
5 
6 #include <linux/swapops.h> /* depends on mm.h include */
7 
8 static DEFINE_MUTEX(swap_cgroup_mutex);
9 struct swap_cgroup_ctrl {
10 	struct page **map;
11 	unsigned long length;
12 	spinlock_t	lock;
13 };
14 
15 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
16 
17 struct swap_cgroup {
18 	unsigned short		id;
19 };
20 #define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
21 
22 /*
23  * SwapCgroup implements "lookup" and "exchange" operations.
24  * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
25  * against SwapCache. At swap_free(), this is accessed directly from swap.
26  *
27  * This means,
28  *  - we have no race in "exchange" when we're accessed via SwapCache because
29  *    SwapCache(and its swp_entry) is under lock.
30  *  - When called via swap_free(), there is no user of this entry and no race.
31  * Then, we don't need lock around "exchange".
32  *
33  * TODO: we can push these buffers out to HIGHMEM.
34  */
35 
36 /*
37  * allocate buffer for swap_cgroup.
38  */
swap_cgroup_prepare(int type)39 static int swap_cgroup_prepare(int type)
40 {
41 	struct page *page;
42 	struct swap_cgroup_ctrl *ctrl;
43 	unsigned long idx, max;
44 
45 	ctrl = &swap_cgroup_ctrl[type];
46 
47 	for (idx = 0; idx < ctrl->length; idx++) {
48 		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
49 		if (!page)
50 			goto not_enough_page;
51 		ctrl->map[idx] = page;
52 
53 		if (!(idx % SWAP_CLUSTER_MAX))
54 			cond_resched();
55 	}
56 	return 0;
57 not_enough_page:
58 	max = idx;
59 	for (idx = 0; idx < max; idx++)
60 		__free_page(ctrl->map[idx]);
61 
62 	return -ENOMEM;
63 }
64 
__lookup_swap_cgroup(struct swap_cgroup_ctrl * ctrl,pgoff_t offset)65 static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
66 						pgoff_t offset)
67 {
68 	struct page *mappage;
69 	struct swap_cgroup *sc;
70 
71 	mappage = ctrl->map[offset / SC_PER_PAGE];
72 	sc = page_address(mappage);
73 	return sc + offset % SC_PER_PAGE;
74 }
75 
lookup_swap_cgroup(swp_entry_t ent,struct swap_cgroup_ctrl ** ctrlp)76 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
77 					struct swap_cgroup_ctrl **ctrlp)
78 {
79 	pgoff_t offset = swp_offset(ent);
80 	struct swap_cgroup_ctrl *ctrl;
81 
82 	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
83 	if (ctrlp)
84 		*ctrlp = ctrl;
85 	return __lookup_swap_cgroup(ctrl, offset);
86 }
87 
88 /**
89  * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
90  * @ent: swap entry to be cmpxchged
91  * @old: old id
92  * @new: new id
93  *
94  * Returns old id at success, 0 at failure.
95  * (There is no mem_cgroup using 0 as its id)
96  */
swap_cgroup_cmpxchg(swp_entry_t ent,unsigned short old,unsigned short new)97 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
98 					unsigned short old, unsigned short new)
99 {
100 	struct swap_cgroup_ctrl *ctrl;
101 	struct swap_cgroup *sc;
102 	unsigned long flags;
103 	unsigned short retval;
104 
105 	sc = lookup_swap_cgroup(ent, &ctrl);
106 
107 	spin_lock_irqsave(&ctrl->lock, flags);
108 	retval = sc->id;
109 	if (retval == old)
110 		sc->id = new;
111 	else
112 		retval = 0;
113 	spin_unlock_irqrestore(&ctrl->lock, flags);
114 	return retval;
115 }
116 
117 /**
118  * swap_cgroup_record - record mem_cgroup for a set of swap entries
119  * @ent: the first swap entry to be recorded into
120  * @id: mem_cgroup to be recorded
121  * @nr_ents: number of swap entries to be recorded
122  *
123  * Returns old value at success, 0 at failure.
124  * (Of course, old value can be 0.)
125  */
swap_cgroup_record(swp_entry_t ent,unsigned short id,unsigned int nr_ents)126 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
127 				  unsigned int nr_ents)
128 {
129 	struct swap_cgroup_ctrl *ctrl;
130 	struct swap_cgroup *sc;
131 	unsigned short old;
132 	unsigned long flags;
133 	pgoff_t offset = swp_offset(ent);
134 	pgoff_t end = offset + nr_ents;
135 
136 	sc = lookup_swap_cgroup(ent, &ctrl);
137 
138 	spin_lock_irqsave(&ctrl->lock, flags);
139 	old = sc->id;
140 	for (;;) {
141 		VM_BUG_ON(sc->id != old);
142 		sc->id = id;
143 		offset++;
144 		if (offset == end)
145 			break;
146 		if (offset % SC_PER_PAGE)
147 			sc++;
148 		else
149 			sc = __lookup_swap_cgroup(ctrl, offset);
150 	}
151 	spin_unlock_irqrestore(&ctrl->lock, flags);
152 
153 	return old;
154 }
155 
156 /**
157  * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
158  * @ent: swap entry to be looked up.
159  *
160  * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
161  */
lookup_swap_cgroup_id(swp_entry_t ent)162 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
163 {
164 	return lookup_swap_cgroup(ent, NULL)->id;
165 }
166 
swap_cgroup_swapon(int type,unsigned long max_pages)167 int swap_cgroup_swapon(int type, unsigned long max_pages)
168 {
169 	void *array;
170 	unsigned long array_size;
171 	unsigned long length;
172 	struct swap_cgroup_ctrl *ctrl;
173 
174 	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
175 	array_size = length * sizeof(void *);
176 
177 	array = vzalloc(array_size);
178 	if (!array)
179 		goto nomem;
180 
181 	ctrl = &swap_cgroup_ctrl[type];
182 	mutex_lock(&swap_cgroup_mutex);
183 	ctrl->length = length;
184 	ctrl->map = array;
185 	spin_lock_init(&ctrl->lock);
186 	if (swap_cgroup_prepare(type)) {
187 		/* memory shortage */
188 		ctrl->map = NULL;
189 		ctrl->length = 0;
190 		mutex_unlock(&swap_cgroup_mutex);
191 		vfree(array);
192 		goto nomem;
193 	}
194 	mutex_unlock(&swap_cgroup_mutex);
195 
196 	return 0;
197 nomem:
198 	pr_info("couldn't allocate enough memory for swap_cgroup\n");
199 	pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
200 	return -ENOMEM;
201 }
202 
swap_cgroup_swapoff(int type)203 void swap_cgroup_swapoff(int type)
204 {
205 	struct page **map;
206 	unsigned long i, length;
207 	struct swap_cgroup_ctrl *ctrl;
208 
209 	mutex_lock(&swap_cgroup_mutex);
210 	ctrl = &swap_cgroup_ctrl[type];
211 	map = ctrl->map;
212 	length = ctrl->length;
213 	ctrl->map = NULL;
214 	ctrl->length = 0;
215 	mutex_unlock(&swap_cgroup_mutex);
216 
217 	if (map) {
218 		for (i = 0; i < length; i++) {
219 			struct page *page = map[i];
220 			if (page)
221 				__free_page(page);
222 			if (!(i % SWAP_CLUSTER_MAX))
223 				cond_resched();
224 		}
225 		vfree(map);
226 	}
227 }
228