1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * (C) Copyright 2014 Google, Inc
4 *
5 * Memory Type Range Regsters - these are used to tell the CPU whether
6 * memory is cacheable and if so the cache write mode to use.
7 *
8 * These can speed up booting. See the mtrr command.
9 *
10 * Reference: Intel Architecture Software Developer's Manual, Volume 3:
11 * System Programming
12 */
13
14 /*
15 * Note that any console output (e.g. debug()) in this file will likely fail
16 * since the MTRR registers are sometimes in flux.
17 */
18
19 #include <common.h>
20 #include <cpu_func.h>
21 #include <log.h>
22 #include <sort.h>
23 #include <asm/cache.h>
24 #include <asm/global_data.h>
25 #include <asm/io.h>
26 #include <asm/mp.h>
27 #include <asm/msr.h>
28 #include <asm/mtrr.h>
29
30 DECLARE_GLOBAL_DATA_PTR;
31
32 /* Prepare to adjust MTRRs */
mtrr_open(struct mtrr_state * state,bool do_caches)33 void mtrr_open(struct mtrr_state *state, bool do_caches)
34 {
35 if (!gd->arch.has_mtrr)
36 return;
37
38 if (do_caches) {
39 state->enable_cache = dcache_status();
40
41 if (state->enable_cache)
42 disable_caches();
43 }
44 state->deftype = native_read_msr(MTRR_DEF_TYPE_MSR);
45 wrmsrl(MTRR_DEF_TYPE_MSR, state->deftype & ~MTRR_DEF_TYPE_EN);
46 }
47
48 /* Clean up after adjusting MTRRs, and enable them */
mtrr_close(struct mtrr_state * state,bool do_caches)49 void mtrr_close(struct mtrr_state *state, bool do_caches)
50 {
51 if (!gd->arch.has_mtrr)
52 return;
53
54 wrmsrl(MTRR_DEF_TYPE_MSR, state->deftype | MTRR_DEF_TYPE_EN);
55 if (do_caches && state->enable_cache)
56 enable_caches();
57 }
58
set_var_mtrr(uint reg,uint type,uint64_t start,uint64_t size)59 static void set_var_mtrr(uint reg, uint type, uint64_t start, uint64_t size)
60 {
61 u64 mask;
62
63 wrmsrl(MTRR_PHYS_BASE_MSR(reg), start | type);
64 mask = ~(size - 1);
65 mask &= (1ULL << CONFIG_CPU_ADDR_BITS) - 1;
66 wrmsrl(MTRR_PHYS_MASK_MSR(reg), mask | MTRR_PHYS_MASK_VALID);
67 }
68
mtrr_read_all(struct mtrr_info * info)69 void mtrr_read_all(struct mtrr_info *info)
70 {
71 int reg_count = mtrr_get_var_count();
72 int i;
73
74 for (i = 0; i < reg_count; i++) {
75 info->mtrr[i].base = native_read_msr(MTRR_PHYS_BASE_MSR(i));
76 info->mtrr[i].mask = native_read_msr(MTRR_PHYS_MASK_MSR(i));
77 }
78 }
79
mtrr_write_all(struct mtrr_info * info)80 void mtrr_write_all(struct mtrr_info *info)
81 {
82 int reg_count = mtrr_get_var_count();
83 struct mtrr_state state;
84 int i;
85
86 for (i = 0; i < reg_count; i++) {
87 mtrr_open(&state, true);
88 wrmsrl(MTRR_PHYS_BASE_MSR(i), info->mtrr[i].base);
89 wrmsrl(MTRR_PHYS_MASK_MSR(i), info->mtrr[i].mask);
90 mtrr_close(&state, true);
91 }
92 }
93
write_mtrrs(void * arg)94 static void write_mtrrs(void *arg)
95 {
96 struct mtrr_info *info = arg;
97
98 mtrr_write_all(info);
99 }
100
read_mtrrs(void * arg)101 static void read_mtrrs(void *arg)
102 {
103 struct mtrr_info *info = arg;
104
105 mtrr_read_all(info);
106 }
107
108 /**
109 * mtrr_copy_to_aps() - Copy the MTRRs from the boot CPU to other CPUs
110 *
111 * @return 0 on success, -ve on failure
112 */
mtrr_copy_to_aps(void)113 static int mtrr_copy_to_aps(void)
114 {
115 struct mtrr_info info;
116 int ret;
117
118 ret = mp_run_on_cpus(MP_SELECT_BSP, read_mtrrs, &info);
119 if (ret == -ENXIO)
120 return 0;
121 else if (ret)
122 return log_msg_ret("bsp", ret);
123
124 ret = mp_run_on_cpus(MP_SELECT_APS, write_mtrrs, &info);
125 if (ret)
126 return log_msg_ret("bsp", ret);
127
128 return 0;
129 }
130
h_comp_mtrr(const void * p1,const void * p2)131 static int h_comp_mtrr(const void *p1, const void *p2)
132 {
133 const struct mtrr_request *req1 = p1;
134 const struct mtrr_request *req2 = p2;
135
136 s64 diff = req1->start - req2->start;
137
138 return diff < 0 ? -1 : diff > 0 ? 1 : 0;
139 }
140
mtrr_commit(bool do_caches)141 int mtrr_commit(bool do_caches)
142 {
143 struct mtrr_request *req = gd->arch.mtrr_req;
144 struct mtrr_state state;
145 int ret;
146 int i;
147
148 debug("%s: enabled=%d, count=%d\n", __func__, gd->arch.has_mtrr,
149 gd->arch.mtrr_req_count);
150 if (!gd->arch.has_mtrr)
151 return -ENOSYS;
152
153 debug("open\n");
154 mtrr_open(&state, do_caches);
155 debug("open done\n");
156 qsort(req, gd->arch.mtrr_req_count, sizeof(*req), h_comp_mtrr);
157 for (i = 0; i < gd->arch.mtrr_req_count; i++, req++)
158 set_var_mtrr(i, req->type, req->start, req->size);
159
160 /* Clear the ones that are unused */
161 debug("clear\n");
162 for (; i < mtrr_get_var_count(); i++)
163 wrmsrl(MTRR_PHYS_MASK_MSR(i), 0);
164 debug("close\n");
165 mtrr_close(&state, do_caches);
166 debug("mtrr done\n");
167
168 if (gd->flags & GD_FLG_RELOC) {
169 ret = mtrr_copy_to_aps();
170 if (ret)
171 return log_msg_ret("copy", ret);
172 }
173
174 return 0;
175 }
176
mtrr_add_request(int type,uint64_t start,uint64_t size)177 int mtrr_add_request(int type, uint64_t start, uint64_t size)
178 {
179 struct mtrr_request *req;
180 uint64_t mask;
181
182 debug("%s: count=%d\n", __func__, gd->arch.mtrr_req_count);
183 if (!gd->arch.has_mtrr)
184 return -ENOSYS;
185
186 if (gd->arch.mtrr_req_count == MAX_MTRR_REQUESTS)
187 return -ENOSPC;
188 req = &gd->arch.mtrr_req[gd->arch.mtrr_req_count++];
189 req->type = type;
190 req->start = start;
191 req->size = size;
192 debug("%d: type=%d, %08llx %08llx\n", gd->arch.mtrr_req_count - 1,
193 req->type, req->start, req->size);
194 mask = ~(req->size - 1);
195 mask &= (1ULL << CONFIG_CPU_ADDR_BITS) - 1;
196 mask |= MTRR_PHYS_MASK_VALID;
197 debug(" %016llx %016llx\n", req->start | req->type, mask);
198
199 return 0;
200 }
201
mtrr_get_var_count(void)202 int mtrr_get_var_count(void)
203 {
204 return msr_read(MSR_MTRR_CAP_MSR).lo & MSR_MTRR_CAP_VCNT;
205 }
206
get_free_var_mtrr(void)207 static int get_free_var_mtrr(void)
208 {
209 struct msr_t maskm;
210 int vcnt;
211 int i;
212
213 vcnt = mtrr_get_var_count();
214
215 /* Identify the first var mtrr which is not valid */
216 for (i = 0; i < vcnt; i++) {
217 maskm = msr_read(MTRR_PHYS_MASK_MSR(i));
218 if ((maskm.lo & MTRR_PHYS_MASK_VALID) == 0)
219 return i;
220 }
221
222 /* No free var mtrr */
223 return -ENOSPC;
224 }
225
mtrr_set_next_var(uint type,uint64_t start,uint64_t size)226 int mtrr_set_next_var(uint type, uint64_t start, uint64_t size)
227 {
228 int mtrr;
229
230 mtrr = get_free_var_mtrr();
231 if (mtrr < 0)
232 return mtrr;
233
234 set_var_mtrr(mtrr, type, start, size);
235 debug("MTRR %x: start=%x, size=%x\n", mtrr, (uint)start, (uint)size);
236
237 return 0;
238 }
239
240 /** enum mtrr_opcode - supported operations for mtrr_do_oper() */
241 enum mtrr_opcode {
242 MTRR_OP_SET,
243 MTRR_OP_SET_VALID,
244 };
245
246 /**
247 * struct mtrr_oper - An MTRR operation to perform on a CPU
248 *
249 * @opcode: Indicates operation to perform
250 * @reg: MTRR reg number to select (0-7, -1 = all)
251 * @valid: Valid value to write for MTRR_OP_SET_VALID
252 * @base: Base value to write for MTRR_OP_SET
253 * @mask: Mask value to write for MTRR_OP_SET
254 */
255 struct mtrr_oper {
256 enum mtrr_opcode opcode;
257 int reg;
258 bool valid;
259 u64 base;
260 u64 mask;
261 };
262
mtrr_do_oper(void * arg)263 static void mtrr_do_oper(void *arg)
264 {
265 struct mtrr_oper *oper = arg;
266 u64 mask;
267
268 switch (oper->opcode) {
269 case MTRR_OP_SET_VALID:
270 mask = native_read_msr(MTRR_PHYS_MASK_MSR(oper->reg));
271 if (oper->valid)
272 mask |= MTRR_PHYS_MASK_VALID;
273 else
274 mask &= ~MTRR_PHYS_MASK_VALID;
275 wrmsrl(MTRR_PHYS_MASK_MSR(oper->reg), mask);
276 break;
277 case MTRR_OP_SET:
278 wrmsrl(MTRR_PHYS_BASE_MSR(oper->reg), oper->base);
279 wrmsrl(MTRR_PHYS_MASK_MSR(oper->reg), oper->mask);
280 break;
281 }
282 }
283
mtrr_start_op(int cpu_select,struct mtrr_oper * oper)284 static int mtrr_start_op(int cpu_select, struct mtrr_oper *oper)
285 {
286 struct mtrr_state state;
287 int ret;
288
289 mtrr_open(&state, true);
290 ret = mp_run_on_cpus(cpu_select, mtrr_do_oper, oper);
291 mtrr_close(&state, true);
292 if (ret)
293 return log_msg_ret("run", ret);
294
295 return 0;
296 }
297
mtrr_set_valid(int cpu_select,int reg,bool valid)298 int mtrr_set_valid(int cpu_select, int reg, bool valid)
299 {
300 struct mtrr_oper oper;
301
302 oper.opcode = MTRR_OP_SET_VALID;
303 oper.reg = reg;
304 oper.valid = valid;
305
306 return mtrr_start_op(cpu_select, &oper);
307 }
308
mtrr_set(int cpu_select,int reg,u64 base,u64 mask)309 int mtrr_set(int cpu_select, int reg, u64 base, u64 mask)
310 {
311 struct mtrr_oper oper;
312
313 oper.opcode = MTRR_OP_SET;
314 oper.reg = reg;
315 oper.base = base;
316 oper.mask = mask;
317
318 return mtrr_start_op(cpu_select, &oper);
319 }
320