1 /*
2  * xen-mceinj.c: utilities to inject fake MCE for x86.
3  * Copyright (c) 2010, Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; If not, see <http://www.gnu.org/licenses/>.
16  *
17  * Authors: Yunhong Jiang <yunhong.jiang@intel.com>
18  *          Haicheng Li <haicheng.li@intel.com>
19  *          Xudong Hao <xudong.hao@intel.com>
20  */
21 
22 
23 #define _GNU_SOURCE
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <unistd.h>
27 #include <string.h>
28 #include <getopt.h>
29 #include <errno.h>
30 #include <signal.h>
31 #include <stdarg.h>
32 
33 #define XC_WANT_COMPAT_MAP_FOREIGN_API
34 #include <xenctrl.h>
35 #include <xenguest.h>
36 #include <inttypes.h>
37 #include <sys/time.h>
38 #include <xen/arch-x86/xen-mca.h>
39 #include <xenstore.h>
40 
41 #define MCi_type_CTL        0x0
42 #define MCi_type_STATUS     0x1
43 #define MCi_type_ADDR       0x2
44 #define MCi_type_MISC       0x3
45 #define MCi_type_CTL2       0x4
46 
47 #define INVALID_MSR         ~0UL
48 
49 /* Intel MSRs */
50 #define MSR_IA32_MCG_CAP         0x00000179
51 #define MSR_IA32_MCG_STATUS      0x0000017a
52 #define MSR_IA32_MCG_CTL         0x0000017b
53 #define MSR_IA32_MC0_CTL         0x00000400
54 #define MSR_IA32_MC0_STATUS      0x00000401
55 #define MSR_IA32_MC0_ADDR        0x00000402
56 #define MSR_IA32_MC0_MISC        0x00000403
57 #define MSR_IA32_MC0_CTL2        0x00000280
58 
59 #define MCG_STATUS_LMCE          0x8
60 
61 struct mce_info {
62     const char *description;
63     uint8_t mcg_stat;
64     unsigned int bank;
65     uint64_t mci_stat;
66     uint64_t mci_misc;
67     bool cmci;
68 };
69 
70 static struct mce_info mce_table[] = {
71     /* LLC (Last Level Cache) EWB (Explicit Write Back) SRAO MCE */
72     {
73         .description = "MCE_SRAO_MEM",
74         .mcg_stat = 0x5,
75         .bank = 7,
76         .mci_stat = 0xBD2000008000017Aull,
77         .mci_misc = 0x86ull,
78     },
79     /* Memory Patrol Scrub SRAO MCE */
80     {
81         .description = "MCE_SRAO_LLC",
82         .mcg_stat = 0x5,
83         .bank = 8,
84         .mci_stat = 0xBD000000004000CFull,
85         .mci_misc = 0x86ull,
86     },
87     /* LLC EWB UCNA Error */
88     {
89         .description = "CMCI_UCNA_LLC",
90         .mcg_stat = 0x0,
91         .bank = 9,
92         .mci_stat = 0xBC20000080000136ull,
93         .mci_misc = 0x86ull,
94         .cmci = true,
95     },
96     /* AMD L1 instruction cache data or tag parity. */
97     {
98         .description = "AMD L1 icache parity",
99         .mcg_stat = 0x5,
100         .bank = 1,
101         .mci_stat = 0x9400000000000151ull,
102         .mci_misc = 0x86ull,
103     },
104     /* LLC (Last Level Cache) EWB (Explicit Write Back) SRAO MCE */
105     {
106         .description = "MCE_SRAO_MEM (Fatal)",
107         .mcg_stat = 0x5,
108         .bank = 7,
109         .mci_stat = 0xBF2000008000017Aull,
110         .mci_misc = 0x86ull,
111     },
112 };
113 #define MCE_TABLE_SIZE (sizeof(mce_table)/sizeof(mce_table[0]))
114 
115 #define LOGFILE stdout
116 
117 int dump;
118 int lmce;
119 struct xen_mc_msrinject msr_inj;
120 
Lprintf(const char * fmt,...)121 static void Lprintf(const char *fmt, ...)
122 {
123     char *buf;
124     va_list args;
125 
126     va_start(args, fmt);
127     if (vasprintf(&buf, fmt, args) < 0)
128         abort();
129     fprintf(LOGFILE, "%s\n", buf);
130     va_end(args);
131     free(buf);
132 }
133 
err(xc_interface * xc_handle,const char * fmt,...)134 static void err(xc_interface *xc_handle, const char *fmt, ...)
135 {
136     char *buf;
137     va_list args;
138 
139     va_start(args, fmt);
140     if (vasprintf(&buf, fmt, args) < 0)
141         abort();
142     perror(buf);
143     va_end(args);
144     free(buf);
145 
146     if ( xc_handle )
147         xc_interface_close(xc_handle);
148     exit(EXIT_FAILURE);
149 }
150 
init_msr_inj(void)151 static void init_msr_inj(void)
152 {
153     memset(&msr_inj, 0, sizeof(msr_inj));
154 }
155 
flush_msr_inj(xc_interface * xc_handle)156 static int flush_msr_inj(xc_interface *xc_handle)
157 {
158     struct xen_mc mc;
159 
160     mc.cmd = XEN_MC_msrinject;
161     mc.interface_version = XEN_MCA_INTERFACE_VERSION;
162     mc.u.mc_msrinject = msr_inj;
163 
164     return xc_mca_op(xc_handle, &mc);
165 }
166 
mca_cpuinfo(xc_interface * xc_handle)167 static unsigned int mca_cpuinfo(xc_interface *xc_handle)
168 {
169     struct xen_mc mc;
170 
171     memset(&mc, 0, sizeof(struct xen_mc));
172 
173     mc.cmd = XEN_MC_physcpuinfo;
174     mc.interface_version = XEN_MCA_INTERFACE_VERSION;
175 
176     if (!xc_mca_op(xc_handle, &mc))
177         return mc.u.mc_physcpuinfo.ncpus;
178     else
179         return 0;
180 }
181 
inject_cmci(xc_interface * xc_handle,unsigned int cpu_nr)182 static int inject_cmci(xc_interface *xc_handle, unsigned int cpu_nr)
183 {
184     struct xen_mc mc;
185     unsigned int nr_cpus;
186 
187     memset(&mc, 0, sizeof(struct xen_mc));
188 
189     nr_cpus = mca_cpuinfo(xc_handle);
190     if (!nr_cpus)
191         err(xc_handle, "Failed to get mca_cpuinfo");
192     if (cpu_nr >= nr_cpus)
193         err(xc_handle, "-c %u is larger than %u", cpu_nr, nr_cpus - 1);
194 
195     mc.cmd = XEN_MC_inject_v2;
196     mc.interface_version = XEN_MCA_INTERFACE_VERSION;
197 
198     mc.u.mc_inject_v2.flags |= XEN_MC_INJECT_CPU_BROADCAST;
199     mc.u.mc_inject_v2.flags |= XEN_MC_INJECT_TYPE_CMCI;
200     mc.u.mc_inject_v2.cpumap.nr_bits = nr_cpus;
201 
202     return xc_mca_op(xc_handle, &mc);
203 }
204 
inject_mce(xc_interface * xc_handle,int cpu_nr)205 static int inject_mce(xc_interface *xc_handle, int cpu_nr)
206 {
207     struct xen_mc mc;
208 
209     memset(&mc, 0, sizeof(struct xen_mc));
210 
211     mc.cmd = XEN_MC_mceinject;
212     mc.interface_version = XEN_MCA_INTERFACE_VERSION;
213     mc.u.mc_mceinject.mceinj_cpunr = cpu_nr;
214 
215     return xc_mca_op(xc_handle, &mc);
216 }
217 
inject_lmce(xc_interface * xc_handle,unsigned int cpu)218 static int inject_lmce(xc_interface *xc_handle, unsigned int cpu)
219 {
220     uint8_t *cpumap = NULL;
221     size_t cpumap_size, line, shift;
222     unsigned int nr_cpus;
223     int ret;
224 
225     nr_cpus = mca_cpuinfo(xc_handle);
226     if ( !nr_cpus )
227         err(xc_handle, "Failed to get mca_cpuinfo");
228     if ( cpu >= nr_cpus )
229         err(xc_handle, "-c %u is larger than %u", cpu, nr_cpus - 1);
230 
231     cpumap_size = (nr_cpus + 7) / 8;
232     cpumap = malloc(cpumap_size);
233     if ( !cpumap )
234         err(xc_handle, "Failed to allocate cpumap\n");
235     memset(cpumap, 0, cpumap_size);
236     line = cpu / 8;
237     shift = cpu % 8;
238     memset(cpumap + line, 1 << shift, 1);
239 
240     ret = xc_mca_op_inject_v2(xc_handle, XEN_MC_INJECT_TYPE_LMCE,
241                               cpumap, cpumap_size * 8);
242 
243     free(cpumap);
244     return ret;
245 }
246 
bank_addr(int bank,int type)247 static uint64_t bank_addr(int bank, int type)
248 {
249     uint64_t addr;
250 
251     switch ( type )
252     {
253     case MCi_type_CTL:
254     case MCi_type_STATUS:
255     case MCi_type_ADDR:
256     case MCi_type_MISC:
257         addr = MSR_IA32_MC0_CTL + (bank * 4) + type;
258         break;
259     case MCi_type_CTL2:
260         addr = MSR_IA32_MC0_CTL2 + bank;
261         break;
262     default:
263         addr = INVALID_MSR;
264         break;
265     }
266 
267     return addr;
268 }
269 
add_msr_intpose(xc_interface * xc_handle,uint32_t cpu_nr,uint32_t flags,uint64_t msr,uint64_t val,domid_t domid)270 static int add_msr_intpose(xc_interface *xc_handle,
271                            uint32_t cpu_nr,
272                            uint32_t flags,
273                            uint64_t msr,
274                            uint64_t val,
275                            domid_t domid)
276 {
277     uint32_t count;
278 
279     if ( (msr_inj.mcinj_count &&
280           (cpu_nr != msr_inj.mcinj_cpunr || flags != msr_inj.mcinj_flags ||
281            domid != msr_inj.mcinj_domid)) ||
282          msr_inj.mcinj_count == MC_MSRINJ_MAXMSRS )
283     {
284         flush_msr_inj(xc_handle);
285         init_msr_inj();
286     }
287     count= msr_inj.mcinj_count;
288 
289     if ( !count )
290     {
291         msr_inj.mcinj_cpunr = cpu_nr;
292         msr_inj.mcinj_flags = flags;
293         msr_inj.mcinj_domid = domid;
294     }
295     msr_inj.mcinj_msr[count].reg = msr;
296     msr_inj.mcinj_msr[count].value = val;
297     msr_inj.mcinj_count++;
298 
299     return 0;
300 }
301 
add_msr_bank_intpose(xc_interface * xc_handle,uint32_t cpu_nr,uint32_t flags,uint32_t type,uint32_t bank,uint64_t val,domid_t domid)302 static int add_msr_bank_intpose(xc_interface *xc_handle,
303                                 uint32_t cpu_nr,
304                                 uint32_t flags,
305                                 uint32_t type,
306                                 uint32_t bank,
307                                 uint64_t val,
308                                 domid_t domid)
309 {
310     uint64_t msr;
311 
312     msr = bank_addr(bank, type);
313     if ( msr == INVALID_MSR )
314         return -1;
315     return add_msr_intpose(xc_handle, cpu_nr, flags, msr, val, domid);
316 }
317 
inject_mcg_status(xc_interface * xc_handle,uint32_t cpu_nr,uint64_t val,domid_t domid)318 static int inject_mcg_status(xc_interface *xc_handle,
319                              uint32_t cpu_nr,
320                              uint64_t val,
321                              domid_t domid)
322 {
323     return add_msr_intpose(xc_handle, cpu_nr, MC_MSRINJ_F_INTERPOSE,
324                            MSR_IA32_MCG_STATUS, val, domid);
325 }
326 
inject_mci_status(xc_interface * xc_handle,uint32_t cpu_nr,uint64_t bank,uint64_t val,domid_t domid)327 static int inject_mci_status(xc_interface *xc_handle,
328                              uint32_t cpu_nr,
329                              uint64_t bank,
330                              uint64_t val,
331                              domid_t domid)
332 {
333     return add_msr_bank_intpose(xc_handle, cpu_nr, MC_MSRINJ_F_INTERPOSE,
334                                 MCi_type_STATUS, bank, val, domid);
335 }
336 
inject_mci_misc(xc_interface * xc_handle,uint32_t cpu_nr,uint64_t bank,uint64_t val,domid_t domid)337 static int inject_mci_misc(xc_interface *xc_handle,
338                            uint32_t cpu_nr,
339                            uint64_t bank,
340                            uint64_t val,
341                            domid_t domid)
342 {
343     return add_msr_bank_intpose(xc_handle, cpu_nr, MC_MSRINJ_F_INTERPOSE,
344                                 MCi_type_MISC, bank, val, domid);
345 }
346 
inject_mci_addr(xc_interface * xc_handle,uint32_t cpu_nr,uint64_t bank,uint64_t val,domid_t domid)347 static int inject_mci_addr(xc_interface *xc_handle,
348                            uint32_t cpu_nr,
349                            uint64_t bank,
350                            uint64_t val,
351                            domid_t domid)
352 {
353     return add_msr_bank_intpose(xc_handle, cpu_nr,
354                                 MC_MSRINJ_F_INTERPOSE |
355                                 ((domid >= DOMID_FIRST_RESERVED &&
356                                   domid != DOMID_SELF) ?
357                                  0 : MC_MSRINJ_F_GPADDR),
358                                 MCi_type_ADDR, bank, val, domid);
359 }
360 
inject(xc_interface * xc_handle,struct mce_info * mce,uint32_t cpu_nr,uint32_t domain,uint64_t gaddr)361 static int inject(xc_interface *xc_handle, struct mce_info *mce,
362                   uint32_t cpu_nr, uint32_t domain, uint64_t gaddr)
363 {
364     int ret = 0;
365     uint8_t mcg_status = mce->mcg_stat;
366 
367     if ( lmce )
368     {
369         if ( mce->cmci )
370             err(xc_handle, "No support to inject CMCI as LMCE");
371         mcg_status |= MCG_STATUS_LMCE;
372     }
373     ret = inject_mcg_status(xc_handle, cpu_nr, mcg_status, domain);
374     if ( ret )
375         err(xc_handle, "Failed to inject MCG_STATUS MSR");
376 
377     ret = inject_mci_status(xc_handle, cpu_nr,
378                             mce->bank, mce->mci_stat, domain);
379     if ( ret )
380         err(xc_handle, "Failed to inject MCi_STATUS MSR");
381 
382     ret = inject_mci_misc(xc_handle, cpu_nr,
383                           mce->bank, mce->mci_misc, domain);
384     if ( ret )
385         err(xc_handle, "Failed to inject MCi_MISC MSR");
386 
387     ret = inject_mci_addr(xc_handle, cpu_nr, mce->bank, gaddr, domain);
388     if ( ret )
389         err(xc_handle, "Failed to inject MCi_ADDR MSR");
390 
391     ret = flush_msr_inj(xc_handle);
392     if ( ret )
393         err(xc_handle, "Failed to inject MSR");
394     if ( mce->cmci )
395         ret = inject_cmci(xc_handle, cpu_nr);
396     else if ( lmce )
397         ret = inject_lmce(xc_handle, cpu_nr);
398     else
399         ret = inject_mce(xc_handle, cpu_nr);
400     if ( ret )
401         err(xc_handle, "Failed to inject MCE error");
402 
403     return 0;
404 }
405 
xs_get_dom_mem(int domid)406 static long xs_get_dom_mem(int domid)
407 {
408     char path[128];
409     char *memstr;
410     uint64_t mem;
411     unsigned int plen;
412     struct xs_handle *xs;
413 
414     xs = xs_daemon_open();
415     if (!xs)
416         return -1;
417 
418     sprintf(path, "/local/domain/%d/memory/target", domid);
419     memstr = xs_read(xs, XBT_NULL, path, &plen);
420     xs_daemon_close(xs);
421 
422     if (!memstr || !plen)
423         return -1;
424 
425     mem = atoll(memstr)*1024;
426     free(memstr);
427 
428     return mem;
429 }
430 
431 static struct option opts[] = {
432     {"cpu", 0, 0, 'c'},
433     {"domain", 0, 0, 'd'},
434     {"dump", 0, 0, 'D'},
435     {"help", 0, 0, 'h'},
436     {"page", 0, 0, 'p'},
437     {"lmce", 0, 0, 'l'},
438     {"", 0, 0, '\0'}
439 };
440 
help(void)441 static void help(void)
442 {
443     unsigned int i;
444 
445     printf("Usage: xen-mceinj [OPTION]...\n"
446            "\n"
447            "Mandatory arguments to long options are mandatory"
448            "for short options too.\n"
449            "  -D, --dump           dump addr info without error injection\n"
450            "  -c, --cpu=CPU        target CPU\n"
451            "  -d, --domain=DOMID   target domain, the default is Xen itself\n"
452            "  -h, --help           print this page\n"
453            "  -p, --page=ADDR      physical address to report\n"
454            "  -l, --lmce           inject as LMCE (Intel only)\n"
455            "  -t, --type=ERROR     error type\n");
456 
457     for ( i = 0; i < MCE_TABLE_SIZE; i++ )
458         printf("                       %2d : %s\n",
459                i, mce_table[i].description);
460 }
461 
main(int argc,char * argv[])462 int main(int argc, char *argv[])
463 {
464     int type = 0;
465     int c, opt_index;
466     uint32_t domid;
467     xc_interface *xc_handle;
468     unsigned int cpu_nr;
469     uint64_t gaddr, max_gpa;
470 
471     /* Default Value */
472     domid = DOMID_XEN;
473     gaddr = 0x180020;
474     cpu_nr = 0;
475 
476     init_msr_inj();
477     xc_handle = xc_interface_open(0, 0, 0);
478     if ( !xc_handle ) {
479         Lprintf("Failed to get xc interface");
480         exit(EXIT_FAILURE);
481     }
482 
483     while ( 1 ) {
484         c = getopt_long(argc, argv, "c:Dd:t:hp:l", opts, &opt_index);
485         if ( c == -1 )
486             break;
487         switch ( c ) {
488         case 'D':
489             dump=1;
490             break;
491         case 'c':
492             cpu_nr = strtoul(optarg, &optarg, 10);
493             if ( strlen(optarg) != 0 )
494                 err(xc_handle, "Please input a digit parameter for CPU");
495             break;
496         case 'd':
497             domid = strtol(optarg, &optarg, 10);
498             if ( strlen(optarg) != 0 )
499                 err(xc_handle, "Please input a digit parameter for domain");
500             break;
501         case 'p':
502             gaddr = strtol(optarg, &optarg, 0);
503             if ( strlen(optarg) != 0 )
504                 err(xc_handle, "Please input correct page address");
505             break;
506         case 't':
507             type = strtol(optarg, NULL, 0);
508             break;
509         case 'l':
510             lmce = 1;
511             break;
512         case 'h':
513         default:
514             help();
515             return 0;
516         }
517     }
518 
519     if ( domid != DOMID_XEN ) {
520         max_gpa = xs_get_dom_mem(domid);
521         Lprintf("get domain %d max gpa is: 0x%lx", domid, max_gpa);
522         if ( gaddr >= max_gpa )
523             err(xc_handle, "Fail: gaddr exceeds max_gpa 0x%lx", max_gpa);
524     }
525     Lprintf("get gaddr of error inject is: 0x%lx", gaddr);
526 
527     if ( dump ) {
528         if ( domid == DOMID_XEN )
529             Lprintf("Xen: gaddr=0x%lx", gaddr);
530         else
531             Lprintf("Dom%d: gaddr=0x%lx", domid, gaddr);
532         goto out;
533     }
534 
535     if ( type < 0 || type >= MCE_TABLE_SIZE ) {
536         err(xc_handle, "Unsupported error type");
537         goto out;
538     }
539 
540     inject(xc_handle, &mce_table[type], cpu_nr, domid, gaddr);
541 
542  out:
543     xc_interface_close(xc_handle);
544     return 0;
545 }
546