1/*
2 * Relocate a kexec_image to its destination and call it.
3 *
4 * Copyright (C) 2013 Citrix Systems R&D Ltd.
5 *
6 * Portions derived from Linux's arch/x86/kernel/relocate_kernel_64.S.
7 *
8 *   Copyright (C) 2002-2005 Eric Biederman  <ebiederm@xmission.com>
9 *
10 * This source code is licensed under the GNU General Public License,
11 * Version 2.  See the file COPYING for more details.
12 */
13
14        .file __FILE__
15
16#include <xen/kimage.h>
17
18#include <asm/asm_defns.h>
19#include <asm/msr-index.h>
20#include <asm/page.h>
21#include <asm/machine_kexec.h>
22
23        .section .text.kexec, "ax", @progbits
24        .align PAGE_SIZE
25        .code64
26
27ENTRY(kexec_reloc)
28        /* %rdi - code page maddr */
29        /* %rsi - page table maddr */
30        /* %rdx - indirection page maddr */
31        /* %rcx - entry maddr (%rbp) */
32        /* %r8 - flags */
33
34        movq    %rcx, %rbp
35
36        /* Setup stack. */
37        leaq    (reloc_stack - kexec_reloc)(%rdi), %rsp
38
39        /* Load reloc page table. */
40        movq    %rsi, %cr3
41
42        /* Jump to identity mapped code. */
43        leaq    (identity_mapped - kexec_reloc)(%rdi), %rax
44        jmpq    *%rax
45
46identity_mapped:
47        /*
48         * Set cr0 to a known state:
49         *  - Paging enabled
50         *  - Alignment check disabled
51         *  - Write protect disabled
52         *  - No task switch
53         *  - Don't do FP software emulation.
54         *  - Protected mode enabled
55         */
56        movq    %cr0, %rax
57        andl    $~(X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax
58        orl     $(X86_CR0_PG | X86_CR0_PE), %eax
59        movq    %rax, %cr0
60
61        /*
62         * Set cr4 to a known state:
63         *  - physical address extension enabled
64         */
65        movl    $X86_CR4_PAE, %eax
66        movq    %rax, %cr4
67
68        movq    %rdx, %rdi
69        call    relocate_pages
70
71        /* Need to switch to 32-bit mode? */
72        testq   $KEXEC_RELOC_FLAG_COMPAT, %r8
73        jnz     call_32_bit
74
75call_64_bit:
76        /* Call the image entry point.  This should never return. */
77        callq   *%rbp
78        ud2
79
80call_32_bit:
81        /* Setup IDT. */
82        lidt    compat_mode_idt(%rip)
83
84        /* Load compat GDT. */
85        leaq    compat_mode_gdt(%rip), %rax
86        movq    %rax, (compat_mode_gdt_desc + 2)(%rip)
87        lgdt    compat_mode_gdt_desc(%rip)
88
89        /* Relocate compatibility mode entry point address. */
90        leal    compatibility_mode(%rip), %eax
91        movl    %eax, compatibility_mode_far(%rip)
92
93        /* Enter compatibility mode. */
94        ljmp    *compatibility_mode_far(%rip)
95
96relocate_pages:
97        /* %rdi - indirection page maddr */
98        pushq   %rbx
99
100        cld
101        movq    %rdi, %rbx
102        xorl    %edi, %edi
103        xorl    %esi, %esi
104
105next_entry: /* top, read another word for the indirection page */
106
107        movq    (%rbx), %rcx
108        addq    $8, %rbx
109is_dest:
110        testb   $IND_DESTINATION, %cl
111        jz      is_ind
112        movq    %rcx, %rdi
113        andq    $PAGE_MASK, %rdi
114        jmp     next_entry
115is_ind:
116        testb   $IND_INDIRECTION, %cl
117        jz      is_done
118        movq    %rcx, %rbx
119        andq    $PAGE_MASK, %rbx
120        jmp     next_entry
121is_done:
122        testb   $IND_DONE, %cl
123        jnz     done
124is_source:
125        testb   $IND_SOURCE, %cl
126        jz      is_zero
127        movq    %rcx, %rsi      /* For every source page do a copy */
128        andq    $PAGE_MASK, %rsi
129        movl    $(PAGE_SIZE / 8), %ecx
130        rep movsq
131        jmp     next_entry
132is_zero:
133        testb   $IND_ZERO, %cl
134        jz      next_entry
135        movl    $(PAGE_SIZE / 8), %ecx  /* Zero the destination page. */
136        xorl    %eax, %eax
137        rep stosq
138        jmp     next_entry
139done:
140        popq    %rbx
141        ret
142
143        .code32
144
145compatibility_mode:
146        /* Setup some sane segments. */
147        movl    $0x0008, %eax
148        movl    %eax, %ds
149        movl    %eax, %es
150        movl    %eax, %fs
151        movl    %eax, %gs
152        movl    %eax, %ss
153
154        /* Disable paging and therefore leave 64 bit mode. */
155        movl    %cr0, %eax
156        andl    $~X86_CR0_PG, %eax
157        movl    %eax, %cr0
158
159        /* Disable long mode */
160        movl    $MSR_EFER, %ecx
161        rdmsr
162        andl    $~EFER_LME, %eax
163        wrmsr
164
165        /* Clear cr4 to disable PAE. */
166        xorl    %eax, %eax
167        movl    %eax, %cr4
168
169        /* Call the image entry point.  This should never return. */
170        call    *%ebp
171        ud2
172
173        .align 4
174compatibility_mode_far:
175        .long 0x00000000             /* set in call_32_bit above */
176        .word 0x0010
177
178compat_mode_gdt_desc:
179        .word .Lcompat_mode_gdt_end - compat_mode_gdt -1
180        .quad 0x0000000000000000     /* set in call_32_bit above */
181
182        .align 8
183compat_mode_gdt:
184        .quad 0x0000000000000000     /* null                              */
185        .quad 0x00cf93000000ffff     /* 0x0008 ring 0 data                */
186        .quad 0x00cf9b000000ffff     /* 0x0010 ring 0 code, compatibility */
187.Lcompat_mode_gdt_end:
188
189compat_mode_idt:
190        .word 0                      /* limit */
191        .long 0                      /* base */
192
193        /*
194         * 16 words of stack are more than enough.
195         */
196        .fill 16,8,0
197reloc_stack:
198
199        .globl kexec_reloc_size
200kexec_reloc_size:
201        .long . - kexec_reloc
202