1        .text
2        .code64
3
4ENTRY(__high_start)
5        /* Install relocated data selectors. */
6        lgdt    boot_gdtr(%rip)
7        mov     $(__HYPERVISOR_DS64),%ecx
8        mov     %ecx,%ds
9        mov     %ecx,%es
10        mov     %ecx,%fs
11        mov     %ecx,%gs
12        mov     %ecx,%ss
13
14        /* Enable minimal CR4 features. */
15        mov     $XEN_MINIMAL_CR4,%rcx
16        mov     %rcx,%cr4
17
18        mov     stack_start(%rip),%rsp
19
20        /* Reset EFLAGS (subsumes CLI and CLD). */
21        pushq   $0
22        popf
23
24        /* Reload code selector. */
25        pushq   $__HYPERVISOR_CS
26        leaq    1f(%rip),%rax
27        pushq   %rax
28        lretq
291:
30        test    %ebx,%ebx
31        jz      .L_bsp
32
33        /* APs.  Set up shadow stacks before entering C. */
34#ifdef CONFIG_XEN_SHSTK
35        testl   $cpufeat_mask(X86_FEATURE_XEN_SHSTK), \
36                CPUINFO_FEATURE_OFFSET(X86_FEATURE_XEN_SHSTK) + boot_cpu_data(%rip)
37        je      .L_ap_shstk_done
38
39        /* Set up MSR_S_CET. */
40        mov     $MSR_S_CET, %ecx
41        xor     %edx, %edx
42        mov     $CET_SHSTK_EN | CET_WRSS_EN, %eax
43        wrmsr
44
45        /* Derive MSR_PL0_SSP from %rsp (token written when stack is allocated). */
46        mov     $MSR_PL0_SSP, %ecx
47        mov     %rsp, %rdx
48        shr     $32, %rdx
49        mov     %esp, %eax
50        and     $~(STACK_SIZE - 1), %eax
51        or      $(PRIMARY_SHSTK_SLOT + 1) * PAGE_SIZE - 8, %eax
52        wrmsr
53
54        /* Enable CET.  MSR_INTERRUPT_SSP_TABLE is set up later in load_system_tables(). */
55        mov     $XEN_MINIMAL_CR4 | X86_CR4_CET, %ecx
56        mov     %rcx, %cr4
57        setssbsy
58#endif
59
60.L_ap_shstk_done:
61        call    start_secondary
62        BUG     /* start_secondary() shouldn't return. */
63
64.L_bsp:
65        /* Pass off the Multiboot info structure to C land (if applicable). */
66        mov     multiboot_ptr(%rip),%edi
67        call    __start_xen
68        BUG     /* __start_xen() shouldn't return. */
69
70        .data
71        .align 8
72multiboot_ptr:
73        .long   0
74
75GLOBAL(stack_start)
76        .quad   cpu0_stack + STACK_SIZE - CPUINFO_sizeof
77
78        .section .data.page_aligned, "aw", @progbits
79        .align PAGE_SIZE, 0
80/*
81 * Mapping of first 2 megabytes of memory. This is mapped with 4kB mappings
82 * to avoid type conflicts with fixed-range MTRRs covering the lowest megabyte
83 * of physical memory. In any case the VGA hole should be mapped with type UC.
84 * Uses 1x 4k page.
85 */
86l1_directmap:
87        pfn = 0
88        .rept L1_PAGETABLE_ENTRIES
89        /* VGA hole (0xa0000-0xc0000) should be mapped UC-. */
90        .if pfn >= 0xa0 && pfn < 0xc0
91        .quad (pfn << PAGE_SHIFT) | __PAGE_HYPERVISOR_UCMINUS | _PAGE_GLOBAL | MAP_SMALL_PAGES
92        .else
93        .quad (pfn << PAGE_SHIFT) | PAGE_HYPERVISOR_RWX | MAP_SMALL_PAGES
94        .endif
95        pfn = pfn + 1
96        .endr
97        .size l1_directmap, . - l1_directmap
98
99/*
100 * __page_tables_{start,end} cover the range of pagetables which need
101 * relocating as Xen moves around physical memory.  i.e. each sym_offs()
102 * reference to a different pagetable in the Xen image.
103 */
104GLOBAL(__page_tables_start)
105
106/*
107 * Space for 4G worth of 2M mappings, first 2M actually mapped via
108 * l1_directmap[].  Uses 4x 4k pages.
109 */
110GLOBAL(l2_directmap)
111        .quad sym_offs(l1_directmap) + __PAGE_HYPERVISOR
112        .fill 4 * L2_PAGETABLE_ENTRIES - 1, 8, 0
113        .size l2_directmap, . - l2_directmap
114
115/*
116 * L2 mapping the Xen text/data/bss region, constructed dynamically.
117 * Executable fixmap is hooked up statically.
118 * Uses 1x 4k page.
119 */
120GLOBAL(l2_xenmap)
121        idx = 0
122        .rept L2_PAGETABLE_ENTRIES
123        .if idx == l2_table_offset(FIXADDR_X_TOP - 1)
124        .quad sym_offs(l1_fixmap_x) + __PAGE_HYPERVISOR
125        .else
126        .quad 0
127        .endif
128        idx = idx + 1
129        .endr
130        .size l2_xenmap, . - l2_xenmap
131
132/* L2 mapping the fixmap.  Uses 1x 4k page. */
133l2_fixmap:
134        idx = 0
135        .rept L2_PAGETABLE_ENTRIES
136        .if idx == l2_table_offset(FIXADDR_TOP - 1)
137        .quad sym_offs(l1_fixmap) + __PAGE_HYPERVISOR
138        .else
139        .quad 0
140        .endif
141        idx = idx + 1
142        .endr
143        .size l2_fixmap, . - l2_fixmap
144
145/* Direct map, initially covering the 4 l2_directmap tables.  Uses 1x 4k page. */
146l3_directmap:
147        idx = 0
148        .rept 4
149        .quad sym_offs(l2_directmap) + (idx << PAGE_SHIFT) + __PAGE_HYPERVISOR
150        idx = idx + 1
151        .endr
152        .fill L3_PAGETABLE_ENTRIES - 4, 8, 0
153        .size l3_directmap, . - l3_directmap
154
155/* L3 mapping the fixmap.  Uses 1x 4k page. */
156l3_xenmap:
157        idx = 0
158        .rept L3_PAGETABLE_ENTRIES
159        .if idx == l3_table_offset(XEN_VIRT_START)
160        .quad sym_offs(l2_xenmap) + __PAGE_HYPERVISOR
161        .elseif idx == l3_table_offset(FIXADDR_TOP - 1)
162        .quad sym_offs(l2_fixmap) + __PAGE_HYPERVISOR
163        .else
164        .quad 0
165        .endif
166        idx = idx + 1
167        .endr
168        .size l3_xenmap, . - l3_xenmap
169
170/* Top-level master (and idle-domain) page directory. */
171GLOBAL(idle_pg_table)
172        .quad sym_offs(l3_bootmap) + __PAGE_HYPERVISOR
173        idx = 1
174        .rept L4_PAGETABLE_ENTRIES - 1
175        .if idx == l4_table_offset(DIRECTMAP_VIRT_START)
176        .quad sym_offs(l3_directmap) + __PAGE_HYPERVISOR
177        .elseif idx == l4_table_offset(XEN_VIRT_START)
178        .quad sym_offs(l3_xenmap) + __PAGE_HYPERVISOR
179        .else
180        .quad 0
181        .endif
182        idx = idx + 1
183        .endr
184        .size idle_pg_table, . - idle_pg_table
185
186GLOBAL(__page_tables_end)
187
188/* Init pagetables. Enough page directories to map into 4GB. */
189        .section .init.data, "aw", @progbits
190        .align PAGE_SIZE, 0
191
192l1_bootmap:
193        .fill L1_PAGETABLE_ENTRIES, 8, 0
194        .size l1_bootmap, . - l1_bootmap
195
196GLOBAL(l2_bootmap)
197        .fill 4 * L2_PAGETABLE_ENTRIES, 8, 0
198        .size l2_bootmap, . - l2_bootmap
199
200GLOBAL(l3_bootmap)
201        .fill L3_PAGETABLE_ENTRIES, 8, 0
202        .size l3_bootmap, . - l3_bootmap
203