1/*
2 * Copyright (c) 2013-2020, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <arch.h>
8#include <asm_macros.S>
9
10	.globl	flush_dcache_range
11	.globl	clean_dcache_range
12	.globl	inv_dcache_range
13	.globl	dcsw_op_louis
14	.globl	dcsw_op_all
15	.globl	dcsw_op_level1
16	.globl	dcsw_op_level2
17	.globl	dcsw_op_level3
18
19/*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22.macro do_dcache_maintenance_by_mva op
23	/* Exit early if size is zero */
24	cbz	x1, exit_loop_\op
25	dcache_line_size x2, x3
26	add	x1, x0, x1
27	sub	x3, x2, #1
28	bic	x0, x0, x3
29loop_\op:
30	dc	\op, x0
31	add	x0, x0, x2
32	cmp	x0, x1
33	b.lo	loop_\op
34	dsb	sy
35exit_loop_\op:
36	ret
37.endm
38	/* ------------------------------------------
39	 * Clean+Invalidate from base address till
40	 * size. 'x0' = addr, 'x1' = size
41	 * ------------------------------------------
42	 */
43func flush_dcache_range
44	do_dcache_maintenance_by_mva civac
45endfunc flush_dcache_range
46
47	/* ------------------------------------------
48	 * Clean from base address till size.
49	 * 'x0' = addr, 'x1' = size
50	 * ------------------------------------------
51	 */
52func clean_dcache_range
53	do_dcache_maintenance_by_mva cvac
54endfunc clean_dcache_range
55
56	/* ------------------------------------------
57	 * Invalidate from base address till
58	 * size. 'x0' = addr, 'x1' = size
59	 * ------------------------------------------
60	 */
61func inv_dcache_range
62	do_dcache_maintenance_by_mva ivac
63endfunc inv_dcache_range
64
65
66	/* ---------------------------------------------------------------
67	 * Data cache operations by set/way to the level specified
68	 *
69	 * The main function, do_dcsw_op requires:
70	 * x0: The operation type (0-2), as defined in arch.h
71	 * x3: The last cache level to operate on
72	 * x9: clidr_el1
73	 * x10: The cache level to begin operation from
74	 * and will carry out the operation on each data cache from level 0
75	 * to the level in x3 in sequence
76	 *
77	 * The dcsw_op macro sets up the x3 and x9 parameters based on
78	 * clidr_el1 cache information before invoking the main function
79	 * ---------------------------------------------------------------
80	 */
81
82	.macro	dcsw_op shift, fw, ls
83	mrs	x9, clidr_el1
84	ubfx	x3, x9, \shift, \fw
85	lsl	x3, x3, \ls
86	mov	x10, xzr
87	b	do_dcsw_op
88	.endm
89
90func do_dcsw_op
91	cbz	x3, exit
92	adr	x14, dcsw_loop_table	// compute inner loop address
93	add	x14, x14, x0, lsl #5	// inner loop is 8x32-bit instructions
94#if ENABLE_BTI
95	add	x14, x14, x0, lsl #2	// inner loop is + "bti j" instruction
96#endif
97	mov	x0, x9
98	mov	w8, #1
99loop1:
100	add	x2, x10, x10, lsr #1	// work out 3x current cache level
101	lsr	x1, x0, x2		// extract cache type bits from clidr
102	and	x1, x1, #7		// mask the bits for current cache only
103	cmp	x1, #2			// see what cache we have at this level
104	b.lo	level_done		// nothing to do if no cache or icache
105
106	msr	csselr_el1, x10		// select current cache level in csselr
107	isb				// isb to sych the new cssr&csidr
108	mrs	x1, ccsidr_el1		// read the new ccsidr
109	and	x2, x1, #7		// extract the length of the cache lines
110	add	x2, x2, #4		// add 4 (line length offset)
111	ubfx	x4, x1, #3, #10		// maximum way number
112	clz	w5, w4			// bit position of way size increment
113	lsl	w9, w4, w5		// w9 = aligned max way number
114	lsl	w16, w8, w5		// w16 = way number loop decrement
115	orr	w9, w10, w9		// w9 = combine way and cache number
116	ubfx	w6, w1, #13, #15	// w6 = max set number
117	lsl	w17, w8, w2		// w17 = set number loop decrement
118	dsb	sy			// barrier before we start this level
119	br	x14			// jump to DC operation specific loop
120
121	.macro	dcsw_loop _op
122#if ENABLE_BTI
123	bti	j
124#endif
125loop2_\_op:
126	lsl	w7, w6, w2		// w7 = aligned max set number
127
128loop3_\_op:
129	orr	w11, w9, w7		// combine cache, way and set number
130	dc	\_op, x11
131	subs	w7, w7, w17		// decrement set number
132	b.hs	loop3_\_op
133
134	subs	x9, x9, x16		// decrement way number
135	b.hs	loop2_\_op
136
137	b	level_done
138	.endm
139
140level_done:
141	add	x10, x10, #2		// increment cache number
142	cmp	x3, x10
143	b.hi	loop1
144	msr	csselr_el1, xzr		// select cache level 0 in csselr
145	dsb	sy			// barrier to complete final cache operation
146	isb
147exit:
148	ret
149endfunc do_dcsw_op
150
151dcsw_loop_table:
152	dcsw_loop isw
153	dcsw_loop cisw
154	dcsw_loop csw
155
156
157func dcsw_op_louis
158	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
159endfunc dcsw_op_louis
160
161
162func dcsw_op_all
163	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
164endfunc dcsw_op_all
165
166	/* ---------------------------------------------------------------
167	 *  Helper macro for data cache operations by set/way for the
168	 *  level specified
169	 * ---------------------------------------------------------------
170	 */
171	.macro dcsw_op_level level
172	mrs	x9, clidr_el1
173	mov	x3, \level
174	sub	x10, x3, #2
175	b	do_dcsw_op
176	.endm
177
178	/* ---------------------------------------------------------------
179	 * Data cache operations by set/way for level 1 cache
180	 *
181	 * The main function, do_dcsw_op requires:
182	 * x0: The operation type (0-2), as defined in arch.h
183	 * ---------------------------------------------------------------
184	 */
185func dcsw_op_level1
186	dcsw_op_level #(1 << LEVEL_SHIFT)
187endfunc dcsw_op_level1
188
189	/* ---------------------------------------------------------------
190	 * Data cache operations by set/way for level 2 cache
191	 *
192	 * The main function, do_dcsw_op requires:
193	 * x0: The operation type (0-2), as defined in arch.h
194	 * ---------------------------------------------------------------
195	 */
196func dcsw_op_level2
197	dcsw_op_level #(2 << LEVEL_SHIFT)
198endfunc dcsw_op_level2
199
200	/* ---------------------------------------------------------------
201	 * Data cache operations by set/way for level 3 cache
202	 *
203	 * The main function, do_dcsw_op requires:
204	 * x0: The operation type (0-2), as defined in arch.h
205	 * ---------------------------------------------------------------
206	 */
207func dcsw_op_level3
208	dcsw_op_level #(3 << LEVEL_SHIFT)
209endfunc dcsw_op_level3
210