1/*
2 * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7#include <arch.h>
8#include <asm_macros.S>
9
10	.globl	flush_dcache_range
11	.globl	clean_dcache_range
12	.globl	inv_dcache_range
13	.globl	dcsw_op_louis
14	.globl	dcsw_op_all
15	.globl	dcsw_op_level1
16	.globl	dcsw_op_level2
17	.globl	dcsw_op_level3
18
19/*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2
23	/* Exit early if size is zero */
24	cmp	r1, #0
25	beq	exit_loop_\op
26	dcache_line_size r2, r3
27	add	r1, r0, r1
28	sub	r3, r2, #1
29	bic	r0, r0, r3
30loop_\op:
31	stcopr	r0, \coproc, \opc1, \CRn, \CRm, \opc2
32	add	r0, r0, r2
33	cmp	r0, r1
34	blo	loop_\op
35	dsb	sy
36exit_loop_\op:
37	bx	lr
38.endm
39
40	/* ------------------------------------------
41	 * Clean+Invalidate from base address till
42	 * size. 'r0' = addr, 'r1' = size
43	 * ------------------------------------------
44	 */
45func flush_dcache_range
46	do_dcache_maintenance_by_mva cimvac, DCCIMVAC
47endfunc flush_dcache_range
48
49	/* ------------------------------------------
50	 * Clean from base address till size.
51	 * 'r0' = addr, 'r1' = size
52	 * ------------------------------------------
53	 */
54func clean_dcache_range
55	do_dcache_maintenance_by_mva cmvac, DCCMVAC
56endfunc clean_dcache_range
57
58	/* ------------------------------------------
59	 * Invalidate from base address till
60	 * size. 'r0' = addr, 'r1' = size
61	 * ------------------------------------------
62	 */
63func inv_dcache_range
64	do_dcache_maintenance_by_mva imvac, DCIMVAC
65endfunc inv_dcache_range
66
67	/* ----------------------------------------------------------------
68	 * Data cache operations by set/way to the level specified
69	 *
70	 * The main function, do_dcsw_op requires:
71	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
72	 * as defined in arch.h
73	 * r1: The cache level to begin operation from
74	 * r2: clidr_el1
75	 * r3: The last cache level to operate on
76	 * and will carry out the operation on each data cache from level 0
77	 * to the level in r3 in sequence
78	 *
79	 * The dcsw_op macro sets up the r2 and r3 parameters based on
80	 * clidr_el1 cache information before invoking the main function
81	 * ----------------------------------------------------------------
82	 */
83
84	.macro	dcsw_op shift, fw, ls
85	ldcopr	r2, CLIDR
86	ubfx	r3, r2, \shift, \fw
87	lsl	r3, r3, \ls
88	mov	r1, #0
89	b	do_dcsw_op
90	.endm
91
92func do_dcsw_op
93	push	{r4-r12, lr}
94	adr	r11, dcsw_loop_table	// compute cache op based on the operation type
95	add	r6, r11, r0, lsl #3	// cache op is 2x32-bit instructions
96loop1:
97	add	r10, r1, r1, LSR #1	// Work out 3x current cache level
98	mov	r12, r2, LSR r10	// extract cache type bits from clidr
99	and	r12, r12, #7		// mask the bits for current cache only
100	cmp	r12, #2			// see what cache we have at this level
101	blo	level_done		// no cache or only instruction cache at this level
102
103	stcopr	r1, CSSELR		// select current cache level in csselr
104	isb				// isb to sych the new cssr&csidr
105	ldcopr	r12, CCSIDR		// read the new ccsidr
106	and	r10, r12, #7		// extract the length of the cache lines
107	add	r10, r10, #4		// add 4 (r10 = line length offset)
108	ubfx	r4, r12, #3, #10	// r4 = maximum way number (right aligned)
109	clz	r5, r4			// r5 = the bit position of the way size increment
110	mov	r9, r4			// r9 working copy of the aligned max way number
111
112loop2:
113	ubfx	r7, r12, #13, #15	// r7 = max set number (right aligned)
114
115loop3:
116	orr	r0, r1, r9, LSL r5	// factor in the way number and cache level into r0
117	orr	r0, r0, r7, LSL r10	// factor in the set number
118
119	blx	r6
120	subs	r7, r7, #1		// decrement the set number
121	bhs	loop3
122	subs	r9, r9, #1		// decrement the way number
123	bhs	loop2
124level_done:
125	add	r1, r1, #2		// increment the cache number
126	cmp	r3, r1
127	// Ensure completion of previous cache maintenance instruction. Note
128	// this also mitigates erratum 814220 on Cortex-A7
129	dsb	sy
130	bhi	loop1
131
132	mov	r6, #0
133	stcopr	r6, CSSELR		//select cache level 0 in csselr
134	dsb	sy
135	isb
136	pop	{r4-r12, pc}
137
138dcsw_loop_table:
139	stcopr	r0, DCISW
140	bx	lr
141	stcopr	r0, DCCISW
142	bx	lr
143	stcopr	r0, DCCSW
144	bx	lr
145
146endfunc do_dcsw_op
147
148	/* ---------------------------------------------------------------
149	 * Data cache operations by set/way till PoU.
150	 *
151	 * The function requires :
152	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
153	 * as defined in arch.h
154	 * ---------------------------------------------------------------
155	 */
156func dcsw_op_louis
157	dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
158endfunc	dcsw_op_louis
159
160	/* ---------------------------------------------------------------
161	 * Data cache operations by set/way till PoC.
162	 *
163	 * The function requires :
164	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
165	 * as defined in arch.h
166	 * ---------------------------------------------------------------
167	 */
168func dcsw_op_all
169	dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
170endfunc	dcsw_op_all
171
172
173	/* ---------------------------------------------------------------
174	 *  Helper macro for data cache operations by set/way for the
175	 *  level specified
176	 * ---------------------------------------------------------------
177	 */
178	.macro	dcsw_op_level level
179	ldcopr	r2, CLIDR
180	mov	r3, \level
181	sub	r1, r3, #2
182	b	do_dcsw_op
183	.endm
184
185	/* ---------------------------------------------------------------
186	 * Data cache operations by set/way for level 1 cache
187	 *
188	 * The main function, do_dcsw_op requires:
189	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
190	 * as defined in arch.h
191	 * ---------------------------------------------------------------
192	 */
193func dcsw_op_level1
194	dcsw_op_level #(1 << LEVEL_SHIFT)
195endfunc dcsw_op_level1
196
197	/* ---------------------------------------------------------------
198	 * Data cache operations by set/way for level 2 cache
199	 *
200	 * The main function, do_dcsw_op requires:
201	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
202	 * as defined in arch.h
203	 * ---------------------------------------------------------------
204	 */
205func dcsw_op_level2
206	dcsw_op_level #(2 << LEVEL_SHIFT)
207endfunc dcsw_op_level2
208
209	/* ---------------------------------------------------------------
210	 * Data cache operations by set/way for level 3 cache
211	 *
212	 * The main function, do_dcsw_op requires:
213	 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
214	 * as defined in arch.h
215	 * ---------------------------------------------------------------
216	 */
217func dcsw_op_level3
218	dcsw_op_level #(3 << LEVEL_SHIFT)
219endfunc dcsw_op_level3
220