]>
git.zerfleddert.de Git - ms2-kexec/blob - tlbflush.h
d2005de383b8c105cf85368a5a7539bdaae0410b
2 * arch/arm/include/asm/tlbflush.h
4 * Copyright (C) 1999-2003 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 #ifndef _ASMARM_TLBFLUSH_H
11 #define _ASMARM_TLBFLUSH_H
17 #define TLB_V3_PAGE (1 << 0)
18 #define TLB_V4_U_PAGE (1 << 1)
19 #define TLB_V4_D_PAGE (1 << 2)
20 #define TLB_V4_I_PAGE (1 << 3)
21 #define TLB_V6_U_PAGE (1 << 4)
22 #define TLB_V6_D_PAGE (1 << 5)
23 #define TLB_V6_I_PAGE (1 << 6)
25 #define TLB_V3_FULL (1 << 8)
26 #define TLB_V4_U_FULL (1 << 9)
27 #define TLB_V4_D_FULL (1 << 10)
28 #define TLB_V4_I_FULL (1 << 11)
29 #define TLB_V6_U_FULL (1 << 12)
30 #define TLB_V6_D_FULL (1 << 13)
31 #define TLB_V6_I_FULL (1 << 14)
33 #define TLB_V6_U_ASID (1 << 16)
34 #define TLB_V6_D_ASID (1 << 17)
35 #define TLB_V6_I_ASID (1 << 18)
37 #define TLB_BTB (1 << 28)
39 /* Unified Inner Shareable TLB operations (ARMv7 MP extensions) */
40 #define TLB_V7_UIS_PAGE (1 << 19)
41 #define TLB_V7_UIS_FULL (1 << 20)
42 #define TLB_V7_UIS_ASID (1 << 21)
44 /* Inner Shareable BTB operation (ARMv7 MP extensions) */
45 #define TLB_V7_IS_BTB (1 << 22)
47 #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */
48 #define TLB_DCLEAN (1 << 30)
49 #define TLB_WB (1 << 31)
55 * We have the following to choose from:
57 * v4 - ARMv4 without write buffer
58 * v4wb - ARMv4 with write buffer without I TLB flush entry instruction
59 * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
60 * fr - Feroceon (v4wbi with non-outer-cacheable page table walks)
61 * fa - Faraday (v4 with write buffer with UTLB and branch target buffer (BTB))
62 * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction
63 * v7wbi - identical to v6wbi
68 #ifdef CONFIG_SMP_ON_UP
72 #define v3_tlb_flags (TLB_V3_FULL | TLB_V3_PAGE)
74 #ifdef CONFIG_CPU_TLB_V3
75 # define v3_possible_flags v3_tlb_flags
76 # define v3_always_flags v3_tlb_flags
83 # define v3_possible_flags 0
84 # define v3_always_flags (-1UL)
87 #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE)
89 #ifdef CONFIG_CPU_TLB_V4WT
90 # define v4_possible_flags v4_tlb_flags
91 # define v4_always_flags v4_tlb_flags
98 # define v4_possible_flags 0
99 # define v4_always_flags (-1UL)
102 #define fa_tlb_flags (TLB_WB | TLB_BTB | TLB_DCLEAN | \
103 TLB_V4_U_FULL | TLB_V4_U_PAGE)
105 #ifdef CONFIG_CPU_TLB_FA
106 # define fa_possible_flags fa_tlb_flags
107 # define fa_always_flags fa_tlb_flags
114 # define fa_possible_flags 0
115 # define fa_always_flags (-1UL)
118 #define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \
119 TLB_V4_I_FULL | TLB_V4_D_FULL | \
120 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
122 #ifdef CONFIG_CPU_TLB_V4WBI
123 # define v4wbi_possible_flags v4wbi_tlb_flags
124 # define v4wbi_always_flags v4wbi_tlb_flags
131 # define v4wbi_possible_flags 0
132 # define v4wbi_always_flags (-1UL)
135 #define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \
136 TLB_V4_I_FULL | TLB_V4_D_FULL | \
137 TLB_V4_I_PAGE | TLB_V4_D_PAGE)
139 #ifdef CONFIG_CPU_TLB_FEROCEON
140 # define fr_possible_flags fr_tlb_flags
141 # define fr_always_flags fr_tlb_flags
148 # define fr_possible_flags 0
149 # define fr_always_flags (-1UL)
152 #define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \
153 TLB_V4_I_FULL | TLB_V4_D_FULL | \
156 #ifdef CONFIG_CPU_TLB_V4WB
157 # define v4wb_possible_flags v4wb_tlb_flags
158 # define v4wb_always_flags v4wb_tlb_flags
165 # define v4wb_possible_flags 0
166 # define v4wb_always_flags (-1UL)
169 #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_BTB | \
170 TLB_V6_I_FULL | TLB_V6_D_FULL | \
171 TLB_V6_I_PAGE | TLB_V6_D_PAGE | \
172 TLB_V6_I_ASID | TLB_V6_D_ASID)
174 #ifdef CONFIG_CPU_TLB_V6
175 # define v6wbi_possible_flags v6wbi_tlb_flags
176 # define v6wbi_always_flags v6wbi_tlb_flags
183 # define v6wbi_possible_flags 0
184 # define v6wbi_always_flags (-1UL)
187 #define v7wbi_tlb_flags_smp (TLB_WB | TLB_DCLEAN | TLB_V7_IS_BTB | \
188 TLB_V7_UIS_FULL | TLB_V7_UIS_PAGE | TLB_V7_UIS_ASID)
189 #define v7wbi_tlb_flags_up (TLB_WB | TLB_DCLEAN | TLB_BTB | \
190 TLB_V6_U_FULL | TLB_V6_U_PAGE | TLB_V6_U_ASID)
192 #ifdef CONFIG_CPU_TLB_V7
194 # ifdef CONFIG_SMP_ON_UP
195 # define v7wbi_possible_flags (v7wbi_tlb_flags_smp | v7wbi_tlb_flags_up)
196 # define v7wbi_always_flags (v7wbi_tlb_flags_smp & v7wbi_tlb_flags_up)
197 # elif defined(CONFIG_SMP)
198 # define v7wbi_possible_flags v7wbi_tlb_flags_smp
199 # define v7wbi_always_flags v7wbi_tlb_flags_smp
201 # define v7wbi_possible_flags v7wbi_tlb_flags_up
202 # define v7wbi_always_flags v7wbi_tlb_flags_up
210 # define v7wbi_possible_flags 0
211 # define v7wbi_always_flags (-1UL)
215 #error Unknown TLB model
220 #include <linux/sched.h>
223 void (*flush_user_range
)(unsigned long, unsigned long, struct vm_area_struct
*);
224 void (*flush_kern_range
)(unsigned long, unsigned long);
225 unsigned long tlb_flags
;
229 * Select the calling method
233 #define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range
234 #define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range
238 #define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range)
239 #define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range)
241 extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct
*);
242 extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
246 extern struct cpu_tlb_fns cpu_tlb
;
248 #define __cpu_tlb_flags cpu_tlb.tlb_flags
254 * The arch/arm/mm/tlb-*.S files implement these methods.
256 * The TLB specific code is expected to perform whatever tests it
257 * needs to determine if it should invalidate the TLB for each
258 * call. Start addresses are inclusive and end addresses are
259 * exclusive; it is safe to round these addresses down.
263 * Invalidate the entire TLB.
267 * Invalidate all TLB entries in a particular address
269 * - mm - mm_struct describing address space
271 * flush_tlb_range(mm,start,end)
273 * Invalidate a range of TLB entries in the specified
275 * - mm - mm_struct describing address space
276 * - start - start address (may not be aligned)
277 * - end - end address (exclusive, may not be aligned)
279 * flush_tlb_page(vaddr,vma)
281 * Invalidate the specified page in the specified address range.
282 * - vaddr - virtual address (may not be aligned)
283 * - vma - vma_struct describing address range
285 * flush_kern_tlb_page(kaddr)
287 * Invalidate the TLB entry for the specified page. The address
288 * will be in the kernels virtual memory space. Current uses
289 * only require the D-TLB to be invalidated.
290 * - kaddr - Kernel virtual memory address
294 * We optimise the code below by:
295 * - building a set of TLB flags that might be set in __cpu_tlb_flags
296 * - building a set of TLB flags that will always be set in __cpu_tlb_flags
297 * - if we're going to need __cpu_tlb_flags, access it once and only once
299 * This allows us to build optimal assembly for the single-CPU type case,
300 * and as close to optimal given the compiler constrants for multi-CPU
301 * case. We could do better for the multi-CPU case if the compiler
302 * implemented the "%?" method, but this has been discontinued due to too
303 * many people getting it wrong.
305 #define possible_tlb_flags (v3_possible_flags | \
306 v4_possible_flags | \
307 v4wbi_possible_flags | \
308 fr_possible_flags | \
309 v4wb_possible_flags | \
310 fa_possible_flags | \
311 v6wbi_possible_flags | \
312 v7wbi_possible_flags)
314 #define always_tlb_flags (v3_always_flags & \
316 v4wbi_always_flags & \
318 v4wb_always_flags & \
320 v6wbi_always_flags & \
323 #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
325 static inline void local_flush_tlb_all(void)
328 const unsigned int __tlb_flag
= __cpu_tlb_flags
;
330 if (tlb_flag(TLB_WB
))
333 if (tlb_flag(TLB_V3_FULL
))
334 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero
) : "cc");
335 if (tlb_flag(TLB_V4_U_FULL
| TLB_V6_U_FULL
))
336 asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero
) : "cc");
337 if (tlb_flag(TLB_V4_D_FULL
| TLB_V6_D_FULL
))
338 asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero
) : "cc");
339 if (tlb_flag(TLB_V4_I_FULL
| TLB_V6_I_FULL
))
340 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero
) : "cc");
341 if (tlb_flag(TLB_V7_UIS_FULL
))
342 asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero
) : "cc");
344 if (tlb_flag(TLB_BTB
)) {
345 /* flush the branch target cache */
346 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero
) : "cc");
350 if (tlb_flag(TLB_V7_IS_BTB
)) {
351 /* flush the branch target cache */
352 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero
) : "cc");
358 static inline void local_flush_tlb_mm(struct mm_struct
*mm
)
361 const int asid
= ASID(mm
);
362 const unsigned int __tlb_flag
= __cpu_tlb_flags
;
364 if (tlb_flag(TLB_WB
))
367 if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm
))) {
368 if (tlb_flag(TLB_V3_FULL
))
369 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero
) : "cc");
370 if (tlb_flag(TLB_V4_U_FULL
))
371 asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero
) : "cc");
372 if (tlb_flag(TLB_V4_D_FULL
))
373 asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero
) : "cc");
374 if (tlb_flag(TLB_V4_I_FULL
))
375 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero
) : "cc");
379 if (tlb_flag(TLB_V6_U_ASID
))
380 asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid
) : "cc");
381 if (tlb_flag(TLB_V6_D_ASID
))
382 asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid
) : "cc");
383 if (tlb_flag(TLB_V6_I_ASID
))
384 asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid
) : "cc");
385 if (tlb_flag(TLB_V7_UIS_ASID
))
386 #ifdef CONFIG_ARM_ERRATA_720789
387 asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero
) : "cc");
389 asm("mcr p15, 0, %0, c8, c3, 2" : : "r" (asid
) : "cc");
392 if (tlb_flag(TLB_BTB
)) {
393 /* flush the branch target cache */
394 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero
) : "cc");
397 if (tlb_flag(TLB_V7_IS_BTB
)) {
398 /* flush the branch target cache */
399 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero
) : "cc");
406 local_flush_tlb_page(struct vm_area_struct
*vma
, unsigned long uaddr
)
409 const unsigned int __tlb_flag
= __cpu_tlb_flags
;
411 uaddr
= (uaddr
& PAGE_MASK
) | ASID(vma
->vm_mm
);
413 if (tlb_flag(TLB_WB
))
416 if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma
->vm_mm
))) {
417 if (tlb_flag(TLB_V3_PAGE
))
418 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr
) : "cc");
419 if (tlb_flag(TLB_V4_U_PAGE
))
420 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr
) : "cc");
421 if (tlb_flag(TLB_V4_D_PAGE
))
422 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr
) : "cc");
423 if (tlb_flag(TLB_V4_I_PAGE
))
424 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr
) : "cc");
425 if (!tlb_flag(TLB_V4_I_PAGE
) && tlb_flag(TLB_V4_I_FULL
))
426 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero
) : "cc");
429 if (tlb_flag(TLB_V6_U_PAGE
))
430 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr
) : "cc");
431 if (tlb_flag(TLB_V6_D_PAGE
))
432 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr
) : "cc");
433 if (tlb_flag(TLB_V6_I_PAGE
))
434 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr
) : "cc");
435 if (tlb_flag(TLB_V7_UIS_PAGE
))
436 #ifdef CONFIG_ARM_ERRATA_720789
437 asm("mcr p15, 0, %0, c8, c3, 3" : : "r" (uaddr
& PAGE_MASK
) : "cc");
439 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (uaddr
) : "cc");
442 if (tlb_flag(TLB_BTB
)) {
443 /* flush the branch target cache */
444 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero
) : "cc");
447 if (tlb_flag(TLB_V7_IS_BTB
)) {
448 /* flush the branch target cache */
449 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero
) : "cc");
455 static inline void local_flush_tlb_kernel_page(unsigned long kaddr
)
458 const unsigned int __tlb_flag
= __cpu_tlb_flags
;
462 if (tlb_flag(TLB_WB
))
465 if (tlb_flag(TLB_V3_PAGE
))
466 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr
) : "cc");
467 if (tlb_flag(TLB_V4_U_PAGE
))
468 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr
) : "cc");
469 if (tlb_flag(TLB_V4_D_PAGE
))
470 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr
) : "cc");
471 if (tlb_flag(TLB_V4_I_PAGE
))
472 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr
) : "cc");
473 if (!tlb_flag(TLB_V4_I_PAGE
) && tlb_flag(TLB_V4_I_FULL
))
474 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero
) : "cc");
476 if (tlb_flag(TLB_V6_U_PAGE
))
477 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr
) : "cc");
478 if (tlb_flag(TLB_V6_D_PAGE
))
479 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr
) : "cc");
480 if (tlb_flag(TLB_V6_I_PAGE
))
481 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr
) : "cc");
482 if (tlb_flag(TLB_V7_UIS_PAGE
))
483 asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr
) : "cc");
485 if (tlb_flag(TLB_BTB
)) {
486 /* flush the branch target cache */
487 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero
) : "cc");
491 if (tlb_flag(TLB_V7_IS_BTB
)) {
492 /* flush the branch target cache */
493 asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero
) : "cc");
502 * Flush a PMD entry (word aligned, or double-word aligned) to
503 * RAM if the TLB for the CPU we are running on requires this.
504 * This is typically used when we are creating PMD entries.
508 * Clean (but don't drain the write buffer) if the CPU requires
509 * these operations. This is typically used when we are removing
512 static inline void flush_pmd_entry(pmd_t
*pmd
)
514 const unsigned int __tlb_flag
= __cpu_tlb_flags
;
516 if (tlb_flag(TLB_DCLEAN
))
517 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd"
518 : : "r" (pmd
) : "cc");
520 if (tlb_flag(TLB_L2CLEAN_FR
))
521 asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd"
522 : : "r" (pmd
) : "cc");
524 if (tlb_flag(TLB_WB
))
528 static inline void clean_pmd_entry(pmd_t
*pmd
)
530 const unsigned int __tlb_flag
= __cpu_tlb_flags
;
532 if (tlb_flag(TLB_DCLEAN
))
533 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd"
534 : : "r" (pmd
) : "cc");
536 if (tlb_flag(TLB_L2CLEAN_FR
))
537 asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd"
538 : : "r" (pmd
) : "cc");
542 #undef always_tlb_flags
543 #undef possible_tlb_flags
546 * Convert calls to our calling convention.
548 #define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
549 #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e)
552 #define flush_tlb_all local_flush_tlb_all
553 #define flush_tlb_mm local_flush_tlb_mm
554 #define flush_tlb_page local_flush_tlb_page
555 #define flush_tlb_kernel_page local_flush_tlb_kernel_page
556 #define flush_tlb_range local_flush_tlb_range
557 #define flush_tlb_kernel_range local_flush_tlb_kernel_range
559 extern void flush_tlb_all(void);
560 extern void flush_tlb_mm(struct mm_struct
*mm
);
561 extern void flush_tlb_page(struct vm_area_struct
*vma
, unsigned long uaddr
);
562 extern void flush_tlb_kernel_page(unsigned long kaddr
);
563 extern void flush_tlb_range(struct vm_area_struct
*vma
, unsigned long start
, unsigned long end
);
564 extern void flush_tlb_kernel_range(unsigned long start
, unsigned long end
);
568 * If PG_dcache_clean is not set for the page, we need to ensure that any
569 * cache entries for the kernels virtual memory range are written
570 * back to the page. On ARMv6 and later, the cache coherency is handled via
571 * the set_pte_at() function.
573 #if __LINUX_ARM_ARCH__ < 6
574 extern void update_mmu_cache(struct vm_area_struct
*vma
, unsigned long addr
,
577 static inline void update_mmu_cache(struct vm_area_struct
*vma
,
578 unsigned long addr
, pte_t
*ptep
)
585 #endif /* CONFIG_MMU */