path: root/arch/arm64/include/asm/pgtable.h
diff options
authorJungseok Lee <>2014-05-12 18:40:51 +0900
committerCatalin Marinas <>2014-07-23 15:27:40 +0100
commitc79b954bf6c006f2d3dd9d01f231abeead13a410 (patch)
treeb7d4375c12d2b61554af0e5eb7596b22c60c643e /arch/arm64/include/asm/pgtable.h
parent57e0139041a978c0cfa4d2366c96ea3418e7d553 (diff)
arm64: mm: Implement 4 levels of translation tables
This patch implements 4 levels of translation tables since 3 levels of page tables with 4KB pages cannot support 40-bit physical address space described in [1] due to the following issue. It is a restriction that kernel logical memory map with 4KB + 3 levels (0xffffffc000000000-0xffffffffffffffff) cannot cover RAM region from 544GB to 1024GB in [1]. Specifically, ARM64 kernel fails to create mapping for this region in map_mem function since __phys_to_virt for this region reaches to address overflow. If SoC design follows the document, [1], over 32GB RAM would be placed from 544GB. Even 64GB system is supposed to use the region from 544GB to 576GB for only 32GB RAM. Naturally, it would reach to enable 4 levels of page tables to avoid hacking __virt_to_phys and __phys_to_virt. However, it is recommended 4 levels of page table should be only enabled if memory map is too sparse or there is about 512GB RAM. References ---------- [1]: Principles of ARM Memory Maps, White Paper, Issue C Signed-off-by: Jungseok Lee <> Reviewed-by: Sungjinn Chung <> Acked-by: Kukjin Kim <> Reviewed-by: Christoffer Dall <> Reviewed-by: Steve Capper <> [ MEMBLOCK_INITIAL_LIMIT removed, same as PUD_SIZE] [ early_ioremap_init() updated for 4 levels] [ 48-bit VA depends on BROKEN until KVM is fixed] Signed-off-by: Catalin Marinas <> Tested-by: Jungseok Lee <>
Diffstat (limited to 'arch/arm64/include/asm/pgtable.h')
1 files changed, 40 insertions, 0 deletions
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 6d5854972a77..d9b23efdaded 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -35,7 +35,11 @@
#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS)
+#ifndef CONFIG_ARM64_4_LEVELS
#define VMALLOC_END (PAGE_OFFSET - UL(0x400000000) - SZ_64K)
+#define VMALLOC_END (PAGE_OFFSET - UL(0x40000000000) - SZ_64K)
#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
@@ -44,12 +48,16 @@
#ifndef __ASSEMBLY__
extern void __pte_error(const char *file, int line, unsigned long val);
extern void __pmd_error(const char *file, int line, unsigned long val);
+extern void __pud_error(const char *file, int line, unsigned long val);
extern void __pgd_error(const char *file, int line, unsigned long val);
#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
+#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
#define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd))
@@ -347,6 +355,30 @@ static inline pmd_t *pud_page_vaddr(pud_t pud)
#endif /* CONFIG_ARM64_2_LEVELS */
+#define pgd_none(pgd) (!pgd_val(pgd))
+#define pgd_bad(pgd) (!(pgd_val(pgd) & 2))
+#define pgd_present(pgd) (pgd_val(pgd))
+static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
+ *pgdp = pgd;
+ dsb(ishst);
+static inline void pgd_clear(pgd_t *pgdp)
+ set_pgd(pgdp, __pgd(0));
+static inline pud_t *pgd_page_vaddr(pgd_t pgd)
+ return __va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
+#endif /* CONFIG_ARM64_4_LEVELS */
/* to find an entry in a page-table-directory */
#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
@@ -355,6 +387,14 @@ static inline pmd_t *pud_page_vaddr(pud_t pud)
/* to find an entry in a kernel page-table-directory */
#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
+#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
+static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr)
+ return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr);
/* Find an entry in the second-level page table.. */
#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))