diff options
Diffstat (limited to 'arch/arm/cpu/mmu_64.c')
-rw-r--r-- | arch/arm/cpu/mmu_64.c | 203 |
1 files changed, 154 insertions, 49 deletions
diff --git a/arch/arm/cpu/mmu_64.c b/arch/arm/cpu/mmu_64.c index 06049e0003..f907421da9 100644 --- a/arch/arm/cpu/mmu_64.c +++ b/arch/arm/cpu/mmu_64.c @@ -19,10 +19,15 @@ #include <asm/cache.h> #include <memory.h> #include <asm/system_info.h> +#include <linux/pagemap.h> +#include <tee/optee.h> #include "mmu_64.h" -static uint64_t *ttb; +static uint64_t *get_ttb(void) +{ + return (uint64_t *)get_ttbr(current_el()); +} static void set_table(uint64_t *pt, uint64_t *table_addr) { @@ -32,7 +37,20 @@ static void set_table(uint64_t *pt, uint64_t *table_addr) *pt = val; } -static uint64_t *create_table(void) +#ifdef __PBL__ +static uint64_t *alloc_pte(void) +{ + static unsigned int idx; + + idx++; + + if (idx * GRANULE_SIZE >= ARM_EARLY_PAGETABLE_SIZE) + return NULL; + + return (void *)get_ttb() + idx * GRANULE_SIZE; +} +#else +static uint64_t *alloc_pte(void) { uint64_t *new_table = xmemalign(GRANULE_SIZE, GRANULE_SIZE); @@ -41,6 +59,7 @@ static uint64_t *create_table(void) return new_table; } +#endif static __maybe_unused uint64_t *find_pte(uint64_t addr) { @@ -49,7 +68,7 @@ static __maybe_unused uint64_t *find_pte(uint64_t addr) uint64_t idx; int i; - pte = ttb; + pte = get_ttb(); for (i = 0; i < 4; i++) { block_shift = level2shift(i); @@ -81,7 +100,10 @@ static void split_block(uint64_t *pte, int level) /* level describes the parent level, we need the child ones */ levelshift = level2shift(level + 1); - new_table = create_table(); + new_table = alloc_pte(); + if (!new_table) + panic("Unable to allocate PTE\n"); + for (i = 0; i < MAX_PTE_ENTRIES; i++) { new_table[i] = old_pte | (i << levelshift); @@ -98,6 +120,7 @@ static void split_block(uint64_t *pte, int level) static void create_sections(uint64_t virt, uint64_t phys, uint64_t size, uint64_t attr) { + uint64_t *ttb = get_ttb(); uint64_t block_size; uint64_t block_shift; uint64_t *pte; @@ -107,13 +130,12 @@ static void create_sections(uint64_t virt, uint64_t phys, uint64_t size, uint64_t type; int level; - if (!ttb) - arm_mmu_not_initialized_error(); - addr = virt; attr &= ~PTE_TYPE_MASK; + size = PAGE_ALIGN(size); + while (size) { table = ttb; for (level = 0; level < 4; level++) { @@ -123,7 +145,8 @@ static void create_sections(uint64_t virt, uint64_t phys, uint64_t size, pte = table + idx; - if (size >= block_size && IS_ALIGNED(addr, block_size)) { + if (size >= block_size && IS_ALIGNED(addr, block_size) && + IS_ALIGNED(phys, block_size)) { type = (level == 3) ? PTE_TYPE_PAGE : PTE_TYPE_BLOCK; *pte = phys | attr | type; @@ -143,23 +166,42 @@ static void create_sections(uint64_t virt, uint64_t phys, uint64_t size, tlb_invalidate(); } -int arch_remap_range(void *_start, size_t size, unsigned flags) +static unsigned long get_pte_attrs(unsigned flags) { - unsigned long attrs; - switch (flags) { case MAP_CACHED: - attrs = CACHED_MEM; - break; + return CACHED_MEM; case MAP_UNCACHED: - attrs = attrs_uncached_mem(); - break; + return attrs_uncached_mem(); + case MAP_FAULT: + return 0x0; default: - return -EINVAL; + return ~0UL; } +} + +static void early_remap_range(uint64_t addr, size_t size, unsigned flags) +{ + unsigned long attrs = get_pte_attrs(flags); + + if (WARN_ON(attrs == ~0UL)) + return; + + create_sections(addr, addr, size, attrs); +} + +int arch_remap_range(void *virt_addr, phys_addr_t phys_addr, size_t size, unsigned flags) +{ + unsigned long attrs = get_pte_attrs(flags); + + if (attrs == ~0UL) + return -EINVAL; + + create_sections((uint64_t)virt_addr, phys_addr, (uint64_t)size, attrs); + + if (flags == MAP_UNCACHED) + dma_inv_range(virt_addr, size); - create_sections((uint64_t)_start, (uint64_t)_start, (uint64_t)size, - attrs); return 0; } @@ -169,14 +211,18 @@ static void mmu_enable(void) set_cr(get_cr() | CR_M | CR_C | CR_I); } -void zero_page_access(void) +static void create_guard_page(void) { - create_sections(0x0, 0x0, PAGE_SIZE, CACHED_MEM); -} + ulong guard_page; -void zero_page_faulting(void) -{ - create_sections(0x0, 0x0, PAGE_SIZE, 0x0); + if (!IS_ENABLED(CONFIG_STACK_GUARD_PAGE)) + return; + + guard_page = arm_mem_guard_page_get(); + request_sdram_region("guard page", guard_page, PAGE_SIZE); + remap_range((void *)guard_page, PAGE_SIZE, MAP_FAULT); + + pr_debug("Created guard page\n"); } /* @@ -184,30 +230,38 @@ void zero_page_faulting(void) */ void __mmu_init(bool mmu_on) { + uint64_t *ttb = get_ttb(); struct memory_bank *bank; - unsigned int el; - - if (mmu_on) - mmu_disable(); - - ttb = create_table(); - el = current_el(); - set_ttbr_tcr_mair(el, (uint64_t)ttb, calc_tcr(el, BITS_PER_VA), - MEMORY_ATTRIBUTES); - - pr_debug("ttb: 0x%p\n", ttb); - /* create a flat mapping */ - create_sections(0, 0, 1UL << (BITS_PER_VA - 1), attrs_uncached_mem()); + if (!request_sdram_region("ttb", (unsigned long)ttb, + ARM_EARLY_PAGETABLE_SIZE)) + /* + * This can mean that: + * - the early MMU code has put the ttb into a place + * which we don't have inside our available memory + * - Somebody else has occupied the ttb region which means + * the ttb will get corrupted. + */ + pr_crit("Can't request SDRAM region for ttb at %p\n", ttb); + + for_each_memory_bank(bank) { + struct resource *rsv; + resource_size_t pos; + + pos = bank->start; + + /* Skip reserved regions */ + for_each_reserved_region(bank, rsv) { + remap_range((void *)pos, rsv->start - pos, MAP_CACHED); + pos = rsv->end + 1; + } - /* Map sdram cached. */ - for_each_memory_bank(bank) - create_sections(bank->start, bank->start, bank->size, CACHED_MEM); + remap_range((void *)pos, bank->start + bank->size - pos, MAP_CACHED); + } /* Make zero page faulting to catch NULL pointer derefs */ zero_page_faulting(); - - mmu_enable(); + create_guard_page(); } void mmu_disable(void) @@ -241,15 +295,66 @@ void dma_flush_range(void *ptr, size_t size) v8_flush_dcache_range(start, end); } -void dma_sync_single_for_device(dma_addr_t address, size_t size, - enum dma_data_direction dir) +static void init_range(size_t total_level0_tables) { + uint64_t *ttb = get_ttb(); + uint64_t addr = 0; + + while (total_level0_tables--) { + early_remap_range(addr, L0_XLAT_SIZE, MAP_UNCACHED); + split_block(ttb, 0); + addr += L0_XLAT_SIZE; + ttb++; + } +} + +void mmu_early_enable(unsigned long membase, unsigned long memsize) +{ + int el; + u64 optee_membase; + unsigned long ttb = arm_mem_ttb(membase + memsize); + + if (get_cr() & CR_M) + return; + + pr_debug("enabling MMU, ttb @ 0x%08lx\n", ttb); + + el = current_el(); + set_ttbr_tcr_mair(el, ttb, calc_tcr(el, BITS_PER_VA), MEMORY_ATTRIBUTES); + if (el == 3) + set_ttbr_tcr_mair(2, ttb, calc_tcr(2, BITS_PER_VA), MEMORY_ATTRIBUTES); + + memset((void *)ttb, 0, GRANULE_SIZE); + /* - * FIXME: This function needs a device argument to support non 1:1 mappings + * Assume maximum BITS_PER_PA set to 40 bits. + * Set 1:1 mapping of VA->PA. So to cover the full 1TB range we need 2 tables. */ + init_range(2); + + early_remap_range(membase, memsize, MAP_CACHED); + + if (optee_get_membase(&optee_membase)) + optee_membase = membase + memsize - OPTEE_SIZE; - if (dir == DMA_FROM_DEVICE) - v8_inv_dcache_range(address, address + size - 1); - else - v8_flush_dcache_range(address, address + size - 1); + early_remap_range(optee_membase, OPTEE_SIZE, MAP_FAULT); + + early_remap_range(PAGE_ALIGN_DOWN((uintptr_t)_stext), PAGE_ALIGN(_etext - _stext), MAP_CACHED); + + mmu_enable(); +} + +void mmu_early_disable(void) +{ + unsigned int cr; + + cr = get_cr(); + cr &= ~(CR_M | CR_C); + + set_cr(cr); + v8_flush_dcache_all(); + tlb_invalidate(); + + dsb(); + isb(); } |