arm/kernel/tcm.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * Copyright (C) 2008-2009 ST-Ericsson AB
0004  * TCM memory handling for ARM systems
0005  *
0006  * Author: Linus Walleij <linus.walleij@stericsson.com>
0007  * Author: Rickard Andersson <rickard.andersson@stericsson.com>
0008  */
0009 #include <linux/init.h>
0010 #include <linux/kernel.h>
0011 #include <linux/module.h>
0012 #include <linux/stddef.h>
0013 #include <linux/ioport.h>
0014 #include <linux/genalloc.h>
0015 #include <linux/string.h> /* memcpy */
0016 #include <asm/cputype.h>
0017 #include <asm/mach/map.h>
0018 #include <asm/memory.h>
0019 #include <asm/system_info.h>
0020 #include <asm/traps.h>
0021 #include <asm/tcm.h>
0022
0023 #define TCMTR_FORMAT_MASK   0xe0000000U
0024
0025 static struct gen_pool *tcm_pool;
0026 static bool dtcm_present;
0027 static bool itcm_present;
0028
0029 /* TCM section definitions from the linker */
0030 extern char __itcm_start, __sitcm_text, __eitcm_text;
0031 extern char __dtcm_start, __sdtcm_data, __edtcm_data;
0032
0033 /* These will be increased as we run */
0034 static u32 dtcm_end = DTCM_OFFSET;
0035 static u32 itcm_end = ITCM_OFFSET;
0036
0037 /*
0038  * TCM memory resources
0039  */
0040 static struct resource dtcm_res = {
0041     .name = "DTCM RAM",
0042     .start = DTCM_OFFSET,
0043     .end = DTCM_OFFSET,
0044     .flags = IORESOURCE_MEM
0045 };
0046
0047 static struct resource itcm_res = {
0048     .name = "ITCM RAM",
0049     .start = ITCM_OFFSET,
0050     .end = ITCM_OFFSET,
0051     .flags = IORESOURCE_MEM
0052 };
0053
0054 static struct map_desc dtcm_iomap[] __initdata = {
0055     {
0056         .virtual    = DTCM_OFFSET,
0057         .pfn        = __phys_to_pfn(DTCM_OFFSET),
0058         .length     = 0,
0059         .type       = MT_MEMORY_RW_DTCM
0060     }
0061 };
0062
0063 static struct map_desc itcm_iomap[] __initdata = {
0064     {
0065         .virtual    = ITCM_OFFSET,
0066         .pfn        = __phys_to_pfn(ITCM_OFFSET),
0067         .length     = 0,
0068         .type       = MT_MEMORY_RWX_ITCM,
0069     }
0070 };
0071
0072 /*
0073  * Allocate a chunk of TCM memory
0074  */
0075 void *tcm_alloc(size_t len)
0076 {
0077     unsigned long vaddr;
0078
0079     if (!tcm_pool)
0080         return NULL;
0081
0082     vaddr = gen_pool_alloc(tcm_pool, len);
0083     if (!vaddr)
0084         return NULL;
0085
0086     return (void *) vaddr;
0087 }
0088 EXPORT_SYMBOL(tcm_alloc);
0089
0090 /*
0091  * Free a chunk of TCM memory
0092  */
0093 void tcm_free(void *addr, size_t len)
0094 {
0095     gen_pool_free(tcm_pool, (unsigned long) addr, len);
0096 }
0097 EXPORT_SYMBOL(tcm_free);
0098
0099 bool tcm_dtcm_present(void)
0100 {
0101     return dtcm_present;
0102 }
0103 EXPORT_SYMBOL(tcm_dtcm_present);
0104
0105 bool tcm_itcm_present(void)
0106 {
0107     return itcm_present;
0108 }
0109 EXPORT_SYMBOL(tcm_itcm_present);
0110
0111 static int __init setup_tcm_bank(u8 type, u8 bank, u8 banks,
0112                   u32 *offset)
0113 {
0114     const int tcm_sizes[16] = { 0, -1, -1, 4, 8, 16, 32, 64, 128,
0115                     256, 512, 1024, -1, -1, -1, -1 };
0116     u32 tcm_region;
0117     int tcm_size;
0118
0119     /*
0120      * If there are more than one TCM bank of this type,
0121      * select the TCM bank to operate on in the TCM selection
0122      * register.
0123      */
0124     if (banks > 1)
0125         asm("mcr    p15, 0, %0, c9, c2, 0"
0126             : /* No output operands */
0127             : "r" (bank));
0128
0129     /* Read the special TCM region register c9, 0 */
0130     if (!type)
0131         asm("mrc    p15, 0, %0, c9, c1, 0"
0132             : "=r" (tcm_region));
0133     else
0134         asm("mrc    p15, 0, %0, c9, c1, 1"
0135             : "=r" (tcm_region));
0136
0137     tcm_size = tcm_sizes[(tcm_region >> 2) & 0x0f];
0138     if (tcm_size < 0) {
0139         pr_err("CPU: %sTCM%d of unknown size\n",
0140                type ? "I" : "D", bank);
0141         return -EINVAL;
0142     } else if (tcm_size > 32) {
0143         pr_err("CPU: %sTCM%d larger than 32k found\n",
0144                type ? "I" : "D", bank);
0145         return -EINVAL;
0146     } else {
0147         pr_info("CPU: found %sTCM%d %dk @ %08x, %senabled\n",
0148             type ? "I" : "D",
0149             bank,
0150             tcm_size,
0151             (tcm_region & 0xfffff000U),
0152             (tcm_region & 1) ? "" : "not ");
0153     }
0154
0155     /* Not much fun you can do with a size 0 bank */
0156     if (tcm_size == 0)
0157         return 0;
0158
0159     /* Force move the TCM bank to where we want it, enable */
0160     tcm_region = *offset | (tcm_region & 0x00000ffeU) | 1;
0161
0162     if (!type)
0163         asm("mcr    p15, 0, %0, c9, c1, 0"
0164             : /* No output operands */
0165             : "r" (tcm_region));
0166     else
0167         asm("mcr    p15, 0, %0, c9, c1, 1"
0168             : /* No output operands */
0169             : "r" (tcm_region));
0170
0171     /* Increase offset */
0172     *offset += (tcm_size << 10);
0173
0174     pr_info("CPU: moved %sTCM%d %dk to %08x, enabled\n",
0175         type ? "I" : "D",
0176         bank,
0177         tcm_size,
0178         (tcm_region & 0xfffff000U));
0179     return 0;
0180 }
0181
0182 /*
0183  * When we are running in the non-secure world and the secure world
0184  * has not explicitly given us access to the TCM we will get an
0185  * undefined error when reading the TCM region register in the
0186  * setup_tcm_bank function (above).
0187  *
0188  * There are two variants of this register read that we need to trap,
0189  * the read for the data TCM and the read for the instruction TCM:
0190  *  c0370628:       ee196f11        mrc     15, 0, r6, cr9, cr1, {0}
0191  *  c0370674:       ee196f31        mrc     15, 0, r6, cr9, cr1, {1}
0192  *
0193  * Our undef hook mask explicitly matches all fields of the encoded
0194  * instruction other than the destination register.  The mask also
0195  * only allows operand 2 to have the values 0 or 1.
0196  *
0197  * The undefined hook is defined as __init and __initdata, and therefore
0198  * must be removed before tcm_init returns.
0199  *
0200  * In this particular case (MRC with ARM condition code ALways) the
0201  * Thumb-2 and ARM instruction encoding are identical, so this hook
0202  * will work on a Thumb-2 kernel.
0203  *
0204  * See A8.8.107, DDI0406C_C ARM Architecture Reference Manual, Encoding
0205  * T1/A1 for the bit-by-bit details.
0206  *
0207  *  mrc   p15, 0, XX, c9, c1, 0
0208  *  mrc   p15, 0, XX, c9, c1, 1
0209  *   |  |  |   |   |   |   |  +---- opc2           0|1 = 000|001
0210  *   |  |  |   |   |   |   +------- CRm              0 = 0001
0211  *   |  |  |   |   |   +----------- CRn              0 = 1001
0212  *   |  |  |   |   +--------------- Rt               ? = ????
0213  *   |  |  |   +------------------- opc1             0 =  000
0214  *   |  |  +----------------------- coproc          15 = 1111
0215  *   |  +-------------------------- condition   ALways = 1110
0216  *   +----------------------------- instruction    MRC = 1110
0217  *
0218  * Encoding this as per A8.8.107 of DDI0406C, Encoding T1/A1, yields:
0219  *  1111 1111 1111 1111 0000 1111 1101 1111 Required Mask
0220  *  1110 1110 0001 1001 ???? 1111 0001 0001 mrc p15, 0, XX, c9, c1, 0
0221  *  1110 1110 0001 1001 ???? 1111 0011 0001 mrc p15, 0, XX, c9, c1, 1
0222  *  [  ] [  ] [ ]| [  ] [  ] [  ] [ ]| +--- CRm
0223  *    |    |   | |   |    |    |   | +----- SBO
0224  *    |    |   | |   |    |    |   +------- opc2
0225  *    |    |   | |   |    |    +----------- coproc
0226  *    |    |   | |   |    +---------------- Rt
0227  *    |    |   | |   +--------------------- CRn
0228  *    |    |   | +------------------------- SBO
0229  *    |    |   +--------------------------- opc1
0230  *    |    +------------------------------- instruction
0231  *    +------------------------------------ condition
0232  */
0233 #define TCM_REGION_READ_MASK        0xffff0fdf
0234 #define TCM_REGION_READ_INSTR       0xee190f11
0235 #define DEST_REG_SHIFT          12
0236 #define DEST_REG_MASK           0xf
0237
0238 static int __init tcm_handler(struct pt_regs *regs, unsigned int instr)
0239 {
0240     regs->uregs[(instr >> DEST_REG_SHIFT) & DEST_REG_MASK] = 0;
0241     regs->ARM_pc += 4;
0242     return 0;
0243 }
0244
0245 static struct undef_hook tcm_hook __initdata = {
0246     .instr_mask = TCM_REGION_READ_MASK,
0247     .instr_val  = TCM_REGION_READ_INSTR,
0248     .cpsr_mask  = MODE_MASK,
0249     .cpsr_val   = SVC_MODE,
0250     .fn     = tcm_handler
0251 };
0252
0253 /*
0254  * This initializes the TCM memory
0255  */
0256 void __init tcm_init(void)
0257 {
0258     u32 tcm_status;
0259     u8 dtcm_banks;
0260     u8 itcm_banks;
0261     size_t dtcm_code_sz = &__edtcm_data - &__sdtcm_data;
0262     size_t itcm_code_sz = &__eitcm_text - &__sitcm_text;
0263     char *start;
0264     char *end;
0265     char *ram;
0266     int ret;
0267     int i;
0268
0269     /*
0270      * Prior to ARMv5 there is no TCM, and trying to read the status
0271      * register will hang the processor.
0272      */
0273     if (cpu_architecture() < CPU_ARCH_ARMv5) {
0274         if (dtcm_code_sz || itcm_code_sz)
0275             pr_info("CPU TCM: %u bytes of DTCM and %u bytes of "
0276                 "ITCM code compiled in, but no TCM present "
0277                 "in pre-v5 CPU\n", dtcm_code_sz, itcm_code_sz);
0278         return;
0279     }
0280
0281     tcm_status = read_cpuid_tcmstatus();
0282
0283     /*
0284      * This code only supports v6-compatible TCMTR implementations.
0285      */
0286     if (tcm_status & TCMTR_FORMAT_MASK)
0287         return;
0288
0289     dtcm_banks = (tcm_status >> 16) & 0x03;
0290     itcm_banks = (tcm_status & 0x03);
0291
0292     register_undef_hook(&tcm_hook);
0293
0294     /* Values greater than 2 for D/ITCM banks are "reserved" */
0295     if (dtcm_banks > 2)
0296         dtcm_banks = 0;
0297     if (itcm_banks > 2)
0298         itcm_banks = 0;
0299
0300     /* Setup DTCM if present */
0301     if (dtcm_banks > 0) {
0302         for (i = 0; i < dtcm_banks; i++) {
0303             ret = setup_tcm_bank(0, i, dtcm_banks, &dtcm_end);
0304             if (ret)
0305                 goto unregister;
0306         }
0307         /* This means you compiled more code than fits into DTCM */
0308         if (dtcm_code_sz > (dtcm_end - DTCM_OFFSET)) {
0309             pr_info("CPU DTCM: %u bytes of code compiled to "
0310                 "DTCM but only %lu bytes of DTCM present\n",
0311                 dtcm_code_sz, (dtcm_end - DTCM_OFFSET));
0312             goto no_dtcm;
0313         }
0314         /*
0315          * This means that the DTCM sizes were 0 or the DTCM banks
0316          * were inaccessible due to TrustZone configuration.
0317          */
0318         if (!(dtcm_end - DTCM_OFFSET))
0319             goto no_dtcm;
0320         dtcm_res.end = dtcm_end - 1;
0321         request_resource(&iomem_resource, &dtcm_res);
0322         dtcm_iomap[0].length = dtcm_end - DTCM_OFFSET;
0323         iotable_init(dtcm_iomap, 1);
0324         /* Copy data from RAM to DTCM */
0325         start = &__sdtcm_data;
0326         end   = &__edtcm_data;
0327         ram   = &__dtcm_start;
0328         memcpy(start, ram, dtcm_code_sz);
0329         pr_debug("CPU DTCM: copied data from %p - %p\n",
0330              start, end);
0331         dtcm_present = true;
0332     } else if (dtcm_code_sz) {
0333         pr_info("CPU DTCM: %u bytes of code compiled to DTCM but no "
0334             "DTCM banks present in CPU\n", dtcm_code_sz);
0335     }
0336
0337 no_dtcm:
0338     /* Setup ITCM if present */
0339     if (itcm_banks > 0) {
0340         for (i = 0; i < itcm_banks; i++) {
0341             ret = setup_tcm_bank(1, i, itcm_banks, &itcm_end);
0342             if (ret)
0343                 goto unregister;
0344         }
0345         /* This means you compiled more code than fits into ITCM */
0346         if (itcm_code_sz > (itcm_end - ITCM_OFFSET)) {
0347             pr_info("CPU ITCM: %u bytes of code compiled to "
0348                 "ITCM but only %lu bytes of ITCM present\n",
0349                 itcm_code_sz, (itcm_end - ITCM_OFFSET));
0350             goto unregister;
0351         }
0352         /*
0353          * This means that the ITCM sizes were 0 or the ITCM banks
0354          * were inaccessible due to TrustZone configuration.
0355          */
0356         if (!(itcm_end - ITCM_OFFSET))
0357             goto unregister;
0358         itcm_res.end = itcm_end - 1;
0359         request_resource(&iomem_resource, &itcm_res);
0360         itcm_iomap[0].length = itcm_end - ITCM_OFFSET;
0361         iotable_init(itcm_iomap, 1);
0362         /* Copy code from RAM to ITCM */
0363         start = &__sitcm_text;
0364         end   = &__eitcm_text;
0365         ram   = &__itcm_start;
0366         memcpy(start, ram, itcm_code_sz);
0367         pr_debug("CPU ITCM: copied code from %p - %p\n",
0368              start, end);
0369         itcm_present = true;
0370     } else if (itcm_code_sz) {
0371         pr_info("CPU ITCM: %u bytes of code compiled to ITCM but no "
0372             "ITCM banks present in CPU\n", itcm_code_sz);
0373     }
0374
0375 unregister:
0376     unregister_undef_hook(&tcm_hook);
0377 }
0378
0379 /*
0380  * This creates the TCM memory pool and has to be done later,
0381  * during the core_initicalls, since the allocator is not yet
0382  * up and running when the first initialization runs.
0383  */
0384 static int __init setup_tcm_pool(void)
0385 {
0386     u32 dtcm_pool_start = (u32) &__edtcm_data;
0387     u32 itcm_pool_start = (u32) &__eitcm_text;
0388     int ret;
0389
0390     /*
0391      * Set up malloc pool, 2^2 = 4 bytes granularity since
0392      * the TCM is sometimes just 4 KiB. NB: pages and cache
0393      * line alignments does not matter in TCM!
0394      */
0395     tcm_pool = gen_pool_create(2, -1);
0396
0397     pr_debug("Setting up TCM memory pool\n");
0398
0399     /* Add the rest of DTCM to the TCM pool */
0400     if (dtcm_present) {
0401         if (dtcm_pool_start < dtcm_end) {
0402             ret = gen_pool_add(tcm_pool, dtcm_pool_start,
0403                        dtcm_end - dtcm_pool_start, -1);
0404             if (ret) {
0405                 pr_err("CPU DTCM: could not add DTCM " \
0406                        "remainder to pool!\n");
0407                 return ret;
0408             }
0409             pr_debug("CPU DTCM: Added %08x bytes @ %08x to " \
0410                  "the TCM memory pool\n",
0411                  dtcm_end - dtcm_pool_start,
0412                  dtcm_pool_start);
0413         }
0414     }
0415
0416     /* Add the rest of ITCM to the TCM pool */
0417     if (itcm_present) {
0418         if (itcm_pool_start < itcm_end) {
0419             ret = gen_pool_add(tcm_pool, itcm_pool_start,
0420                        itcm_end - itcm_pool_start, -1);
0421             if (ret) {
0422                 pr_err("CPU ITCM: could not add ITCM " \
0423                        "remainder to pool!\n");
0424                 return ret;
0425             }
0426             pr_debug("CPU ITCM: Added %08x bytes @ %08x to " \
0427                  "the TCM memory pool\n",
0428                  itcm_end - itcm_pool_start,
0429                  itcm_pool_start);
0430         }
0431     }
0432     return 0;
0433 }
0434
0435 core_initcall(setup_tcm_pool);