arm64/kernel/machine_kexec.c

0001 // SPDX-License-Identifier: GPL-2.0-only
0002 /*
0003  * kexec for arm64
0004  *
0005  * Copyright (C) Linaro.
0006  * Copyright (C) Huawei Futurewei Technologies.
0007  */
0008
0009 #include <linux/interrupt.h>
0010 #include <linux/irq.h>
0011 #include <linux/kernel.h>
0012 #include <linux/kexec.h>
0013 #include <linux/page-flags.h>
0014 #include <linux/set_memory.h>
0015 #include <linux/smp.h>
0016
0017 #include <asm/cacheflush.h>
0018 #include <asm/cpu_ops.h>
0019 #include <asm/daifflags.h>
0020 #include <asm/memory.h>
0021 #include <asm/mmu.h>
0022 #include <asm/mmu_context.h>
0023 #include <asm/page.h>
0024 #include <asm/sections.h>
0025 #include <asm/trans_pgd.h>
0026
0027 /**
0028  * kexec_image_info - For debugging output.
0029  */
0030 #define kexec_image_info(_i) _kexec_image_info(__func__, __LINE__, _i)
0031 static void _kexec_image_info(const char *func, int line,
0032     const struct kimage *kimage)
0033 {
0034     unsigned long i;
0035
0036     pr_debug("%s:%d:\n", func, line);
0037     pr_debug("  kexec kimage info:\n");
0038     pr_debug("    type:        %d\n", kimage->type);
0039     pr_debug("    start:       %lx\n", kimage->start);
0040     pr_debug("    head:        %lx\n", kimage->head);
0041     pr_debug("    nr_segments: %lu\n", kimage->nr_segments);
0042     pr_debug("    dtb_mem: %pa\n", &kimage->arch.dtb_mem);
0043     pr_debug("    kern_reloc: %pa\n", &kimage->arch.kern_reloc);
0044     pr_debug("    el2_vectors: %pa\n", &kimage->arch.el2_vectors);
0045
0046     for (i = 0; i < kimage->nr_segments; i++) {
0047         pr_debug("      segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n",
0048             i,
0049             kimage->segment[i].mem,
0050             kimage->segment[i].mem + kimage->segment[i].memsz,
0051             kimage->segment[i].memsz,
0052             kimage->segment[i].memsz /  PAGE_SIZE);
0053     }
0054 }
0055
0056 void machine_kexec_cleanup(struct kimage *kimage)
0057 {
0058     /* Empty routine needed to avoid build errors. */
0059 }
0060
0061 /**
0062  * machine_kexec_prepare - Prepare for a kexec reboot.
0063  *
0064  * Called from the core kexec code when a kernel image is loaded.
0065  * Forbid loading a kexec kernel if we have no way of hotplugging cpus or cpus
0066  * are stuck in the kernel. This avoids a panic once we hit machine_kexec().
0067  */
0068 int machine_kexec_prepare(struct kimage *kimage)
0069 {
0070     if (kimage->type != KEXEC_TYPE_CRASH && cpus_are_stuck_in_kernel()) {
0071         pr_err("Can't kexec: CPUs are stuck in the kernel.\n");
0072         return -EBUSY;
0073     }
0074
0075     return 0;
0076 }
0077
0078 /**
0079  * kexec_segment_flush - Helper to flush the kimage segments to PoC.
0080  */
0081 static void kexec_segment_flush(const struct kimage *kimage)
0082 {
0083     unsigned long i;
0084
0085     pr_debug("%s:\n", __func__);
0086
0087     for (i = 0; i < kimage->nr_segments; i++) {
0088         pr_debug("  segment[%lu]: %016lx - %016lx, 0x%lx bytes, %lu pages\n",
0089             i,
0090             kimage->segment[i].mem,
0091             kimage->segment[i].mem + kimage->segment[i].memsz,
0092             kimage->segment[i].memsz,
0093             kimage->segment[i].memsz /  PAGE_SIZE);
0094
0095         dcache_clean_inval_poc(
0096             (unsigned long)phys_to_virt(kimage->segment[i].mem),
0097             (unsigned long)phys_to_virt(kimage->segment[i].mem) +
0098                 kimage->segment[i].memsz);
0099     }
0100 }
0101
0102 /* Allocates pages for kexec page table */
0103 static void *kexec_page_alloc(void *arg)
0104 {
0105     struct kimage *kimage = (struct kimage *)arg;
0106     struct page *page = kimage_alloc_control_pages(kimage, 0);
0107     void *vaddr = NULL;
0108
0109     if (!page)
0110         return NULL;
0111
0112     vaddr = page_address(page);
0113     memset(vaddr, 0, PAGE_SIZE);
0114
0115     return vaddr;
0116 }
0117
0118 int machine_kexec_post_load(struct kimage *kimage)
0119 {
0120     int rc;
0121     pgd_t *trans_pgd;
0122     void *reloc_code = page_to_virt(kimage->control_code_page);
0123     long reloc_size;
0124     struct trans_pgd_info info = {
0125         .trans_alloc_page   = kexec_page_alloc,
0126         .trans_alloc_arg    = kimage,
0127     };
0128
0129     /* If in place, relocation is not used, only flush next kernel */
0130     if (kimage->head & IND_DONE) {
0131         kexec_segment_flush(kimage);
0132         kexec_image_info(kimage);
0133         return 0;
0134     }
0135
0136     kimage->arch.el2_vectors = 0;
0137     if (is_hyp_nvhe()) {
0138         rc = trans_pgd_copy_el2_vectors(&info,
0139                         &kimage->arch.el2_vectors);
0140         if (rc)
0141             return rc;
0142     }
0143
0144     /* Create a copy of the linear map */
0145     trans_pgd = kexec_page_alloc(kimage);
0146     if (!trans_pgd)
0147         return -ENOMEM;
0148     rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END);
0149     if (rc)
0150         return rc;
0151     kimage->arch.ttbr1 = __pa(trans_pgd);
0152     kimage->arch.zero_page = __pa_symbol(empty_zero_page);
0153
0154     reloc_size = __relocate_new_kernel_end - __relocate_new_kernel_start;
0155     memcpy(reloc_code, __relocate_new_kernel_start, reloc_size);
0156     kimage->arch.kern_reloc = __pa(reloc_code);
0157     rc = trans_pgd_idmap_page(&info, &kimage->arch.ttbr0,
0158                   &kimage->arch.t0sz, reloc_code);
0159     if (rc)
0160         return rc;
0161     kimage->arch.phys_offset = virt_to_phys(kimage) - (long)kimage;
0162
0163     /* Flush the reloc_code in preparation for its execution. */
0164     dcache_clean_inval_poc((unsigned long)reloc_code,
0165                    (unsigned long)reloc_code + reloc_size);
0166     icache_inval_pou((uintptr_t)reloc_code,
0167              (uintptr_t)reloc_code + reloc_size);
0168     kexec_image_info(kimage);
0169
0170     return 0;
0171 }
0172
0173 /**
0174  * machine_kexec - Do the kexec reboot.
0175  *
0176  * Called from the core kexec code for a sys_reboot with LINUX_REBOOT_CMD_KEXEC.
0177  */
0178 void machine_kexec(struct kimage *kimage)
0179 {
0180     bool in_kexec_crash = (kimage == kexec_crash_image);
0181     bool stuck_cpus = cpus_are_stuck_in_kernel();
0182
0183     /*
0184      * New cpus may have become stuck_in_kernel after we loaded the image.
0185      */
0186     BUG_ON(!in_kexec_crash && (stuck_cpus || (num_online_cpus() > 1)));
0187     WARN(in_kexec_crash && (stuck_cpus || smp_crash_stop_failed()),
0188         "Some CPUs may be stale, kdump will be unreliable.\n");
0189
0190     pr_info("Bye!\n");
0191
0192     local_daif_mask();
0193
0194     /*
0195      * Both restart and kernel_reloc will shutdown the MMU, disable data
0196      * caches. However, restart will start new kernel or purgatory directly,
0197      * kernel_reloc contains the body of arm64_relocate_new_kernel
0198      * In kexec case, kimage->start points to purgatory assuming that
0199      * kernel entry and dtb address are embedded in purgatory by
0200      * userspace (kexec-tools).
0201      * In kexec_file case, the kernel starts directly without purgatory.
0202      */
0203     if (kimage->head & IND_DONE) {
0204         typeof(cpu_soft_restart) *restart;
0205
0206         cpu_install_idmap();
0207         restart = (void *)__pa_symbol(function_nocfi(cpu_soft_restart));
0208         restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
0209             0, 0);
0210     } else {
0211         void (*kernel_reloc)(struct kimage *kimage);
0212
0213         if (is_hyp_nvhe())
0214             __hyp_set_vectors(kimage->arch.el2_vectors);
0215         cpu_install_ttbr0(kimage->arch.ttbr0, kimage->arch.t0sz);
0216         kernel_reloc = (void *)kimage->arch.kern_reloc;
0217         kernel_reloc(kimage);
0218     }
0219
0220     BUG(); /* Should never get here. */
0221 }
0222
0223 static void machine_kexec_mask_interrupts(void)
0224 {
0225     unsigned int i;
0226     struct irq_desc *desc;
0227
0228     for_each_irq_desc(i, desc) {
0229         struct irq_chip *chip;
0230         int ret;
0231
0232         chip = irq_desc_get_chip(desc);
0233         if (!chip)
0234             continue;
0235
0236         /*
0237          * First try to remove the active state. If this
0238          * fails, try to EOI the interrupt.
0239          */
0240         ret = irq_set_irqchip_state(i, IRQCHIP_STATE_ACTIVE, false);
0241
0242         if (ret && irqd_irq_inprogress(&desc->irq_data) &&
0243             chip->irq_eoi)
0244             chip->irq_eoi(&desc->irq_data);
0245
0246         if (chip->irq_mask)
0247             chip->irq_mask(&desc->irq_data);
0248
0249         if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data))
0250             chip->irq_disable(&desc->irq_data);
0251     }
0252 }
0253
0254 /**
0255  * machine_crash_shutdown - shutdown non-crashing cpus and save registers
0256  */
0257 void machine_crash_shutdown(struct pt_regs *regs)
0258 {
0259     local_irq_disable();
0260
0261     /* shutdown non-crashing cpus */
0262     crash_smp_send_stop();
0263
0264     /* for crashing cpu */
0265     crash_save_cpu(regs, smp_processor_id());
0266     machine_kexec_mask_interrupts();
0267
0268     pr_info("Starting crashdump kernel...\n");
0269 }
0270
0271 void arch_kexec_protect_crashkres(void)
0272 {
0273     int i;
0274
0275     for (i = 0; i < kexec_crash_image->nr_segments; i++)
0276         set_memory_valid(
0277             __phys_to_virt(kexec_crash_image->segment[i].mem),
0278             kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 0);
0279 }
0280
0281 void arch_kexec_unprotect_crashkres(void)
0282 {
0283     int i;
0284
0285     for (i = 0; i < kexec_crash_image->nr_segments; i++)
0286         set_memory_valid(
0287             __phys_to_virt(kexec_crash_image->segment[i].mem),
0288             kexec_crash_image->segment[i].memsz >> PAGE_SHIFT, 1);
0289 }
0290
0291 #ifdef CONFIG_HIBERNATION
0292 /*
0293  * To preserve the crash dump kernel image, the relevant memory segments
0294  * should be mapped again around the hibernation.
0295  */
0296 void crash_prepare_suspend(void)
0297 {
0298     if (kexec_crash_image)
0299         arch_kexec_unprotect_crashkres();
0300 }
0301
0302 void crash_post_resume(void)
0303 {
0304     if (kexec_crash_image)
0305         arch_kexec_protect_crashkres();
0306 }
0307
0308 /*
0309  * crash_is_nosave
0310  *
0311  * Return true only if a page is part of reserved memory for crash dump kernel,
0312  * but does not hold any data of loaded kernel image.
0313  *
0314  * Note that all the pages in crash dump kernel memory have been initially
0315  * marked as Reserved as memory was allocated via memblock_reserve().
0316  *
0317  * In hibernation, the pages which are Reserved and yet "nosave" are excluded
0318  * from the hibernation iamge. crash_is_nosave() does thich check for crash
0319  * dump kernel and will reduce the total size of hibernation image.
0320  */
0321
0322 bool crash_is_nosave(unsigned long pfn)
0323 {
0324     int i;
0325     phys_addr_t addr;
0326
0327     if (!crashk_res.end)
0328         return false;
0329
0330     /* in reserved memory? */
0331     addr = __pfn_to_phys(pfn);
0332     if ((addr < crashk_res.start) || (crashk_res.end < addr)) {
0333         if (!crashk_low_res.end)
0334             return false;
0335
0336         if ((addr < crashk_low_res.start) || (crashk_low_res.end < addr))
0337             return false;
0338     }
0339
0340     if (!kexec_crash_image)
0341         return true;
0342
0343     /* not part of loaded kernel image? */
0344     for (i = 0; i < kexec_crash_image->nr_segments; i++)
0345         if (addr >= kexec_crash_image->segment[i].mem &&
0346                 addr < (kexec_crash_image->segment[i].mem +
0347                     kexec_crash_image->segment[i].memsz))
0348             return false;
0349
0350     return true;
0351 }
0352
0353 void crash_free_reserved_phys_range(unsigned long begin, unsigned long end)
0354 {
0355     unsigned long addr;
0356     struct page *page;
0357
0358     for (addr = begin; addr < end; addr += PAGE_SIZE) {
0359         page = phys_to_page(addr);
0360         free_reserved_page(page);
0361     }
0362 }
0363 #endif /* CONFIG_HIBERNATION */