mirror of
https://mirrors.bfsu.edu.cn/git/linux.git
synced 2024-12-05 18:14:07 +08:00
65c0554b73
Logan Gunthorpe reports that hibernation stopped working reliably for him after commitab76f7b4ab
(x86/mm: Set NX on gap between __ex_table and rodata). That turns out to be a consequence of a long-standing issue with the 64-bit image restoration code on x86, which is that the temporary page tables set up by it to avoid page tables corruption when the last bits of the image kernel's memory contents are copied into their original page frames re-use the boot kernel's text mapping, but that mapping may very well get corrupted just like any other part of the page tables. Of course, if that happens, the final jump to the image kernel's entry point will go to nowhere. The exact reason why commitab76f7b4ab
matters here is that it sometimes causes a PMD of a large page to be split into PTEs that are allocated dynamically and get corrupted during image restoration as described above. To fix that issue note that the code copying the last bits of the image kernel's memory contents to the page frames occupied by them previoulsy doesn't use the kernel text mapping, because it runs from a special page covered by the identity mapping set up for that code from scratch. Hence, the kernel text mapping is only needed before that code starts to run and then it will only be used just for the final jump to the image kernel's entry point. Accordingly, the temporary page tables set up in swsusp_arch_resume() on x86-64 need to contain the kernel text mapping too. That mapping is only going to be used for the final jump to the image kernel, so it only needs to cover the image kernel's entry point, because the first thing the image kernel does after getting control back is to switch over to its own original page tables. Moreover, the virtual address of the image kernel's entry point in that mapping has to be the same as the one mapped by the image kernel's page tables. With that in mind, modify the x86-64's arch_hibernation_header_save() and arch_hibernation_header_restore() routines to pass the physical address of the image kernel's entry point (in addition to its virtual address) to the boot kernel (a small piece of assembly code involved in passing the entry point's virtual address to the image kernel is not necessary any more after that, so drop it). Update RESTORE_MAGIC too to reflect the image header format change. Next, in set_up_temporary_mappings(), use the physical and virtual addresses of the image kernel's entry point passed in the image header to set up a minimum kernel text mapping (using memory pages that won't be overwritten by the image kernel's memory contents) that will map those addresses to each other as appropriate. This makes the concern about the possible corruption of the original boot kernel text mapping go away and if the the minimum kernel text mapping used for the final jump marks the image kernel's entry point memory as executable, the jump to it is guaraneed to succeed. Fixes:ab76f7b4ab
(x86/mm: Set NX on gap between __ex_table and rodata) Link: http://marc.info/?l=linux-pm&m=146372852823760&w=2 Reported-by: Logan Gunthorpe <logang@deltatee.com> Reported-and-tested-by: Borislav Petkov <bp@suse.de> Tested-by: Kees Cook <keescook@chromium.org> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
221 lines
5.6 KiB
C
221 lines
5.6 KiB
C
/*
|
|
* Hibernation support for x86-64
|
|
*
|
|
* Distribute under GPLv2
|
|
*
|
|
* Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
|
|
* Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
|
|
* Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
|
|
*/
|
|
|
|
#include <linux/gfp.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/suspend.h>
|
|
|
|
#include <asm/init.h>
|
|
#include <asm/proto.h>
|
|
#include <asm/page.h>
|
|
#include <asm/pgtable.h>
|
|
#include <asm/mtrr.h>
|
|
#include <asm/sections.h>
|
|
#include <asm/suspend.h>
|
|
#include <asm/tlbflush.h>
|
|
|
|
/* Defined in hibernate_asm_64.S */
|
|
extern asmlinkage __visible int restore_image(void);
|
|
|
|
/*
|
|
* Address to jump to in the last phase of restore in order to get to the image
|
|
* kernel's text (this value is passed in the image header).
|
|
*/
|
|
unsigned long restore_jump_address __visible;
|
|
unsigned long jump_address_phys;
|
|
|
|
/*
|
|
* Value of the cr3 register from before the hibernation (this value is passed
|
|
* in the image header).
|
|
*/
|
|
unsigned long restore_cr3 __visible;
|
|
|
|
pgd_t *temp_level4_pgt __visible;
|
|
|
|
unsigned long relocated_restore_code __visible;
|
|
|
|
static int set_up_temporary_text_mapping(void)
|
|
{
|
|
pmd_t *pmd;
|
|
pud_t *pud;
|
|
|
|
/*
|
|
* The new mapping only has to cover the page containing the image
|
|
* kernel's entry point (jump_address_phys), because the switch over to
|
|
* it is carried out by relocated code running from a page allocated
|
|
* specifically for this purpose and covered by the identity mapping, so
|
|
* the temporary kernel text mapping is only needed for the final jump.
|
|
* Moreover, in that mapping the virtual address of the image kernel's
|
|
* entry point must be the same as its virtual address in the image
|
|
* kernel (restore_jump_address), so the image kernel's
|
|
* restore_registers() code doesn't find itself in a different area of
|
|
* the virtual address space after switching over to the original page
|
|
* tables used by the image kernel.
|
|
*/
|
|
pud = (pud_t *)get_safe_page(GFP_ATOMIC);
|
|
if (!pud)
|
|
return -ENOMEM;
|
|
|
|
pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
|
|
if (!pmd)
|
|
return -ENOMEM;
|
|
|
|
set_pmd(pmd + pmd_index(restore_jump_address),
|
|
__pmd((jump_address_phys & PMD_MASK) | __PAGE_KERNEL_LARGE_EXEC));
|
|
set_pud(pud + pud_index(restore_jump_address),
|
|
__pud(__pa(pmd) | _KERNPG_TABLE));
|
|
set_pgd(temp_level4_pgt + pgd_index(restore_jump_address),
|
|
__pgd(__pa(pud) | _KERNPG_TABLE));
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void *alloc_pgt_page(void *context)
|
|
{
|
|
return (void *)get_safe_page(GFP_ATOMIC);
|
|
}
|
|
|
|
static int set_up_temporary_mappings(void)
|
|
{
|
|
struct x86_mapping_info info = {
|
|
.alloc_pgt_page = alloc_pgt_page,
|
|
.pmd_flag = __PAGE_KERNEL_LARGE_EXEC,
|
|
.kernel_mapping = true,
|
|
};
|
|
unsigned long mstart, mend;
|
|
int result;
|
|
int i;
|
|
|
|
temp_level4_pgt = (pgd_t *)get_safe_page(GFP_ATOMIC);
|
|
if (!temp_level4_pgt)
|
|
return -ENOMEM;
|
|
|
|
/* Prepare a temporary mapping for the kernel text */
|
|
result = set_up_temporary_text_mapping();
|
|
if (result)
|
|
return result;
|
|
|
|
/* Set up the direct mapping from scratch */
|
|
for (i = 0; i < nr_pfn_mapped; i++) {
|
|
mstart = pfn_mapped[i].start << PAGE_SHIFT;
|
|
mend = pfn_mapped[i].end << PAGE_SHIFT;
|
|
|
|
result = kernel_ident_mapping_init(&info, temp_level4_pgt,
|
|
mstart, mend);
|
|
|
|
if (result)
|
|
return result;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int relocate_restore_code(void)
|
|
{
|
|
pgd_t *pgd;
|
|
pud_t *pud;
|
|
|
|
relocated_restore_code = get_safe_page(GFP_ATOMIC);
|
|
if (!relocated_restore_code)
|
|
return -ENOMEM;
|
|
|
|
memcpy((void *)relocated_restore_code, &core_restore_code, PAGE_SIZE);
|
|
|
|
/* Make the page containing the relocated code executable */
|
|
pgd = (pgd_t *)__va(read_cr3()) + pgd_index(relocated_restore_code);
|
|
pud = pud_offset(pgd, relocated_restore_code);
|
|
if (pud_large(*pud)) {
|
|
set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX));
|
|
} else {
|
|
pmd_t *pmd = pmd_offset(pud, relocated_restore_code);
|
|
|
|
if (pmd_large(*pmd)) {
|
|
set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX));
|
|
} else {
|
|
pte_t *pte = pte_offset_kernel(pmd, relocated_restore_code);
|
|
|
|
set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX));
|
|
}
|
|
}
|
|
__flush_tlb_all();
|
|
|
|
return 0;
|
|
}
|
|
|
|
int swsusp_arch_resume(void)
|
|
{
|
|
int error;
|
|
|
|
/* We have got enough memory and from now on we cannot recover */
|
|
error = set_up_temporary_mappings();
|
|
if (error)
|
|
return error;
|
|
|
|
error = relocate_restore_code();
|
|
if (error)
|
|
return error;
|
|
|
|
restore_image();
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* pfn_is_nosave - check if given pfn is in the 'nosave' section
|
|
*/
|
|
|
|
int pfn_is_nosave(unsigned long pfn)
|
|
{
|
|
unsigned long nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT;
|
|
unsigned long nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT;
|
|
return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
|
|
}
|
|
|
|
struct restore_data_record {
|
|
unsigned long jump_address;
|
|
unsigned long jump_address_phys;
|
|
unsigned long cr3;
|
|
unsigned long magic;
|
|
};
|
|
|
|
#define RESTORE_MAGIC 0x123456789ABCDEF0UL
|
|
|
|
/**
|
|
* arch_hibernation_header_save - populate the architecture specific part
|
|
* of a hibernation image header
|
|
* @addr: address to save the data at
|
|
*/
|
|
int arch_hibernation_header_save(void *addr, unsigned int max_size)
|
|
{
|
|
struct restore_data_record *rdr = addr;
|
|
|
|
if (max_size < sizeof(struct restore_data_record))
|
|
return -EOVERFLOW;
|
|
rdr->jump_address = (unsigned long)&restore_registers;
|
|
rdr->jump_address_phys = __pa_symbol(&restore_registers);
|
|
rdr->cr3 = restore_cr3;
|
|
rdr->magic = RESTORE_MAGIC;
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* arch_hibernation_header_restore - read the architecture specific data
|
|
* from the hibernation image header
|
|
* @addr: address to read the data from
|
|
*/
|
|
int arch_hibernation_header_restore(void *addr)
|
|
{
|
|
struct restore_data_record *rdr = addr;
|
|
|
|
restore_jump_address = rdr->jump_address;
|
|
jump_address_phys = rdr->jump_address_phys;
|
|
restore_cr3 = rdr->cr3;
|
|
return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL;
|
|
}
|