2012-03-05 19:49:31 +08:00
|
|
|
/*
|
|
|
|
* VDSO implementation for AArch64 and vector page setup for AArch32.
|
|
|
|
*
|
|
|
|
* Copyright (C) 2012 ARM Limited
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*
|
|
|
|
* Author: Will Deacon <will.deacon@arm.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/clocksource.h>
|
|
|
|
#include <linux/elf.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/gfp.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/signal.h>
|
|
|
|
#include <linux/slab.h>
|
2012-10-16 18:44:53 +08:00
|
|
|
#include <linux/timekeeper_internal.h>
|
2012-03-05 19:49:31 +08:00
|
|
|
#include <linux/vmalloc.h>
|
|
|
|
|
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
#include <asm/signal32.h>
|
|
|
|
#include <asm/vdso.h>
|
|
|
|
#include <asm/vdso_datapage.h>
|
|
|
|
|
|
|
|
extern char vdso_start, vdso_end;
|
|
|
|
static unsigned long vdso_pages;
|
|
|
|
static struct page **vdso_pagelist;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The vDSO data page.
|
|
|
|
*/
|
|
|
|
static union {
|
|
|
|
struct vdso_data data;
|
|
|
|
u8 page[PAGE_SIZE];
|
|
|
|
} vdso_data_store __page_aligned_data;
|
|
|
|
struct vdso_data *vdso_data = &vdso_data_store.data;
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
/*
|
|
|
|
* Create and map the vectors page for AArch32 tasks.
|
|
|
|
*/
|
|
|
|
static struct page *vectors_page[1];
|
|
|
|
|
2016-08-15 14:45:44 +08:00
|
|
|
static int __init alloc_vectors_page(void)
|
2012-03-05 19:49:31 +08:00
|
|
|
{
|
|
|
|
extern char __kuser_helper_start[], __kuser_helper_end[];
|
2013-10-11 21:52:14 +08:00
|
|
|
extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[];
|
|
|
|
|
2012-03-05 19:49:31 +08:00
|
|
|
int kuser_sz = __kuser_helper_end - __kuser_helper_start;
|
2013-10-11 21:52:14 +08:00
|
|
|
int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start;
|
2012-03-05 19:49:31 +08:00
|
|
|
unsigned long vpage;
|
|
|
|
|
|
|
|
vpage = get_zeroed_page(GFP_ATOMIC);
|
|
|
|
|
|
|
|
if (!vpage)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
/* kuser helpers */
|
|
|
|
memcpy((void *)vpage + 0x1000 - kuser_sz, __kuser_helper_start,
|
|
|
|
kuser_sz);
|
|
|
|
|
|
|
|
/* sigreturn code */
|
|
|
|
memcpy((void *)vpage + AARCH32_KERN_SIGRET_CODE_OFFSET,
|
2013-10-11 21:52:14 +08:00
|
|
|
__aarch32_sigret_code_start, sigret_sz);
|
2012-03-05 19:49:31 +08:00
|
|
|
|
|
|
|
flush_icache_range(vpage, vpage + PAGE_SIZE);
|
|
|
|
vectors_page[0] = virt_to_page(vpage);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
arch_initcall(alloc_vectors_page);
|
|
|
|
|
|
|
|
int aarch32_setup_vectors_page(struct linux_binprm *bprm, int uses_interp)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
unsigned long addr = AARCH32_VECTORS_BASE;
|
2014-07-10 02:22:12 +08:00
|
|
|
static struct vm_special_mapping spec = {
|
|
|
|
.name = "[vectors]",
|
|
|
|
.pages = vectors_page,
|
|
|
|
|
|
|
|
};
|
|
|
|
void *ret;
|
2012-03-05 19:49:31 +08:00
|
|
|
|
2016-05-24 07:25:54 +08:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
return -EINTR;
|
2012-03-05 19:49:31 +08:00
|
|
|
current->mm->context.vdso = (void *)addr;
|
|
|
|
|
|
|
|
/* Map vectors page at the high address. */
|
2014-07-10 02:22:12 +08:00
|
|
|
ret = _install_special_mapping(mm, addr, PAGE_SIZE,
|
|
|
|
VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYEXEC,
|
|
|
|
&spec);
|
2012-03-05 19:49:31 +08:00
|
|
|
|
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
|
2014-07-10 02:22:12 +08:00
|
|
|
return PTR_ERR_OR_ZERO(ret);
|
2012-03-05 19:49:31 +08:00
|
|
|
}
|
|
|
|
#endif /* CONFIG_COMPAT */
|
|
|
|
|
2014-07-10 02:22:12 +08:00
|
|
|
static struct vm_special_mapping vdso_spec[2];
|
|
|
|
|
2012-03-05 19:49:31 +08:00
|
|
|
static int __init vdso_init(void)
|
|
|
|
{
|
2014-02-12 06:28:42 +08:00
|
|
|
int i;
|
|
|
|
|
|
|
|
if (memcmp(&vdso_start, "\177ELF", 4)) {
|
|
|
|
pr_err("vDSO is not a valid ELF object!\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2012-03-05 19:49:31 +08:00
|
|
|
|
|
|
|
vdso_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT;
|
2014-07-10 02:22:13 +08:00
|
|
|
pr_info("vdso: %ld pages (%ld code @ %p, %ld data @ %p)\n",
|
|
|
|
vdso_pages + 1, vdso_pages, &vdso_start, 1L, vdso_data);
|
2012-03-05 19:49:31 +08:00
|
|
|
|
|
|
|
/* Allocate the vDSO pagelist, plus a page for the data. */
|
2014-02-12 06:28:42 +08:00
|
|
|
vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *),
|
2012-03-05 19:49:31 +08:00
|
|
|
GFP_KERNEL);
|
2014-02-12 06:28:42 +08:00
|
|
|
if (vdso_pagelist == NULL)
|
2012-03-05 19:49:31 +08:00
|
|
|
return -ENOMEM;
|
|
|
|
|
2014-07-10 02:22:13 +08:00
|
|
|
/* Grab the vDSO data page. */
|
2016-03-30 22:45:56 +08:00
|
|
|
vdso_pagelist[0] = pfn_to_page(PHYS_PFN(__pa(vdso_data)));
|
2014-07-10 02:22:13 +08:00
|
|
|
|
2012-03-05 19:49:31 +08:00
|
|
|
/* Grab the vDSO code pages. */
|
2014-02-12 06:28:42 +08:00
|
|
|
for (i = 0; i < vdso_pages; i++)
|
2016-03-30 22:45:56 +08:00
|
|
|
vdso_pagelist[i + 1] = pfn_to_page(PHYS_PFN(__pa(&vdso_start)) + i);
|
2012-03-05 19:49:31 +08:00
|
|
|
|
2014-07-10 02:22:12 +08:00
|
|
|
/* Populate the special mapping structures */
|
|
|
|
vdso_spec[0] = (struct vm_special_mapping) {
|
2014-07-10 02:22:13 +08:00
|
|
|
.name = "[vvar]",
|
2014-07-10 02:22:12 +08:00
|
|
|
.pages = vdso_pagelist,
|
|
|
|
};
|
|
|
|
|
|
|
|
vdso_spec[1] = (struct vm_special_mapping) {
|
2014-07-10 02:22:13 +08:00
|
|
|
.name = "[vdso]",
|
|
|
|
.pages = &vdso_pagelist[1],
|
2014-07-10 02:22:12 +08:00
|
|
|
};
|
|
|
|
|
2014-02-12 06:28:42 +08:00
|
|
|
return 0;
|
2012-03-05 19:49:31 +08:00
|
|
|
}
|
|
|
|
arch_initcall(vdso_init);
|
|
|
|
|
|
|
|
int arch_setup_additional_pages(struct linux_binprm *bprm,
|
|
|
|
int uses_interp)
|
|
|
|
{
|
|
|
|
struct mm_struct *mm = current->mm;
|
2014-07-10 02:22:11 +08:00
|
|
|
unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
|
2014-07-10 02:22:12 +08:00
|
|
|
void *ret;
|
2012-03-05 19:49:31 +08:00
|
|
|
|
2014-07-10 02:22:11 +08:00
|
|
|
vdso_text_len = vdso_pages << PAGE_SHIFT;
|
2012-03-05 19:49:31 +08:00
|
|
|
/* Be sure to map the data page */
|
2014-07-10 02:22:11 +08:00
|
|
|
vdso_mapping_len = vdso_text_len + PAGE_SIZE;
|
2012-03-05 19:49:31 +08:00
|
|
|
|
2016-05-24 07:25:54 +08:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
return -EINTR;
|
2012-03-05 19:49:31 +08:00
|
|
|
vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0);
|
|
|
|
if (IS_ERR_VALUE(vdso_base)) {
|
2014-07-10 02:22:12 +08:00
|
|
|
ret = ERR_PTR(vdso_base);
|
2012-03-05 19:49:31 +08:00
|
|
|
goto up_fail;
|
|
|
|
}
|
2014-07-10 02:22:13 +08:00
|
|
|
ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE,
|
|
|
|
VM_READ|VM_MAYREAD,
|
2014-07-10 02:22:12 +08:00
|
|
|
&vdso_spec[0]);
|
|
|
|
if (IS_ERR(ret))
|
2014-07-10 02:22:11 +08:00
|
|
|
goto up_fail;
|
|
|
|
|
2014-07-10 02:22:13 +08:00
|
|
|
vdso_base += PAGE_SIZE;
|
|
|
|
mm->context.vdso = (void *)vdso_base;
|
|
|
|
ret = _install_special_mapping(mm, vdso_base, vdso_text_len,
|
|
|
|
VM_READ|VM_EXEC|
|
|
|
|
VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
|
2014-07-10 02:22:12 +08:00
|
|
|
&vdso_spec[1]);
|
|
|
|
if (IS_ERR(ret))
|
2012-03-05 19:49:31 +08:00
|
|
|
goto up_fail;
|
|
|
|
|
2014-07-10 02:22:13 +08:00
|
|
|
|
2012-03-05 19:49:31 +08:00
|
|
|
up_write(&mm->mmap_sem);
|
2014-07-10 02:22:11 +08:00
|
|
|
return 0;
|
2012-03-05 19:49:31 +08:00
|
|
|
|
2014-07-10 02:22:11 +08:00
|
|
|
up_fail:
|
|
|
|
mm->context.vdso = NULL;
|
|
|
|
up_write(&mm->mmap_sem);
|
2014-07-10 02:22:12 +08:00
|
|
|
return PTR_ERR(ret);
|
2012-03-05 19:49:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Update the vDSO data page to keep in sync with kernel timekeeping.
|
|
|
|
*/
|
2012-10-16 18:44:53 +08:00
|
|
|
void update_vsyscall(struct timekeeper *tk)
|
2012-03-05 19:49:31 +08:00
|
|
|
{
|
2015-03-19 17:09:06 +08:00
|
|
|
u32 use_syscall = strcmp(tk->tkr_mono.clock->name, "arch_sys_counter");
|
2012-03-05 19:49:31 +08:00
|
|
|
|
|
|
|
++vdso_data->tb_seq_count;
|
|
|
|
smp_wmb();
|
|
|
|
|
|
|
|
vdso_data->use_syscall = use_syscall;
|
2015-08-08 10:03:23 +08:00
|
|
|
vdso_data->xtime_coarse_sec = tk->xtime_sec;
|
|
|
|
vdso_data->xtime_coarse_nsec = tk->tkr_mono.xtime_nsec >>
|
|
|
|
tk->tkr_mono.shift;
|
2014-02-04 03:48:52 +08:00
|
|
|
vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec;
|
|
|
|
vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec;
|
2012-03-05 19:49:31 +08:00
|
|
|
|
|
|
|
if (!use_syscall) {
|
arm64: Add support for CLOCK_MONOTONIC_RAW in clock_gettime() vDSO
So far the arm64 clock_gettime() vDSO implementation only supported
the following clocks, falling back to the syscall for the others:
- CLOCK_REALTIME{,_COARSE}
- CLOCK_MONOTONIC{,_COARSE}
This patch adds support for the CLOCK_MONOTONIC_RAW clock, taking
advantage of the recent refactoring of the vDSO time functions. Like
the non-_COARSE clocks, this only works when the "arch_sys_counter"
clocksource is in use (allowing us to read the current time from the
virtual counter register), otherwise we also have to fall back to the
syscall.
Most of the data is shared with CLOCK_MONOTONIC, and the algorithm is
similar. The reference implementation in kernel/time/timekeeping.c
shows that:
- CLOCK_MONOTONIC = tk->wall_to_monotonic + tk->xtime_sec +
timekeeping_get_ns(&tk->tkr_mono)
- CLOCK_MONOTONIC_RAW = tk->raw_time + timekeeping_get_ns(&tk->tkr_raw)
- tkr_mono and tkr_raw are identical (in particular, same
clocksource), except these members:
* mult (only mono's multiplier is NTP-adjusted)
* xtime_nsec (always 0 for raw)
Therefore, tk->raw_time and tkr_raw->mult are now also stored in the
vDSO data page.
Cc: Ali Saidi <ali.saidi@arm.com>
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Reviewed-by: Dave Martin <dave.martin@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-12 18:24:00 +08:00
|
|
|
/* tkr_mono.cycle_last == tkr_raw.cycle_last */
|
2015-03-19 17:09:06 +08:00
|
|
|
vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last;
|
arm64: Add support for CLOCK_MONOTONIC_RAW in clock_gettime() vDSO
So far the arm64 clock_gettime() vDSO implementation only supported
the following clocks, falling back to the syscall for the others:
- CLOCK_REALTIME{,_COARSE}
- CLOCK_MONOTONIC{,_COARSE}
This patch adds support for the CLOCK_MONOTONIC_RAW clock, taking
advantage of the recent refactoring of the vDSO time functions. Like
the non-_COARSE clocks, this only works when the "arch_sys_counter"
clocksource is in use (allowing us to read the current time from the
virtual counter register), otherwise we also have to fall back to the
syscall.
Most of the data is shared with CLOCK_MONOTONIC, and the algorithm is
similar. The reference implementation in kernel/time/timekeeping.c
shows that:
- CLOCK_MONOTONIC = tk->wall_to_monotonic + tk->xtime_sec +
timekeeping_get_ns(&tk->tkr_mono)
- CLOCK_MONOTONIC_RAW = tk->raw_time + timekeeping_get_ns(&tk->tkr_raw)
- tkr_mono and tkr_raw are identical (in particular, same
clocksource), except these members:
* mult (only mono's multiplier is NTP-adjusted)
* xtime_nsec (always 0 for raw)
Therefore, tk->raw_time and tkr_raw->mult are now also stored in the
vDSO data page.
Cc: Ali Saidi <ali.saidi@arm.com>
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Reviewed-by: Dave Martin <dave.martin@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-12 18:24:00 +08:00
|
|
|
vdso_data->raw_time_sec = tk->raw_time.tv_sec;
|
|
|
|
vdso_data->raw_time_nsec = tk->raw_time.tv_nsec;
|
2012-10-16 18:44:53 +08:00
|
|
|
vdso_data->xtime_clock_sec = tk->xtime_sec;
|
2015-03-19 17:09:06 +08:00
|
|
|
vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec;
|
arm64: Add support for CLOCK_MONOTONIC_RAW in clock_gettime() vDSO
So far the arm64 clock_gettime() vDSO implementation only supported
the following clocks, falling back to the syscall for the others:
- CLOCK_REALTIME{,_COARSE}
- CLOCK_MONOTONIC{,_COARSE}
This patch adds support for the CLOCK_MONOTONIC_RAW clock, taking
advantage of the recent refactoring of the vDSO time functions. Like
the non-_COARSE clocks, this only works when the "arch_sys_counter"
clocksource is in use (allowing us to read the current time from the
virtual counter register), otherwise we also have to fall back to the
syscall.
Most of the data is shared with CLOCK_MONOTONIC, and the algorithm is
similar. The reference implementation in kernel/time/timekeeping.c
shows that:
- CLOCK_MONOTONIC = tk->wall_to_monotonic + tk->xtime_sec +
timekeeping_get_ns(&tk->tkr_mono)
- CLOCK_MONOTONIC_RAW = tk->raw_time + timekeeping_get_ns(&tk->tkr_raw)
- tkr_mono and tkr_raw are identical (in particular, same
clocksource), except these members:
* mult (only mono's multiplier is NTP-adjusted)
* xtime_nsec (always 0 for raw)
Therefore, tk->raw_time and tkr_raw->mult are now also stored in the
vDSO data page.
Cc: Ali Saidi <ali.saidi@arm.com>
Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com>
Reviewed-by: Dave Martin <dave.martin@arm.com>
Acked-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-07-12 18:24:00 +08:00
|
|
|
/* tkr_raw.xtime_nsec == 0 */
|
|
|
|
vdso_data->cs_mono_mult = tk->tkr_mono.mult;
|
|
|
|
vdso_data->cs_raw_mult = tk->tkr_raw.mult;
|
|
|
|
/* tkr_mono.shift == tkr_raw.shift */
|
2015-03-19 17:09:06 +08:00
|
|
|
vdso_data->cs_shift = tk->tkr_mono.shift;
|
2012-03-05 19:49:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
smp_wmb();
|
|
|
|
++vdso_data->tb_seq_count;
|
|
|
|
}
|
|
|
|
|
|
|
|
void update_vsyscall_tz(void)
|
|
|
|
{
|
|
|
|
vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
|
|
|
|
vdso_data->tz_dsttime = sys_tz.tz_dsttime;
|
|
|
|
}
|