x86/cet: Sync with Linux kernel 6.6 shadow stack interface

Sync with Linux kernel 6.6 shadow stack interface.  Since only x86-64 is
supported, i386 shadow stack codes are unchanged and CET shouldn't be
enabled for i386.

1. When the shadow stack base in TCB is unset, the default shadow stack
is in use.  Use the current shadow stack pointer as the marker for the
default shadow stack. It is used to identify if the current shadow stack
is the same as the target shadow stack when switching ucontexts.  If yes,
INCSSP will be used to unwind shadow stack.  Otherwise, shadow stack
restore token will be used.
2. Allocate shadow stack with the map_shadow_stack syscall.  Since there
is no function to explicitly release ucontext, there is no place to
release shadow stack allocated by map_shadow_stack in ucontext functions.
Such shadow stacks will be leaked.
3. Rename arch_prctl CET commands to ARCH_SHSTK_XXX.
4. Rewrite the CET control functions with the current kernel shadow stack
interface.

Since CET is no longer enabled by kernel, a separate patch will enable
shadow stack during startup.

sysdeps/unix/sysv/linux/x86/bits/mman.h

@@ -27,6 +27,11 @@
 #define MAP_32BIT	0x40		/* Only give out 32-bit addresses.  */
 #define MAP_ABOVE4G	0x80		/* Only map above 4GB.  */
 
+#ifdef __USE_MISC
+/* Set up a restore token in the newly allocated shadow stack */
+# define SHADOW_STACK_SET_TOKEN 0x1
+#endif
+
 #include <bits/mman-map-flags-generic.h>
 
 /* Include generic Linux declarations.  */

sysdeps/unix/sysv/linux/x86/cpu-features.c

@@ -23,10 +23,15 @@
 static inline int __attribute__ ((always_inline))
 get_cet_status (void)
 {
-  unsigned long long cet_status[3];
+  unsigned long long kernel_feature;
-  if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0)
+  unsigned int status = 0;
-    return cet_status[0];
+  if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS,
-  return 0;
+			     &kernel_feature) == 0)
+    {
+      if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0)
+	status = GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+    }
+  return status;
 }
 
 # ifndef SHARED

sysdeps/unix/sysv/linux/x86/dl-cet.h

@@ -21,12 +21,20 @@
 static inline int __attribute__ ((always_inline))
 dl_cet_disable_cet (unsigned int cet_feature)
 {
-  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE,
+  if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
-				      cet_feature);
+    return -1;
+  long long int kernel_feature = ARCH_SHSTK_SHSTK;
+  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE,
+				      kernel_feature);
 }
 
 static inline int __attribute__ ((always_inline))
-dl_cet_lock_cet (void)
+dl_cet_lock_cet (unsigned int cet_feature)
 {
-  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0);
+  if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
+    return -1;
+  /* Lock all SHSTK features.  */
+  long long int kernel_feature = -1;
+  return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK,
+				      kernel_feature);
 }

sysdeps/unix/sysv/linux/x86/include/asm/prctl.h

@@ -4,24 +4,19 @@
 
 #include_next <asm/prctl.h>
 
-#ifndef ARCH_CET_STATUS
+#ifndef ARCH_SHSTK_ENABLE
-/* CET features:
+/* Enable SHSTK features in unsigned long int features.  */
-   IBT:   GNU_PROPERTY_X86_FEATURE_1_IBT
+# define ARCH_SHSTK_ENABLE		0x5001
-   SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK
+/* Disable SHSTK features in unsigned long int features.  */
- */
+# define ARCH_SHSTK_DISABLE		0x5002
-/* Return CET features in unsigned long long *addr:
+/* Lock SHSTK features in unsigned long int features.  */
-     features: addr[0].
+# define ARCH_SHSTK_LOCK		0x5003
-     shadow stack base address: addr[1].
+/* Unlock SHSTK features in unsigned long int features.  */
-     shadow stack size: addr[2].
+# define ARCH_SHSTK_UNLOCK		0x5004
- */
+/* Return SHSTK features in unsigned long int features.  */
-# define ARCH_CET_STATUS	0x3001
+# define ARCH_SHSTK_STATUS		0x5005
-/* Disable CET features in unsigned int features.  */
+
-# define ARCH_CET_DISABLE	0x3002
+/* ARCH_SHSTK_ features bits */
-/* Lock all CET features.  */
+# define ARCH_SHSTK_SHSTK		0x1
-# define ARCH_CET_LOCK		0x3003
+# define ARCH_SHSTK_WRSS		0x2
-/* Allocate a new shadow stack with unsigned long long *addr:
+#endif
-     IN: requested shadow stack size: *addr.
-     OUT: allocated shadow stack address: *addr.
- */
-# define ARCH_CET_ALLOC_SHSTK	0x3004
-#endif /* ARCH_CET_STATUS */

sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c

@@ -87,15 +87,14 @@ do_test (void)
   ctx[4].uc_link = &ctx[0];
   makecontext (&ctx[4], (void (*) (void)) f1, 0);
 
-  /* NB: When shadow stack is enabled, makecontext calls arch_prctl
+  /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack
-     with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which
+     to allocate a new shadow stack which can be unmapped.  The base
-     can be unmapped.  The base address and size of the new shadow
+     address and size of the new shadow stack are returned in __ssp[1]
-     stack are returned in __ssp[1] and __ssp[2].  makecontext is
+     and __ssp[2].  makecontext is called for CTX1, CTX3 and CTX4.  But
-     called for CTX1, CTX3 and CTX4.  But only CTX1 is used.  New
+     only CTX1 is used.  New shadow stacks are allocated in the order
-     shadow stacks are allocated in the order of CTX3, CTX1, CTX4.
+     of CTX3, CTX1, CTX4.  It is very likely that CTX1's shadow stack is
-     It is very likely that CTX1's shadow stack is placed between
+     placed between CTX3 and CTX4.  We munmap CTX3's and CTX4's shadow
-     CTX3 and CTX4.  We munmap CTX3's and CTX4's shadow stacks to
+     stacks to create gaps above and below CTX1's shadow stack.  We check
-     create gaps above and below CTX1's shadow stack.  We check
      that setcontext CTX1 works correctly in this case.  */
   if (_get_ssp () != 0)
     {

sysdeps/unix/sysv/linux/x86_64/Makefile

@@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl
 endif
 
 ifeq ($(subdir),stdlib)
-sysdep_routines += __start_context
+sysdep_routines += __start_context allocate-shadow-stack
 endif
 
 ifeq ($(subdir),csu)

sysdeps/unix/sysv/linux/x86_64/__start_context.S

@@ -24,20 +24,14 @@
 /* Use CALL to push __start_context onto the new stack as well as the new
    shadow stack.  RDI points to ucontext:
    Incoming:
-     __ssp[0]: The original caller's shadow stack pointer.
-     __ssp[1]: The size of the new shadow stack.
-     __ssp[2]: The size of the new shadow stack.
-   Outgoing:
      __ssp[0]: The new shadow stack pointer.
      __ssp[1]: The base address of the new shadow stack.
      __ssp[2]: The size of the new shadow stack.
  */
 
 ENTRY(__push___start_context)
-	/* Save the pointer to ucontext.  */
-	movq	%rdi, %r9
 	/* Get the original shadow stack pointer.  */
-	rdsspq	%r8
+	rdsspq	%rcx
 	/* Save the original stack pointer.  */
 	movq	%rsp, %rdx
 	/* Load the top of the new stack into RSI.  */
@@ -45,24 +39,12 @@ ENTRY(__push___start_context)
 	/* Add 8 bytes to RSI since CALL will push the 8-byte return
 	   address onto stack.  */
 	leaq	8(%rsi), %rsp
-	/* Allocate the new shadow stack.  The size of the new shadow
+	/* The size of the new shadow stack is stored in __ssp[2].  */
-	   stack is passed in __ssp[1].  */
+	mov	(oSSP + 16)(%rdi), %RSI_LP
-	lea	(oSSP + 8)(%rdi), %RSI_LP
+	/* The new shadow stack base is stored in __ssp[1].  */
-	movl	$ARCH_CET_ALLOC_SHSTK, %edi
+	mov	(oSSP + 8)(%rdi), %RAX_LP
-	movl	$__NR_arch_prctl, %eax
-	/* The new shadow stack base is returned in __ssp[1].  */
-	syscall
-	testq	%rax, %rax
-	jne	L(hlt)		/* This should never happen.  */
-
-	/* Get the size of the new shadow stack.  */
-	movq	8(%rsi), %rdi
-
-	/* Get the base address of the new shadow stack.  */
-	movq	(%rsi), %rsi
-
 	/* Use the restore stoken to restore the new shadow stack.  */
-	rstorssp -8(%rsi, %rdi)
+	rstorssp -8(%rax, %rsi)
 
 	/* Save the restore token on the original shadow stack.  */
 	saveprevssp
@@ -73,18 +55,12 @@ ENTRY(__push___start_context)
 	jmp	__start_context
 1:
 
-	/* Get the new shadow stack pointer.  */
-	rdsspq	%rdi
-
 	/* Use the restore stoken to restore the original shadow stack.  */
-	rstorssp -8(%r8)
+	rstorssp -8(%rcx)
 
 	/* Save the restore token on the new shadow stack.  */
 	saveprevssp
 
-	/* Store the new shadow stack pointer in __ssp[0].  */
-	movq	%rdi, oSSP(%r9)
-
 	/* Restore the original stack.  */
 	mov	%rdx, %rsp
 	ret

sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c

@@ -0,0 +1,55 @@
+/* Helper function to allocate shadow stack.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <libc-pointer-arith.h>
+#include <allocate-shadow-stack.h>
+
+/* NB: This can be treated as a syscall by caller.  */
+
+long int
+__allocate_shadow_stack (size_t stack_size,
+			 shadow_stack_size_t *child_stack)
+{
+#ifdef __NR_map_shadow_stack
+  size_t shadow_stack_size
+    = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT;
+  /* Align shadow stack to 8 bytes.  */
+  shadow_stack_size = ALIGN_UP (shadow_stack_size, 8);
+  /* Since sigaltstack shares shadow stack with the current context in
+     the thread, add extra 20 stack frames in shadow stack for signal
+     handlers.  */
+  shadow_stack_size += 20 * 8;
+  void *shadow_stack = (void *)INLINE_SYSCALL_CALL
+    (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN);
+  /* Report the map_shadow_stack error.  */
+  if (shadow_stack == MAP_FAILED)
+    return -errno;
+
+  /* Save the shadow stack base and size on child stack.  */
+  child_stack[0] = (uintptr_t) shadow_stack;
+  child_stack[1] = shadow_stack_size;
+
+  return 0;
+#else
+  return -ENOSYS;
+#endif
+}

sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h

@@ -0,0 +1,24 @@
+/* Helper function to allocate shadow stack.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <ucontext.h>
+
+typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t;
+
+extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *)
+  attribute_hidden;

sysdeps/unix/sysv/linux/x86_64/getcontext.S

@@ -58,35 +58,15 @@ ENTRY(__getcontext)
 	testl	$X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET
 	jz	L(no_shstk)
 
-	/* Save RDI in RDX which won't be clobbered by syscall.  */
-	movq	%rdi, %rdx
-
 	xorl	%eax, %eax
 	cmpq	%fs:SSP_BASE_OFFSET, %rax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
+	/* When the shadow stack base is unset, the default shadow
-	   which must be the current shadow stack since nothing has
+	   stack is in use.  Use the current shadow stack pointer
-	   been recorded yet.  */
+	   as the marker for the default shadow stack.  */
-	sub	$24, %RSP_LP
+	rdsspq	%rax
-	mov	%RSP_LP, %RSI_LP
-	movl	$ARCH_CET_STATUS, %edi
-	movl	$__NR_arch_prctl, %eax
-	syscall
-	testq	%rax, %rax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movq	8(%rsp), %rax
 	movq	%rax, %fs:SSP_BASE_OFFSET
-	add	$24, %RSP_LP
-
-	/* Restore RDI.  */
-	movq	%rdx, %rdi
 
 L(shadow_stack_bound_recorded):
 	/* Get the current shadow stack pointer.  */
@@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded):
 	/* NB: Save the caller's shadow stack so that we can jump back
 	   to the caller directly.  */
 	addq	$8, %rax
-	movq	%rax, oSSP(%rdx)
+	movq	%rax, oSSP(%rdi)
 
 	/* Save the current shadow stack base in ucontext.  */
 	movq	%fs:SSP_BASE_OFFSET, %rax

sysdeps/unix/sysv/linux/x86_64/makecontext.c

@@ -24,6 +24,7 @@
 # include <pthread.h>
 # include <libc-pointer-arith.h>
 # include <sys/prctl.h>
+# include <allocate-shadow-stack.h>
 #endif
 
 #include "ucontext_i.h"
@@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...)
   if ((feature_1 & X86_FEATURE_1_SHSTK) != 0)
     {
       /* Shadow stack is enabled.  We need to allocate a new shadow
-         stack.  */
+         stack.  NB:
-      unsigned long ssp_size = (((uintptr_t) sp
-				 - (uintptr_t) ucp->uc_stack.ss_sp)
-				>> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT);
-      /* Align shadow stack to 8 bytes.  */
-      ssp_size = ALIGN_UP (ssp_size, 8);
-
-      ucp->__ssp[1] = ssp_size;
-      ucp->__ssp[2] = ssp_size;
-
-      /* Call __push___start_context to allocate a new shadow stack,
-	 push __start_context onto the new stack as well as the new
-	 shadow stack.  NB: After __push___start_context returns,
 	   ucp->__ssp[0]: The new shadow stack pointer.
 	   ucp->__ssp[1]: The base address of the new shadow stack.
 	   ucp->__ssp[2]: The size of the new shadow stack.
        */
+      long int ret
+	= __allocate_shadow_stack (((uintptr_t) sp
+				    - (uintptr_t) ucp->uc_stack.ss_sp),
+				   &ucp->__ssp[1]);
+      if (ret != 0)
+	{
+	  /* FIXME: What should we do?  */
+	  abort ();
+	}
+
+      ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8;
+      /* Call __push___start_context to push __start_context onto the new
+	 stack as well as the new shadow stack.  */
       __push___start_context (ucp);
     }
   else

sysdeps/unix/sysv/linux/x86_64/swapcontext.S

@@ -109,25 +109,11 @@ ENTRY(__swapcontext)
 	cmpq	%fs:SSP_BASE_OFFSET, %rax
 	jnz	L(shadow_stack_bound_recorded)
 
-	/* Get the base address and size of the default shadow stack
+	/* When the shadow stack base is unset, the default shadow
-	   which must be the current shadow stack since nothing has
+	   stack is in use.  Use the current shadow stack pointer
-	   been recorded yet.  */
+	   as the marker for the default shadow stack.  */
-	sub	$24, %RSP_LP
+	rdsspq	%rax
-	mov	%RSP_LP, %RSI_LP
-	movl	$ARCH_CET_STATUS, %edi
-	movl	$__NR_arch_prctl, %eax
-	syscall
-	testq	%rax, %rax
-	jz	L(continue_no_err)
-
-	/* This should never happen.  */
-	hlt
-
-L(continue_no_err):
-	/* Record the base of the current shadow stack.  */
-	movq	8(%rsp), %rax
 	movq	%rax, %fs:SSP_BASE_OFFSET
-	add	$24, %RSP_LP
 
 L(shadow_stack_bound_recorded):
         /* If we unwind the stack, we can't undo stack unwinding.  Just

sysdeps/x86/cpu-features.c

@@ -1121,8 +1121,9 @@ no_cpuid:
 
 # ifndef SHARED
       /* Check if IBT and SHSTK are enabled by kernel.  */
-      if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT)
+      if ((cet_status
-	  || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK))
+	   & (GNU_PROPERTY_X86_FEATURE_1_IBT
+	      | GNU_PROPERTY_X86_FEATURE_1_SHSTK)))
 	{
 	  /* Disable IBT and/or SHSTK if they are enabled by kernel, but
 	     disabled by environment variable:
@@ -1131,9 +1132,11 @@ no_cpuid:
 	   */
 	  unsigned int cet_feature = 0;
 	  if (!CPU_FEATURE_USABLE (IBT))
-	    cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT;
+	    cet_feature |= (cet_status
+			    & GNU_PROPERTY_X86_FEATURE_1_IBT);
 	  if (!CPU_FEATURE_USABLE (SHSTK))
-	    cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+	    cet_feature |= (cet_status
+			    & GNU_PROPERTY_X86_FEATURE_1_SHSTK);
 
 	  if (cet_feature)
 	    {
@@ -1148,7 +1151,9 @@ no_cpuid:
 	     lock CET if IBT or SHSTK is enabled permissively.  */
 	  if (GL(dl_x86_feature_control).ibt != cet_permissive
 	      && GL(dl_x86_feature_control).shstk != cet_permissive)
-	    dl_cet_lock_cet ();
+	    dl_cet_lock_cet (GL(dl_x86_feature_1)
+			     & (GNU_PROPERTY_X86_FEATURE_1_IBT
+				| GNU_PROPERTY_X86_FEATURE_1_SHSTK));
 	}
 # endif
     }

sysdeps/x86/dl-cet.c

@@ -202,7 +202,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info)
 	feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
 
       if (feature_1_lock != 0
-	  && dl_cet_lock_cet () != 0)
+	  && dl_cet_lock_cet (feature_1_lock) != 0)
 	_dl_fatal_printf ("%s: can't lock CET\n", info->program);
     }
 

sysdeps/x86_64/nptl/tls.h

@@ -60,7 +60,7 @@ typedef struct
   void *__private_tm[4];
   /* GCC split stack support.  */
   void *__private_ss;
-  /* The lowest address of shadow stack,  */
+  /* The marker for the current shadow stack.  */
   unsigned long long int ssp_base;
   /* Must be kept even if it is no longer used by glibc since programs,
      like AddressSanitizer, depend on the size of tcbhead_t.  */