gdb: Respect the DW_CC_nocall attribute

It is possible for a compiler to optimize a function in a such ways that
the function does not follow the calling convention of the target.  In
such situation, the compiler can use the DW_AT_calling_convention
attribute with the value DW_CC_nocall to tell the debugger that it is
unsafe to call the function.  The DWARF5 standard states, in 3.3.1.1:

  > If the value of the calling convention attribute is the constant
  > DW_CC_nocall, the subroutine does not obey standard calling
  > conventions, and it may not be safe for the debugger to call this
  > subroutine.

Non standard calling convention can affect GDB's assumptions in multiple
ways, including how arguments are passed to the function, how values are
returned, and so on.  For this reason, it is unsafe for GDB to try to do
the following operations on a function with marked with DW_CC_nocall:

- call / print an expression requiring the function to be evaluated,
- inspect the value a function returns using the 'finish' command,
- force the value returned by a function using the 'return' command.

This patch ensures that if a command which relies on GDB's knowledge of
the target's calling convention is used on a function marked nocall, GDB
prints an appropriate message to the user and does not proceed with the
operation which is unreliable.

Note that it is still possible for someone to use a vendor specific
value for the DW_AT_calling_convention attribute for example to indicate
the use of an alternative calling convention.  This commit does not
prevent this, and target dependent code can be adjusted if one wanted to
support multiple calling conventions.

Tested on x86_64-Linux, with no regression observed.

Change-Id: I72970dae68234cb83edbc0cf71aa3d6002a4a540

gdb/gdbtypes.c

@@ -3903,6 +3903,17 @@ type_byte_order (const struct type *type)
   return byteorder;
 }
 
+/* See gdbtypes.h.  */
+
+bool
+is_nocall_function (const struct type *type)
+{
+  gdb_assert (type->code () == TYPE_CODE_FUNC
+	      || type->code () == TYPE_CODE_METHOD);
+
+  return TYPE_CALLING_CONVENTION (type) == DW_CC_nocall;
+}
+
 
 /* Overload resolution.  */
 

gdb/gdbtypes.h

@@ -2865,4 +2865,14 @@ extern enum bfd_endian type_byte_order (const struct type *type);
 
 extern unsigned int overload_debug;
 
+/* Return whether the function type represented by TYPE is marked as unsafe
+   to call by the debugger.
+
+   This usually indicates that the function does not follow the target's
+   standard calling convention.
+
+   The TYPE argument must be of code TYPE_CODE_FUNC or TYPE_CODE_METHOD.  */
+
+extern bool is_nocall_function (const struct type *type);
+
 #endif /* GDBTYPES_H */

gdb/infcall.c

@@ -812,6 +812,11 @@ call_function_by_hand_dummy (struct value *function,
   type *values_type;
   CORE_ADDR funaddr = find_function_addr (function, &values_type, &ftype);
 
+  if (is_nocall_function (ftype))
+    error (_("Cannot call the function '%s' which does not follow the "
+	     "target calling convention."),
+	   get_function_name (funaddr, name_buf, sizeof (name_buf)));
+
   if (values_type == NULL)
     values_type = default_return_type;
   if (values_type == NULL)

gdb/infcmd.c

@@ -1457,6 +1457,15 @@ get_return_value (struct symbol *func_symbol, struct value *function)
     = check_typedef (TYPE_TARGET_TYPE (func_symbol->type ()));
   gdb_assert (value_type->code () != TYPE_CODE_VOID);
 
+  if (is_nocall_function (check_typedef (::value_type (function))))
+    {
+      warning (_("Function '%s' does not follow the target calling "
+		 "convention, cannot determine its returned value."),
+	       func_symbol->print_name ());
+
+      return nullptr;
+    }
+
   /* FIXME: 2003-09-27: When returning from a nested inferior function
      call, it's possible (with no help from the architecture vector)
      to locate and return/print a "struct return" value.  This is just

gdb/stack.c

@@ -2737,7 +2737,7 @@ return_command (const char *retval_exp, int from_tty)
   struct symbol *thisfun;
   struct value *return_value = NULL;
   struct value *function = NULL;
-  const char *query_prefix = "";
+  std::string query_prefix;
 
   thisframe = get_selected_frame ("No selected frame.");
   thisfun = get_frame_function (thisframe);
@@ -2793,6 +2793,17 @@ return_command (const char *retval_exp, int from_tty)
 	return_value = NULL;
       else if (thisfun != NULL)
 	{
+	  if (is_nocall_function (check_typedef (value_type (function))))
+	    {
+	      query_prefix =
+		string_printf ("Function '%s' does not follow the target "
+			       "calling convention.\n"
+			       "If you continue, setting the return value "
+			       "will probably lead to unpredictable "
+			       "behaviors.\n",
+			       thisfun->print_name ());
+	    }
+
 	  rv_conv = struct_return_convention (gdbarch, function, return_type);
 	  if (rv_conv == RETURN_VALUE_STRUCT_CONVENTION
 	      || rv_conv == RETURN_VALUE_ABI_RETURNS_ADDRESS)
@@ -2815,12 +2826,13 @@ return_command (const char *retval_exp, int from_tty)
 
       if (thisfun == NULL)
 	confirmed = query (_("%sMake selected stack frame return now? "),
-			   query_prefix);
+			   query_prefix.c_str ());
       else
 	{
 	  if (TYPE_NO_RETURN (thisfun->type ()))
 	    warning (_("Function does not return normally to caller."));
-	  confirmed = query (_("%sMake %s return now? "), query_prefix,
+	  confirmed = query (_("%sMake %s return now? "),
+			     query_prefix.c_str (),
 			     thisfun->print_name ());
 	}
       if (!confirmed)

gdb/testsuite/gdb.dwarf2/calling-convention.c

@@ -0,0 +1,35 @@
+/* This testcase is part of GDB, the GNU debugger.
+
+   Copyright 2022 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+
+/* Dummy foo function.  */
+
+int
+foo (void)
+{
+  asm ("foo_label: .globl foo_label");
+  return 42;
+}
+
+/* Dummy main function.  */
+
+int
+main (void)
+{
+  asm ("main_label: .globl main_label");
+  foo ();
+  return 0;
+}

gdb/testsuite/gdb.dwarf2/calling-convention.exp

@@ -0,0 +1,97 @@
+# Copyright 2022 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This testcase checks that if a function has the DW_AT_calling_convention
+# attribute with the value DW_CC_nocall, then GDB will not:
+# - call the function,
+# - try to access the value returned by the function when using the finish
+#   command,
+# - force a user-provided return value when using the return command.
+#
+# In every case, GDB prints a message to the user indicating the issue.  For
+# the return command, GDB asks the user to confirm if the specified value
+# should be forced.
+
+load_lib dwarf.exp
+
+# This test can only be run on targets which support DWARF-2 and use gas.
+if {![dwarf2_support]} {
+    return 0
+}
+
+standard_testfile .c .S
+
+# First compile the .c file so we can ask GDB what is sizeof(int).
+if { [prepare_for_testing ${testfile}.exp ${testfile} ${srcfile}] } {
+    untested "failed to compile"
+    return -1
+}
+
+# Make some DWARF for the test.
+set asm_file [standard_output_file $srcfile2]
+Dwarf::assemble $asm_file {
+    cu {} {
+	compile_unit {
+	    {language @DW_LANG_C}
+	    {name "calling-convention"}
+	} {
+	    declare_labels int_label
+
+	    int_label: base_type {
+		{byte_size [get_sizeof "int" 4] sdata}
+		{encoding @DW_ATE_signed}
+		{name "int"}
+	    }
+
+	    subprogram {
+		{MACRO_AT_func { foo }}
+		{type :$int_label}
+		{calling_convention @DW_CC_nocall}
+	    }
+
+	    subprogram {
+		{MACRO_AT_func { main }}
+		{type :$int_label}
+	    }
+	}
+    }
+}
+
+if { [prepare_for_testing "failed to prepare" ${testfile} \
+	  [list $srcfile $asm_file] {nodebug}] } {
+    return -1
+}
+
+if {![runto_main]} {
+    return -1
+}
+
+gdb_test "call foo ()" \
+    "Cannot call the function 'foo' which does not follow the target calling convention."
+gdb_breakpoint "foo"
+gdb_continue_to_breakpoint "foo"
+
+gdb_test_multiple "return 35" "" {
+    -re ".*Function 'foo' does not follow the target calling convention.\r\nIf you continue, setting the return value will probably lead to unpredictable behaviors.\r\nMake foo return now?.*\\(y or n\\) $" {
+	send_gdb "n\n"
+	pass $gdb_test_name
+    }
+}
+
+gdb_test "finish" [multi_line \
+    "Run till exit from #0  $hex in foo \\(\\)" \
+    "warning: Function 'foo' does not follow the target calling convention, cannot determine its returned value\." \
+    "$hex in main \\(\\)" \
+    "Value returned has type: int. Cannot determine contents"]