From ba6adb2dc97f3ab03728736f4bb6bd0b3fc0b240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Sat, 17 Jan 2026 05:24:24 +0100 Subject: [PATCH] libunwind: update to LLVM 22 --- lib/libunwind/include/__libunwind_config.h | 13 +- lib/libunwind/include/libunwind.h | 128 ++++++++++- lib/libunwind/include/unwind_arm_ehabi.h | 5 +- lib/libunwind/src/AddressSpace.hpp | 59 +++-- lib/libunwind/src/CompactUnwinder.hpp | 21 +- lib/libunwind/src/DwarfInstructions.hpp | 37 ++-- lib/libunwind/src/DwarfParser.hpp | 82 +++++-- lib/libunwind/src/EHHeaderParser.hpp | 11 +- lib/libunwind/src/Registers.hpp | 239 +++++++++++++++++++-- lib/libunwind/src/Unwind-wasm.c | 14 +- lib/libunwind/src/UnwindCursor.hpp | 216 ++++++++++++++----- lib/libunwind/src/UnwindLevel1.c | 63 ++++-- lib/libunwind/src/UnwindRegistersRestore.S | 51 ++++- lib/libunwind/src/UnwindRegistersSave.S | 88 +++++++- lib/libunwind/src/assembly.h | 6 +- lib/libunwind/src/config.h | 9 + lib/libunwind/src/gcc_personality_v0.c | 85 +++++++- lib/libunwind/src/libunwind.cpp | 108 +++++++++- lib/libunwind/src/libunwind_ext.h | 8 +- lib/libunwind/src/shadow_stack_unwind.h | 4 +- 20 files changed, 1054 insertions(+), 193 deletions(-) diff --git a/lib/libunwind/include/__libunwind_config.h b/lib/libunwind/include/__libunwind_config.h index bb7fe4c83a..980d11ef5d 100644 --- a/lib/libunwind/include/__libunwind_config.h +++ b/lib/libunwind/include/__libunwind_config.h @@ -73,11 +73,11 @@ # define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC # elif defined(__aarch64__) # define _LIBUNWIND_TARGET_AARCH64 1 -# define _LIBUNWIND_CONTEXT_SIZE 66 +#define _LIBUNWIND_CONTEXT_SIZE 67 # if defined(__SEH__) # define _LIBUNWIND_CURSOR_SIZE 164 # else -# define _LIBUNWIND_CURSOR_SIZE 78 +#define _LIBUNWIND_CURSOR_SIZE 79 # endif # define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64 # elif defined(__arm__) @@ -212,4 +212,13 @@ # define _LIBUNWIND_HIGHEST_DWARF_REGISTER 287 #endif // _LIBUNWIND_IS_NATIVE_ONLY +#if defined(__has_feature) +# if __has_feature(ptrauth_calls) && __has_feature(ptrauth_returns) +# define _LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING 1 +# elif __has_feature(ptrauth_calls) != __has_feature(ptrauth_returns) +# error "Either both or none of ptrauth_calls and ptrauth_returns "\ + "is allowed to be enabled" +# endif +#endif + #endif // ____LIBUNWIND_CONFIG_H__ diff --git a/lib/libunwind/include/libunwind.h b/lib/libunwind/include/libunwind.h index b2dae8feed..56ca711027 100644 --- a/lib/libunwind/include/libunwind.h +++ b/lib/libunwind/include/libunwind.h @@ -43,6 +43,109 @@ #define LIBUNWIND_AVAIL #endif +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + + #include + + // `__ptrauth_restricted_intptr` is a feature of apple clang that predates + // support for direct application of `__ptrauth` to integer types. This + // guard is necessary to support compilation with those compiler. + #if __has_extension(ptrauth_restricted_intptr_qualifier) + #define __unwind_ptrauth_restricted_intptr(...) \ + __ptrauth_restricted_intptr(__VA_ARGS__) + #else + #define __unwind_ptrauth_restricted_intptr(...) \ + __ptrauth(__VA_ARGS__) + #endif + + // ptrauth_string_discriminator("unw_proc_info_t::handler") == 0x7405 + #define __ptrauth_unwind_upi_handler_disc 0x7405 + + #define __ptrauth_unwind_upi_handler \ + __ptrauth(ptrauth_key_function_pointer, 1, __ptrauth_unwind_upi_handler_disc) + + #define __ptrauth_unwind_upi_handler_intptr \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_function_pointer, 1,\ + __ptrauth_unwind_upi_handler_disc) + + // ptrauth_string_discriminator("unw_proc_info_t::start_ip") == 0xCA2C + #define __ptrauth_unwind_upi_startip \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_independent_code, 1, 0xCA2C) + + // ptrauth_string_discriminator("unw_proc_info_t::end_ip") == 0xE183 + #define __ptrauth_unwind_upi_endip \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_independent_code, 1, 0xE183) + + // ptrauth_string_discriminator("unw_proc_info_t::lsda") == 0x83DE + #define __ptrauth_unwind_upi_lsda \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x83DE) + + // ptrauth_string_discriminator("unw_proc_info_t::flags") == 0x79A1 + #define __ptrauth_unwind_upi_flags \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x79A1) + + // ptrauth_string_discriminator("unw_proc_info_t::unwind_info") == 0xC20C + #define __ptrauth_unwind_upi_info \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0xC20C) + + // ptrauth_string_discriminator("unw_proc_info_t::extra") == 0x03DF + #define __ptrauth_unwind_upi_extra \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x03DF) + + // ptrauth_string_discriminator("Registers_arm64::link_reg_t") == 0x8301 + #define __ptrauth_unwind_registers_arm64_link_reg \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_code, 1, 0x8301) + + // ptrauth_string_discriminator("UnwindInfoSections::dso_base") == 0x4FF5 + #define __ptrauth_unwind_uis_dso_base \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x4FF5) + + // ptrauth_string_discriminator("UnwindInfoSections::dwarf_section") == 0x4974 + #define __ptrauth_unwind_uis_dwarf_section \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x4974) + + // ptrauth_string_discriminator("UnwindInfoSections::dwarf_section_length") == 0x2A9A + #define __ptrauth_unwind_uis_dwarf_section_length \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x2A9A) + + // ptrauth_string_discriminator("UnwindInfoSections::compact_unwind_section") == 0xA27B + #define __ptrauth_unwind_uis_compact_unwind_section \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0xA27B) + + // ptrauth_string_discriminator("UnwindInfoSections::compact_unwind_section_length") == 0x5D0A + #define __ptrauth_unwind_uis_compact_unwind_section_length \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_process_dependent_data, 1, 0x5D0A) + + // ptrauth_string_discriminator("CIE_Info::personality") == 0x6A40 + #define __ptrauth_unwind_cie_info_personality_disc 0x6A40 + #define __ptrauth_unwind_cie_info_personality \ + __unwind_ptrauth_restricted_intptr(ptrauth_key_function_pointer, 1, \ + __ptrauth_unwind_cie_info_personality_disc) + + // ptrauth_string_discriminator("personality") == 0x7EAD) + #define __ptrauth_unwind_pauthtest_personality_disc 0x7EAD + +#else + + #define __unwind_ptrauth_restricted_intptr(...) + #define __ptrauth_unwind_upi_handler + #define __ptrauth_unwind_upi_handler_intptr + #define __ptrauth_unwind_upi_startip + #define __ptrauth_unwind_upi_endip + #define __ptrauth_unwind_upi_lsda + #define __ptrauth_unwind_upi_flags + #define __ptrauth_unwind_upi_info + #define __ptrauth_unwind_upi_extra + #define __ptrauth_unwind_registers_arm64_link_reg + #define __ptrauth_unwind_uis_dso_base + #define __ptrauth_unwind_uis_dwarf_section + #define __ptrauth_unwind_uis_dwarf_section_length + #define __ptrauth_unwind_uis_compact_unwind_section + #define __ptrauth_unwind_uis_compact_unwind_section_length + #define __ptrauth_unwind_cie_info_personality + +#endif + #if defined(_WIN32) && defined(__SEH__) #define LIBUNWIND_CURSOR_ALIGNMENT_ATTR __attribute__((__aligned__(16))) #else @@ -88,17 +191,18 @@ typedef double unw_fpreg_t; #endif struct unw_proc_info_t { - unw_word_t start_ip; /* start address of function */ - unw_word_t end_ip; /* address after end of function */ - unw_word_t lsda; /* address of language specific data area, */ - /* or zero if not used */ - unw_word_t handler; /* personality routine, or zero if not used */ - unw_word_t gp; /* not used */ - unw_word_t flags; /* not used */ - uint32_t format; /* compact unwind encoding, or zero if none */ - uint32_t unwind_info_size; /* size of DWARF unwind info, or zero if none */ - unw_word_t unwind_info; /* address of DWARF unwind info, or zero */ - unw_word_t extra; /* mach_header of mach-o image containing func */ + unw_word_t __ptrauth_unwind_upi_startip start_ip; /* start address of function */ + unw_word_t __ptrauth_unwind_upi_endip end_ip; /* address after end of function */ + unw_word_t __ptrauth_unwind_upi_lsda lsda; /* address of language specific data area, */ + /* or zero if not used */ + + unw_word_t __ptrauth_unwind_upi_handler_intptr handler; + unw_word_t gp; /* not used */ + unw_word_t __ptrauth_unwind_upi_flags flags; /* not used */ + uint32_t format; /* compact unwind encoding, or zero if none */ + uint32_t unwind_info_size; /* size of DWARF unwind info, or zero if none */ + unw_word_t __ptrauth_unwind_upi_info unwind_info; /* address of DWARF unwind info, or zero */ + unw_word_t __ptrauth_unwind_upi_extra extra; /* mach_header of mach-o image containing func */ }; typedef struct unw_proc_info_t unw_proc_info_t; @@ -130,6 +234,7 @@ extern int unw_is_fpreg(unw_cursor_t *, unw_regnum_t) LIBUNWIND_AVAIL; extern int unw_is_signal_frame(unw_cursor_t *) LIBUNWIND_AVAIL; extern int unw_get_proc_name(unw_cursor_t *, char *, size_t, unw_word_t *) LIBUNWIND_AVAIL; //extern int unw_get_save_loc(unw_cursor_t*, int, unw_save_loc_t*); +extern const char *unw_strerror(int) LIBUNWIND_AVAIL; extern unw_addr_space_t unw_local_addr_space; @@ -532,6 +637,7 @@ enum { UNW_AARCH64_X31 = 31, UNW_AARCH64_SP = 31, UNW_AARCH64_PC = 32, + UNW_AARCH64_VG = 46, // reserved block UNW_AARCH64_RA_SIGN_STATE = 34, diff --git a/lib/libunwind/include/unwind_arm_ehabi.h b/lib/libunwind/include/unwind_arm_ehabi.h index 6277a1457f..68e02e4760 100644 --- a/lib/libunwind/include/unwind_arm_ehabi.h +++ b/lib/libunwind/include/unwind_arm_ehabi.h @@ -125,8 +125,11 @@ _Unwind_VRS_Pop(_Unwind_Context *context, _Unwind_VRS_RegClass regclass, uint32_t discriminator, _Unwind_VRS_DataRepresentation representation); +extern _Unwind_Reason_Code __gnu_unwind_frame(_Unwind_Exception *, + _Unwind_Context *); + #if defined(_LIBUNWIND_UNWIND_LEVEL1_EXTERNAL_LINKAGE) -#define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 extern +#define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 extern __inline__ #else #define _LIBUNWIND_EXPORT_UNWIND_LEVEL1 static __inline__ #endif diff --git a/lib/libunwind/src/AddressSpace.hpp b/lib/libunwind/src/AddressSpace.hpp index 5551c7d4be..52477b16b3 100644 --- a/lib/libunwind/src/AddressSpace.hpp +++ b/lib/libunwind/src/AddressSpace.hpp @@ -129,22 +129,27 @@ struct UnwindInfoSections { defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) || \ defined(_LIBUNWIND_USE_DL_ITERATE_PHDR) // No dso_base for SEH. - uintptr_t dso_base; + uintptr_t __ptrauth_unwind_uis_dso_base + dso_base = 0; #endif #if defined(_LIBUNWIND_USE_DL_ITERATE_PHDR) size_t text_segment_length; #endif #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) - uintptr_t dwarf_section; - size_t dwarf_section_length; + uintptr_t __ptrauth_unwind_uis_dwarf_section + dwarf_section = 0; + size_t __ptrauth_unwind_uis_dwarf_section_length + dwarf_section_length = 0; #endif #if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) uintptr_t dwarf_index_section; size_t dwarf_index_section_length; #endif #if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) - uintptr_t compact_unwind_section; - size_t compact_unwind_section_length; + uintptr_t __ptrauth_unwind_uis_compact_unwind_section + compact_unwind_section = 0; + size_t __ptrauth_unwind_uis_compact_unwind_section_length + compact_unwind_section_length = 0; #endif #if defined(_LIBUNWIND_ARM_EHABI) uintptr_t arm_section; @@ -196,11 +201,16 @@ public: static int64_t getSLEB128(pint_t &addr, pint_t end); pint_t getEncodedP(pint_t &addr, pint_t end, uint8_t encoding, - pint_t datarelBase = 0); - bool findFunctionName(pint_t addr, char *buf, size_t bufLen, - unw_word_t *offset); - bool findUnwindSections(pint_t targetAddr, UnwindInfoSections &info); - bool findOtherFDE(pint_t targetAddr, pint_t &fde); + pint_t datarelBase = 0, pint_t *resultAddr = nullptr); + template + bool findFunctionName(typename R::link_hardened_reg_arg_t addr, char *buf, + size_t bufLen, unw_word_t *offset); + template + bool findUnwindSections(typename R::link_hardened_reg_arg_t targetAddr, + UnwindInfoSections &info); + template + bool findOtherFDE(typename R::link_hardened_reg_arg_t targetAddr, + pint_t &fde); static LocalAddressSpace sThisAddressSpace; }; @@ -269,7 +279,7 @@ inline int64_t LocalAddressSpace::getSLEB128(pint_t &addr, pint_t end) { inline LocalAddressSpace::pint_t LocalAddressSpace::getEncodedP(pint_t &addr, pint_t end, uint8_t encoding, - pint_t datarelBase) { + pint_t datarelBase, pint_t *resultAddr) { pint_t startAddr = addr; const uint8_t *p = (uint8_t *)addr; pint_t result; @@ -353,8 +363,14 @@ LocalAddressSpace::getEncodedP(pint_t &addr, pint_t end, uint8_t encoding, break; } - if (encoding & DW_EH_PE_indirect) + if (encoding & DW_EH_PE_indirect) { + if (resultAddr) + *resultAddr = result; result = getP(result); + } else { + if (resultAddr) + *resultAddr = startAddr; + } return result; } @@ -486,9 +502,9 @@ static int findUnwindSectionsByPhdr(struct dl_phdr_info *pinfo, #endif // defined(_LIBUNWIND_USE_DL_ITERATE_PHDR) - -inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr, - UnwindInfoSections &info) { +template +inline bool LocalAddressSpace::findUnwindSections( + typename R::link_hardened_reg_arg_t targetAddr, UnwindInfoSections &info) { #ifdef __APPLE__ dyld_unwind_sections dyldInfo; if (_dyld_find_unwind_sections((void *)targetAddr, &dyldInfo)) { @@ -658,16 +674,21 @@ inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr, return false; } -inline bool LocalAddressSpace::findOtherFDE(pint_t targetAddr, pint_t &fde) { +template +inline bool +LocalAddressSpace::findOtherFDE(typename R::link_hardened_reg_arg_t targetAddr, + pint_t &fde) { // TO DO: if OS has way to dynamically register FDEs, check that. (void)targetAddr; (void)fde; return false; } -inline bool LocalAddressSpace::findFunctionName(pint_t addr, char *buf, - size_t bufLen, - unw_word_t *offset) { +template +inline bool +LocalAddressSpace::findFunctionName(typename R::link_hardened_reg_arg_t addr, + char *buf, size_t bufLen, + unw_word_t *offset) { #if _LIBUNWIND_USE_DLADDR Dl_info dyldInfo; if (dladdr((void *)addr, &dyldInfo)) { diff --git a/lib/libunwind/src/CompactUnwinder.hpp b/lib/libunwind/src/CompactUnwinder.hpp index a7a8a153d8..cd2e0e3431 100644 --- a/lib/libunwind/src/CompactUnwinder.hpp +++ b/lib/libunwind/src/CompactUnwinder.hpp @@ -601,11 +601,17 @@ int CompactUnwinder_arm64::stepWithCompactEncodingFrameless( savedRegisterLoc -= 8; } + // We load the link register prior to setting the new SP as the authentication + // schema for LR entangles the SP of the old frame into the diversifier. + Registers_arm64::reg_t linkRegister = registers.getRegister(UNW_AARCH64_LR); + // subtract stack size off of sp registers.setSP(savedRegisterLoc); - // set pc to be value in lr - registers.setIP(registers.getRegister(UNW_AARCH64_LR)); + // Set pc to be value in lr. This needs to be performed after the new SP has + // been set, as the PC authentication schema entangles the SP of the new + // frame. + registers.setIP(linkRegister); return UNW_STEP_SUCCESS; } @@ -614,7 +620,7 @@ template int CompactUnwinder_arm64::stepWithCompactEncodingFrame( compact_unwind_encoding_t encoding, uint64_t, A &addressSpace, Registers_arm64 ®isters) { - uint64_t savedRegisterLoc = registers.getFP() - 8; + Registers_arm64::reg_t savedRegisterLoc = registers.getFP() - 8; if (encoding & UNWIND_ARM64_FRAME_X19_X20_PAIR) { registers.setRegister(UNW_AARCH64_X19, addressSpace.get64(savedRegisterLoc)); @@ -680,11 +686,16 @@ int CompactUnwinder_arm64::stepWithCompactEncodingFrame( savedRegisterLoc -= 8; } - uint64_t fp = registers.getFP(); + Registers_arm64::reg_t fp = registers.getFP(); + // fp points to old fp registers.setFP(addressSpace.get64(fp)); - // old sp is fp less saved fp and lr + + // Old sp is fp less saved fp and lr. We need to set this prior to setting + // the lr as the pointer authentication schema for the lr incorporates the + // sp as part of the diversifier. registers.setSP(fp + 16); + // pop return address into pc registers.setIP(addressSpace.get64(fp + 8)); diff --git a/lib/libunwind/src/DwarfInstructions.hpp b/lib/libunwind/src/DwarfInstructions.hpp index e7be0d6d5d..165c4a99e9 100644 --- a/lib/libunwind/src/DwarfInstructions.hpp +++ b/lib/libunwind/src/DwarfInstructions.hpp @@ -22,7 +22,6 @@ #include "dwarf2.h" #include "libunwind_ext.h" - namespace libunwind { @@ -34,8 +33,10 @@ public: typedef typename A::pint_t pint_t; typedef typename A::sint_t sint_t; - static int stepWithDwarf(A &addressSpace, pint_t pc, pint_t fdeStart, - R ®isters, bool &isSignalFrame, bool stage2); + static int stepWithDwarf(A &addressSpace, + typename R::link_hardened_reg_arg_t pc, + pint_t fdeStart, R ®isters, bool &isSignalFrame, + bool stage2); private: @@ -64,9 +65,10 @@ private: static pint_t getCFA(A &addressSpace, const PrologInfo &prolog, const R ®isters) { - if (prolog.cfaRegister != 0) - return (pint_t)((sint_t)registers.getRegister((int)prolog.cfaRegister) + - prolog.cfaRegisterOffset); + if (prolog.cfaRegister != 0) { + uintptr_t cfaRegister = registers.getRegister((int)prolog.cfaRegister); + return (pint_t)(cfaRegister + prolog.cfaRegisterOffset); + } if (prolog.cfaExpression != 0) return evaluateExpression((pint_t)prolog.cfaExpression, addressSpace, registers, 0); @@ -207,16 +209,16 @@ bool DwarfInstructions::isReturnAddressSignedWithPC(A &addressSpace, #endif template -int DwarfInstructions::stepWithDwarf(A &addressSpace, pint_t pc, - pint_t fdeStart, R ®isters, - bool &isSignalFrame, bool stage2) { +int DwarfInstructions::stepWithDwarf( + A &addressSpace, typename R::link_hardened_reg_arg_t pc, pint_t fdeStart, + R ®isters, bool &isSignalFrame, bool stage2) { FDE_Info fdeInfo; CIE_Info cieInfo; if (CFI_Parser::decodeFDE(addressSpace, fdeStart, &fdeInfo, &cieInfo) == NULL) { PrologInfo prolog; - if (CFI_Parser::parseFDEInstructions(addressSpace, fdeInfo, cieInfo, pc, - R::getArch(), &prolog)) { + if (CFI_Parser::template parseFDEInstructions( + addressSpace, fdeInfo, cieInfo, pc, R::getArch(), &prolog)) { // get pointer to cfa (architecture specific) pint_t cfa = getCFA(addressSpace, prolog, registers); @@ -264,7 +266,7 @@ int DwarfInstructions::stepWithDwarf(A &addressSpace, pint_t pc, // by a CFI directive later on. newRegisters.setSP(cfa); - pint_t returnAddress = 0; + typename R::reg_t returnAddress = 0; constexpr int lastReg = R::lastDwarfRegNum(); static_assert(static_cast(CFI_Parser::kMaxRegisterNumber) >= lastReg, @@ -300,7 +302,16 @@ int DwarfInstructions::stepWithDwarf(A &addressSpace, pint_t pc, isSignalFrame = cieInfo.isSignalFrame; -#if defined(_LIBUNWIND_TARGET_AARCH64) +#if defined(_LIBUNWIND_TARGET_AARCH64) && \ + !defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + // There are two ways of return address signing: pac-ret (enabled via + // -mbranch-protection=pac-ret) and ptrauth-returns (enabled as part of + // Apple's arm64e or experimental pauthtest ABI on Linux). The code + // below handles signed RA for pac-ret, while ptrauth-returns uses + // different logic. + // TODO: unify logic for both cases, see + // https://github.com/llvm/llvm-project/issues/160110 + // // If the target is aarch64 then the return address may have been signed // using the v8.3 pointer authentication extensions. The original // return address needs to be authenticated before the return address is diff --git a/lib/libunwind/src/DwarfParser.hpp b/lib/libunwind/src/DwarfParser.hpp index 7e85025dd0..22de49023c 100644 --- a/lib/libunwind/src/DwarfParser.hpp +++ b/lib/libunwind/src/DwarfParser.hpp @@ -23,6 +23,10 @@ #include "config.h" +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) +#include +#endif + namespace libunwind { /// CFI_Parser does basic parsing of a CFI (Call Frame Information) records. @@ -33,6 +37,7 @@ template class CFI_Parser { public: typedef typename A::pint_t pint_t; + typedef pint_t __ptrauth_unwind_cie_info_personality personality_t; /// Information encoded in a CIE (Common Information Entry) struct CIE_Info { @@ -43,7 +48,7 @@ public: uint8_t lsdaEncoding; uint8_t personalityEncoding; uint8_t personalityOffsetInCIE; - pint_t personality; + personality_t personality; uint32_t codeAlignFactor; int dataAlignFactor; bool isSignalFrame; @@ -155,14 +160,17 @@ public: } }; - static bool findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart, - size_t sectionLength, pint_t fdeHint, FDE_Info *fdeInfo, - CIE_Info *cieInfo); + template + static bool findFDE(A &addressSpace, typename R::link_hardened_reg_arg_t pc, + pint_t ehSectionStart, size_t sectionLength, + pint_t fdeHint, FDE_Info *fdeInfo, CIE_Info *cieInfo); static const char *decodeFDE(A &addressSpace, pint_t fdeStart, FDE_Info *fdeInfo, CIE_Info *cieInfo, bool useCIEInfo = false); + template static bool parseFDEInstructions(A &addressSpace, const FDE_Info &fdeInfo, - const CIE_Info &cieInfo, pint_t upToPC, + const CIE_Info &cieInfo, + typename R::link_hardened_reg_arg_t upToPC, int arch, PrologInfo *results); static const char *parseCIE(A &addressSpace, pint_t cie, CIE_Info *cieInfo); @@ -234,9 +242,12 @@ const char *CFI_Parser::decodeFDE(A &addressSpace, pint_t fdeStart, /// Scan an eh_frame section to find an FDE for a pc template -bool CFI_Parser::findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart, - size_t sectionLength, pint_t fdeHint, - FDE_Info *fdeInfo, CIE_Info *cieInfo) { +template +bool CFI_Parser::findFDE(A &addressSpace, + typename R::link_hardened_reg_arg_t pc, + pint_t ehSectionStart, size_t sectionLength, + pint_t fdeHint, FDE_Info *fdeInfo, + CIE_Info *cieInfo) { //fprintf(stderr, "findFDE(0x%llX)\n", (long long)pc); pint_t p = (fdeHint != 0) ? fdeHint : ehSectionStart; const pint_t ehSectionEnd = (sectionLength == SIZE_MAX) @@ -273,7 +284,7 @@ bool CFI_Parser::findFDE(A &addressSpace, pint_t pc, pint_t ehSectionStart, pint_t pcRange = addressSpace.getEncodedP( p, nextCFI, cieInfo->pointerEncoding & 0x0F); // Test if pc is within the function this FDE covers. - if ((pcStart < pc) && (pc <= pcStart + pcRange)) { + if ((pcStart <= pc) && (pc < pcStart + pcRange)) { // parse rest of info fdeInfo->lsda = 0; // check for augmentation length @@ -369,6 +380,7 @@ const char *CFI_Parser::parseCIE(A &addressSpace, pint_t cie, cieInfo->returnAddressRegister = (uint8_t)raReg; // parse augmentation data based on augmentation string const char *result = NULL; + pint_t resultAddr = 0; if (addressSpace.get8(strStart) == 'z') { // parse augmentation data length addressSpace.getULEB128(p, cieContentEnd); @@ -377,13 +389,41 @@ const char *CFI_Parser::parseCIE(A &addressSpace, pint_t cie, case 'z': cieInfo->fdesHaveAugmentationData = true; break; - case 'P': + case 'P': { cieInfo->personalityEncoding = addressSpace.get8(p); ++p; cieInfo->personalityOffsetInCIE = (uint8_t)(p - cie); - cieInfo->personality = addressSpace - .getEncodedP(p, cieContentEnd, cieInfo->personalityEncoding); + pint_t personality = addressSpace.getEncodedP( + p, cieContentEnd, cieInfo->personalityEncoding, + /*datarelBase=*/0, &resultAddr); +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + if (personality) { + // The GOT for the personality function was signed address + // authenticated. Manually re-sign with the CIE_Info::personality + // schema. If we could guarantee the encoding of the personality we + // could avoid this by simply giving resultAddr the correct ptrauth + // schema and performing an assignment. +#if defined(__arm64e__) + const auto oldDiscriminator = resultAddr; +#else + const auto oldDiscriminator = ptrauth_blend_discriminator( + (void *)resultAddr, __ptrauth_unwind_pauthtest_personality_disc); +#endif + const auto discriminator = ptrauth_blend_discriminator( + &cieInfo->personality, + __ptrauth_unwind_cie_info_personality_disc); + void *signedPtr = ptrauth_auth_and_resign( + (void *)personality, ptrauth_key_function_pointer, + oldDiscriminator, ptrauth_key_function_pointer, discriminator); + personality = (pint_t)signedPtr; + } +#endif + // We use memmove to set the CIE personality as we have already + // re-signed the pointer to the correct schema. + memmove((void *)&cieInfo->personality, (void *)&personality, + sizeof(personality)); break; + } case 'L': cieInfo->lsdaEncoding = addressSpace.get8(p); ++p; @@ -417,10 +457,10 @@ const char *CFI_Parser::parseCIE(A &addressSpace, pint_t cie, /// "run" the DWARF instructions and create the abstract PrologInfo for an FDE template -bool CFI_Parser::parseFDEInstructions(A &addressSpace, - const FDE_Info &fdeInfo, - const CIE_Info &cieInfo, pint_t upToPC, - int arch, PrologInfo *results) { +template +bool CFI_Parser::parseFDEInstructions( + A &addressSpace, const FDE_Info &fdeInfo, const CIE_Info &cieInfo, + typename R::link_hardened_reg_arg_t upToPC, int arch, PrologInfo *results) { // Alloca is used for the allocation of the rememberStack entries. It removes // the dependency on new/malloc but the below for loop can not be refactored // into functions. Entry could be saved during the processing of a CIE and @@ -808,12 +848,10 @@ bool CFI_Parser::parseFDEInstructions(A &addressSpace, results->savedRegisters[UNW_AARCH64_RA_SIGN_STATE].value ^ 0x3; results->setRegisterValue(UNW_AARCH64_RA_SIGN_STATE, value, initialState); - // When calculating the value of the PC, it is assumed that the CFI - // instruction is placed before the signing instruction, however it is - // placed after. Because of this, we need to take into account the CFI - // instruction is one instruction call later than expected, and reduce - // the PC value by 4 bytes to compensate. - results->ptrAuthDiversifier = fdeInfo.pcStart + codeOffset - 0x4; + // When using Feat_PAuthLR, the PC value needs to be captured so that + // during unwinding, the correct PC value is used for re-authentication. + // It is assumed that the CFI is placed before the signing instruction. + results->ptrAuthDiversifier = fdeInfo.pcStart + codeOffset; _LIBUNWIND_TRACE_DWARF( "DW_CFA_AARCH64_negate_ra_state_with_pc(pc=0x%" PRIx64 ")\n", static_cast(results->ptrAuthDiversifier)); diff --git a/lib/libunwind/src/EHHeaderParser.hpp b/lib/libunwind/src/EHHeaderParser.hpp index 0662a1321e..b5d927027f 100644 --- a/lib/libunwind/src/EHHeaderParser.hpp +++ b/lib/libunwind/src/EHHeaderParser.hpp @@ -37,8 +37,9 @@ public: static bool decodeEHHdr(A &addressSpace, pint_t ehHdrStart, pint_t ehHdrEnd, EHHeaderInfo &ehHdrInfo); - static bool findFDE(A &addressSpace, pint_t pc, pint_t ehHdrStart, - uint32_t sectionLength, + template + static bool findFDE(A &addressSpace, typename R::link_hardened_reg_arg_t pc, + pint_t ehHdrStart, uint32_t sectionLength, typename CFI_Parser::FDE_Info *fdeInfo, typename CFI_Parser::CIE_Info *cieInfo); @@ -112,8 +113,10 @@ bool EHHeaderParser::decodeTableEntry( } template -bool EHHeaderParser::findFDE(A &addressSpace, pint_t pc, pint_t ehHdrStart, - uint32_t sectionLength, +template +bool EHHeaderParser::findFDE(A &addressSpace, + typename R::link_hardened_reg_arg_t pc, + pint_t ehHdrStart, uint32_t sectionLength, typename CFI_Parser::FDE_Info *fdeInfo, typename CFI_Parser::CIE_Info *cieInfo) { pint_t ehHdrEnd = ehHdrStart + sectionLength; diff --git a/lib/libunwind/src/Registers.hpp b/lib/libunwind/src/Registers.hpp index 2c3bfb7e84..474b17461b 100644 --- a/lib/libunwind/src/Registers.hpp +++ b/lib/libunwind/src/Registers.hpp @@ -17,8 +17,14 @@ #include "config.h" #include "libunwind.h" +#include "libunwind_ext.h" #include "shadow_stack_unwind.h" +#if __has_include() +#include +#define HAVE_SYS_AUXV_H +#endif + namespace libunwind { // For emulating 128-bit registers @@ -60,6 +66,10 @@ public: Registers_x86(); Registers_x86(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -278,6 +288,10 @@ public: Registers_x86_64(); Registers_x86_64(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); @@ -597,6 +611,10 @@ public: Registers_ppc(); Registers_ppc(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -1169,6 +1187,10 @@ public: Registers_ppc64(); Registers_ppc64(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); @@ -1814,7 +1836,9 @@ inline const char *Registers_ppc64::getRegisterName(int regNum) { /// Registers_arm64 holds the register state of a thread in a 64-bit arm /// process. class _LIBUNWIND_HIDDEN Registers_arm64; -extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *); +extern "C" int64_t __libunwind_Registers_arm64_za_disable(); +extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *, + unsigned walkedFrames); #if defined(_LIBUNWIND_USE_GCS) extern "C" void *__libunwind_shstk_get_jump_target() { @@ -1824,8 +1848,21 @@ extern "C" void *__libunwind_shstk_get_jump_target() { class _LIBUNWIND_HIDDEN Registers_arm64 { public: - Registers_arm64(); + Registers_arm64() = default; Registers_arm64(const void *registers); + Registers_arm64(const Registers_arm64 &); + Registers_arm64 &operator=(const Registers_arm64 &); + + typedef uint64_t reg_t; + typedef uint64_t __ptrauth_unwind_registers_arm64_link_reg link_reg_t; + + // Use `link_hardened_reg_arg_t` to pass values of `link_reg_t` type as + // function arguments. We need to use a const l-value reference to keep + // signature of `__ptrauth`-qualified values of `link_reg_t` type on AArch64 + // PAuth-enabled ABI intact. Passing the raw pointer by value would cause + // authentication on the caller side and make the pointer prone to + // substitution if spilled to the stack in the callee. + typedef const link_reg_t &link_hardened_reg_arg_t; bool validRegister(int num) const; uint64_t getRegister(int num) const; @@ -1837,7 +1874,14 @@ public: v128 getVectorRegister(int num) const; void setVectorRegister(int num, v128 value); static const char *getRegisterName(int num); - void jumpto() { __libunwind_Registers_arm64_jumpto(this); } + void jumpto(unsigned walkedFrames = 0) { + zaDisable(); + __libunwind_Registers_arm64_jumpto(this, walkedFrames); + } +#ifdef _LIBUNWIND_TRACE_RET_INJECT + _LIBUNWIND_TRACE_NO_INLINE + void returnto(unsigned walkedFrames) { jumpto(walkedFrames); } +#endif static constexpr int lastDwarfRegNum() { return _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64; } @@ -1845,27 +1889,94 @@ public: uint64_t getSP() const { return _registers.__sp; } void setSP(uint64_t value) { _registers.__sp = value; } - uint64_t getIP() const { return _registers.__pc; } - void setIP(uint64_t value) { _registers.__pc = value; } - uint64_t getFP() const { return _registers.__fp; } - void setFP(uint64_t value) { _registers.__fp = value; } + uint64_t getIP() const { + uint64_t value = _registers.__pc; +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + // Note the value of the PC was signed to its address in the register state + // but everyone else expects it to be sign by the SP, so convert on return. + value = (uint64_t)ptrauth_auth_and_resign((void *)_registers.__pc, + ptrauth_key_return_address, + &_registers.__pc, + ptrauth_key_return_address, + getSP()); +#endif + return value; + } + void setIP(uint64_t value) { +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + // Note the value which was set should have been signed with the SP. + // We then resign with the slot we are being stored in to so that both SP + // and LR can't be spoofed at the same time. + value = (uint64_t)ptrauth_auth_and_resign((void *)value, + ptrauth_key_return_address, + getSP(), + ptrauth_key_return_address, + &_registers.__pc); +#endif + _registers.__pc = value; + } + uint64_t getFP() const { return _registers.__fp; } + void setFP(uint64_t value) { _registers.__fp = value; } + +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + void + loadAndAuthenticateLinkRegister(reg_t inplaceAuthedLinkRegister, + link_reg_t *referenceAuthedLinkRegister) { + // If we are in an arm64/arm64e frame, then the PC should have been signed + // with the SP + *referenceAuthedLinkRegister = + (uint64_t)ptrauth_auth_data((void *)inplaceAuthedLinkRegister, + ptrauth_key_return_address, + _registers.__sp); + } +#endif private: + uint64_t lazyGetVG() const; + + void zaDisable() const { + if (!_misc_registers.__has_sme) + return; + if (__libunwind_Registers_arm64_za_disable() != 0) + _LIBUNWIND_ABORT("SME ZA disable failed"); + } + + static bool checkHasSME() { +#if defined(HAVE_SYS_AUXV_H) + constexpr int hwcap2_sme = (1 << 23); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + return (hwcap2 & hwcap2_sme) != 0; +#endif + // TODO: Support other platforms. + return false; + } + struct GPRs { - uint64_t __x[29]; // x0-x28 - uint64_t __fp; // Frame pointer x29 - uint64_t __lr; // Link register x30 - uint64_t __sp; // Stack pointer x31 - uint64_t __pc; // Program counter - uint64_t __ra_sign_state; // RA sign state register + uint64_t __x[29] = {}; // x0-x28 + uint64_t __fp = 0; // Frame pointer x29 + uint64_t __lr = 0; // Link register x30 + uint64_t __sp = 0; // Stack pointer x31 + uint64_t __pc = 0; // Program counter + uint64_t __ra_sign_state = 0; // RA sign state register }; - GPRs _registers; - double _vectorHalfRegisters[32]; + struct Misc { + mutable uint32_t __vg = 0; // Vector Granule + bool __has_sme = checkHasSME(); + }; + + GPRs _registers = {}; // Currently only the lower double in 128-bit vectore registers // is perserved during unwinding. We could define new register // numbers (> 96) which mean whole vector registers, then this // struct would need to change to contain whole vector registers. + double _vectorHalfRegisters[32] = {}; + + // Miscellaneous/virtual registers. These are stored below the GPRs and FPRs + // as they do not correspond to physical registers, so do not need to be + // saved/restored in UnwindRegistersRestore.S and UnwindRegistersSave.S, and + // we don't want to modify the existing offsets for GPRs and FPRs. + Misc _misc_registers; }; inline Registers_arm64::Registers_arm64(const void *registers) { @@ -1877,11 +1988,31 @@ inline Registers_arm64::Registers_arm64(const void *registers) { memcpy(_vectorHalfRegisters, static_cast(registers) + sizeof(GPRs), sizeof(_vectorHalfRegisters)); + _misc_registers.__vg = 0; + +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + // We have to do some pointer authentication fixups after this copy, + // and as part of that we need to load the source pc without + // authenticating so that we maintain the signature for the resigning + // performed by setIP. + uint64_t pcRegister = 0; + memmove(&pcRegister, ((uint8_t *)&_registers) + offsetof(GPRs, __pc), + sizeof(pcRegister)); + setIP(pcRegister); +#endif } -inline Registers_arm64::Registers_arm64() { - memset(&_registers, 0, sizeof(_registers)); - memset(&_vectorHalfRegisters, 0, sizeof(_vectorHalfRegisters)); +inline Registers_arm64::Registers_arm64(const Registers_arm64 &other) { + *this = other; +} + +inline Registers_arm64 & +Registers_arm64::operator=(const Registers_arm64 &other) { + memmove(static_cast(this), &other, sizeof(*this)); + // We perform this step to ensure that we correctly authenticate and re-sign + // the pc after the bitwise copy. + setIP(other.getIP()); + return *this; } inline bool Registers_arm64::validRegister(int regNum) const { @@ -1895,22 +2026,40 @@ inline bool Registers_arm64::validRegister(int regNum) const { return false; if (regNum == UNW_AARCH64_RA_SIGN_STATE) return true; + if (regNum == UNW_AARCH64_VG) + return true; if ((regNum > 32) && (regNum < 64)) return false; return true; } +inline uint64_t Registers_arm64::lazyGetVG() const { + if (!_misc_registers.__vg) { +#if defined(__aarch64__) + register uint64_t vg asm("x0"); + asm(".inst 0x04e0e3e0" // CNTD x0 + : "=r"(vg)); + _misc_registers.__vg = vg; +#else + _LIBUNWIND_ABORT("arm64 VG undefined"); +#endif + } + return _misc_registers.__vg; +} + inline uint64_t Registers_arm64::getRegister(int regNum) const { if (regNum == UNW_REG_IP || regNum == UNW_AARCH64_PC) - return _registers.__pc; + return getIP(); if (regNum == UNW_REG_SP || regNum == UNW_AARCH64_SP) return _registers.__sp; if (regNum == UNW_AARCH64_RA_SIGN_STATE) return _registers.__ra_sign_state; if (regNum == UNW_AARCH64_FP) - return _registers.__fp; + return getFP(); if (regNum == UNW_AARCH64_LR) return _registers.__lr; + if (regNum == UNW_AARCH64_VG) + return lazyGetVG(); if ((regNum >= 0) && (regNum < 29)) return _registers.__x[regNum]; _LIBUNWIND_ABORT("unsupported arm64 register"); @@ -1918,15 +2067,17 @@ inline uint64_t Registers_arm64::getRegister(int regNum) const { inline void Registers_arm64::setRegister(int regNum, uint64_t value) { if (regNum == UNW_REG_IP || regNum == UNW_AARCH64_PC) - _registers.__pc = value; + setIP(value); else if (regNum == UNW_REG_SP || regNum == UNW_AARCH64_SP) _registers.__sp = value; else if (regNum == UNW_AARCH64_RA_SIGN_STATE) _registers.__ra_sign_state = value; else if (regNum == UNW_AARCH64_FP) - _registers.__fp = value; + setFP(value); else if (regNum == UNW_AARCH64_LR) _registers.__lr = value; + else if (regNum == UNW_AARCH64_VG) + _misc_registers.__vg = value; else if ((regNum >= 0) && (regNum < 29)) _registers.__x[regNum] = value; else @@ -2116,6 +2267,10 @@ public: Registers_arm(); Registers_arm(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -2621,6 +2776,10 @@ public: Registers_or1k(); Registers_or1k(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -2820,6 +2979,10 @@ public: Registers_mips_o32(); Registers_mips_o32(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -3155,6 +3318,10 @@ public: Registers_mips_newabi(); Registers_mips_newabi(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); @@ -3458,6 +3625,10 @@ public: Registers_sparc(); Registers_sparc(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -3644,6 +3815,10 @@ public: Registers_sparc64() = default; Registers_sparc64(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); @@ -3829,6 +4004,10 @@ public: Registers_hexagon(); Registers_hexagon(const void *registers); + typedef uint32_t reg_t; + typedef uint32_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint32_t getRegister(int num) const; void setRegister(int num, uint32_t value); @@ -4044,6 +4223,10 @@ public: Registers_riscv(); Registers_riscv(const void *registers); + typedef ::libunwind::reg_t reg_t; + typedef ::libunwind::reg_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; reg_t getRegister(int num) const; void setRegister(int num, reg_t value); @@ -4341,6 +4524,10 @@ public: Registers_ve(); Registers_ve(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); @@ -4784,6 +4971,10 @@ public: Registers_s390x(); Registers_s390x(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); @@ -5072,6 +5263,10 @@ public: Registers_loongarch(); Registers_loongarch(const void *registers); + typedef uint64_t reg_t; + typedef uint64_t link_reg_t; + typedef const link_reg_t &link_hardened_reg_arg_t; + bool validRegister(int num) const; uint64_t getRegister(int num) const; void setRegister(int num, uint64_t value); diff --git a/lib/libunwind/src/Unwind-wasm.c b/lib/libunwind/src/Unwind-wasm.c index b8b7bc2779..b0d6cd2d00 100644 --- a/lib/libunwind/src/Unwind-wasm.c +++ b/lib/libunwind/src/Unwind-wasm.c @@ -37,13 +37,13 @@ struct _Unwind_LandingPadContext { // function thread_local struct _Unwind_LandingPadContext __wasm_lpad_context; -/// Calls to this function is in landing pads in compiler-generated user code. +/// Calls to this function are in landing pads in compiler-generated user code. /// In other EH schemes, stack unwinding is done by libunwind library, which -/// calls the personality function for each each frame it lands. On the other -/// hand, WebAssembly stack unwinding process is performed by a VM, and the -/// personality function cannot be called from there. So the compiler inserts -/// a call to this function in landing pads in the user code, which in turn -/// calls the personality function. +/// calls the personality function for each frame it lands. On the other hand, +/// WebAssembly stack unwinding process is performed by a VM, and the +/// personality function cannot be called from there. So the compiler inserts a +/// call to this function in landing pads in the user code, which in turn calls +/// the personality function. _Unwind_Reason_Code _Unwind_CallPersonality(void *exception_ptr) { struct _Unwind_Exception *exception_object = (struct _Unwind_Exception *)exception_ptr; @@ -92,7 +92,7 @@ _LIBUNWIND_EXPORT void _Unwind_SetGR(struct _Unwind_Context *context, int index, /// Called by personality handler to get instruction pointer. _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) { - // The result will be used as an 1-based index after decrementing 1, so we + // The result will be used as a 1-based index after decrementing 1, so we // increment 2 here uintptr_t result = ((struct _Unwind_LandingPadContext *)context)->lpad_index + 2; diff --git a/lib/libunwind/src/UnwindCursor.hpp b/lib/libunwind/src/UnwindCursor.hpp index 55db035e62..5838dbcaa9 100644 --- a/lib/libunwind/src/UnwindCursor.hpp +++ b/lib/libunwind/src/UnwindCursor.hpp @@ -41,7 +41,8 @@ #define _LIBUNWIND_CHECK_LINUX_SIGRETURN 1 #endif -#if defined(_LIBUNWIND_TARGET_HAIKU) && defined(_LIBUNWIND_TARGET_X86_64) +#if defined(_LIBUNWIND_TARGET_HAIKU) && \ + (defined(_LIBUNWIND_TARGET_I386) || defined(_LIBUNWIND_TARGET_X86_64)) #include #include #define _LIBUNWIND_CHECK_HAIKU_SIGRETURN 1 @@ -120,7 +121,9 @@ class _LIBUNWIND_HIDDEN DwarfFDECache { typedef typename A::pint_t pint_t; public: static constexpr pint_t kSearchAll = static_cast(-1); - static pint_t findFDE(pint_t mh, pint_t pc); + template + static pint_t findFDE(pint_t mh, typename R::link_hardened_reg_arg_t pc); + static void add(pint_t mh, pint_t ip_start, pint_t ip_end, pint_t fde); static void removeAllIn(pint_t mh); static void iterateCacheEntries(void (*func)(unw_word_t ip_start, @@ -173,7 +176,9 @@ bool DwarfFDECache::_registeredForDyldUnloads = false; #endif template -typename A::pint_t DwarfFDECache::findFDE(pint_t mh, pint_t pc) { +template +typename DwarfFDECache::pint_t +DwarfFDECache::findFDE(pint_t mh, typename R::link_hardened_reg_arg_t pc) { pint_t result = 0; _LIBUNWIND_LOG_IF_FALSE(_lock.lock_shared()); for (entry *p = _buffer; p < _bufferUsed; ++p) { @@ -471,7 +476,9 @@ public: virtual void getInfo(unw_proc_info_t *) { _LIBUNWIND_ABORT("getInfo not implemented"); } - virtual void jumpto() { _LIBUNWIND_ABORT("jumpto not implemented"); } + _LIBUNWIND_TRACE_NO_INLINE virtual void jumpto() { + _LIBUNWIND_ABORT("jumpto not implemented"); + } virtual bool isSignalFrame() { _LIBUNWIND_ABORT("isSignalFrame not implemented"); } @@ -488,6 +495,12 @@ public: virtual void saveVFPAsX() { _LIBUNWIND_ABORT("saveVFPAsX not implemented"); } #endif +#ifdef _LIBUNWIND_TRACE_RET_INJECT + virtual void setWalkedFrames(unsigned) { + _LIBUNWIND_ABORT("setWalkedFrames not implemented"); + } +#endif + #ifdef _AIX virtual uintptr_t getDataRelBase() { _LIBUNWIND_ABORT("getDataRelBase not implemented"); @@ -964,7 +977,8 @@ public: virtual void setFloatReg(int, unw_fpreg_t); virtual int step(bool stage2 = false); virtual void getInfo(unw_proc_info_t *); - virtual void jumpto(); + _LIBUNWIND_TRACE_NO_INLINE + virtual void jumpto(); virtual bool isSignalFrame(); virtual bool getFunctionName(char *buf, size_t len, unw_word_t *off); virtual void setInfoBasedOnIPRegister(bool isReturnAddress = false); @@ -973,6 +987,10 @@ public: virtual void saveVFPAsX(); #endif +#ifdef _LIBUNWIND_TRACE_RET_INJECT + virtual void setWalkedFrames(unsigned); +#endif + #ifdef _AIX virtual uintptr_t getDataRelBase(); #endif @@ -1045,19 +1063,28 @@ private: #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) bool getInfoFromFdeCie(const typename CFI_Parser::FDE_Info &fdeInfo, const typename CFI_Parser::CIE_Info &cieInfo, - pint_t pc, uintptr_t dso_base); - bool getInfoFromDwarfSection(pint_t pc, const UnwindInfoSections §s, - uint32_t fdeSectionOffsetHint=0); + typename R::link_hardened_reg_arg_t pc, + uintptr_t dso_base); + bool getInfoFromDwarfSection(typename R::link_hardened_reg_arg_t pc, + const UnwindInfoSections §s, + uint32_t fdeSectionOffsetHint = 0); int stepWithDwarfFDE(bool stage2) { +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + typename R::reg_t rawPC = this->getReg(UNW_REG_IP); + typename R::link_reg_t pc; + _registers.loadAndAuthenticateLinkRegister(rawPC, &pc); +#else + typename R::link_reg_t pc = this->getReg(UNW_REG_IP); +#endif return DwarfInstructions::stepWithDwarf( - _addressSpace, (pint_t)this->getReg(UNW_REG_IP), - (pint_t)_info.unwind_info, _registers, _isSignalFrame, stage2); + _addressSpace, pc, (pint_t)_info.unwind_info, _registers, + _isSignalFrame, stage2); } #endif #if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) - bool getInfoFromCompactEncodingSection(pint_t pc, - const UnwindInfoSections §s); + bool getInfoFromCompactEncodingSection(typename R::link_hardened_reg_arg_t pc, + const UnwindInfoSections §s); int stepWithCompactEncoding(bool stage2 = false) { #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) if ( compactSaysUseDwarf() ) @@ -1344,9 +1371,12 @@ private: bool _unwindInfoMissing; bool _isSignalFrame; #if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) || \ - defined(_LIBUNWIND_TARGET_HAIKU) + defined(_LIBUNWIND_CHECK_HAIKU_SIGRETURN) bool _isSigReturn = false; #endif +#ifdef _LIBUNWIND_TRACE_RET_INJECT + uint32_t _walkedFrames; +#endif }; @@ -1358,13 +1388,13 @@ UnwindCursor::UnwindCursor(unw_context_t *context, A &as) "UnwindCursor<> does not fit in unw_cursor_t"); static_assert((alignof(UnwindCursor) <= alignof(unw_cursor_t)), "UnwindCursor<> requires more alignment than unw_cursor_t"); - memset(&_info, 0, sizeof(_info)); + memset(static_cast(&_info), 0, sizeof(_info)); } template UnwindCursor::UnwindCursor(A &as, void *) : _addressSpace(as), _unwindInfoMissing(false), _isSignalFrame(false) { - memset(&_info, 0, sizeof(_info)); + memset(static_cast(&_info), 0, sizeof(_info)); // FIXME // fill in _registers from thread arg } @@ -1401,7 +1431,46 @@ void UnwindCursor::setFloatReg(int regNum, unw_fpreg_t value) { } template void UnwindCursor::jumpto() { +#ifdef _LIBUNWIND_TRACE_RET_INJECT + /* + + The value of `_walkedFrames` is computed in `unwind_phase2` and represents the + number of frames walked starting `unwind_phase2` to get to the landing pad. + + ``` + // uc is initialized by __unw_getcontext in the parent frame. + // The first stack frame walked is unwind_phase2. + unsigned framesWalked = 1; + ``` + + To that, we need to add the number of function calls in libunwind between + `unwind_phase2` & `__libunwind_Registers_arm64_jumpto` which performs the long + jump, to rebalance the execution flow. + + ``` + frame #0: libunwind.1.dylib`__libunwind_Registers_arm64_jumpto at UnwindRegistersRestore.S:646 + frame #1: libunwind.1.dylib`libunwind::Registers_arm64::returnto at Registers.hpp:2291:3 + frame #2: libunwind.1.dylib`libunwind::UnwindCursor::jumpto at UnwindCursor.hpp:1474:14 + frame #3: libunwind.1.dylib`__unw_resume at libunwind.cpp:375:7 + frame #4: libunwind.1.dylib`__unw_resume_with_frames_walked at libunwind.cpp:363:10 + frame #5: libunwind.1.dylib`unwind_phase2 at UnwindLevel1.c:328:9 + frame #6: libunwind.1.dylib`_Unwind_RaiseException at UnwindLevel1.c:480:10 + frame #7: libc++abi.dylib`__cxa_throw at cxa_exception.cpp:295:5 + ... + ``` + + If we look at the backtrace from `__libunwind_Registers_arm64_jumpto`, we see + there are 5 frames on the stack to reach `unwind_phase2`. However, only 4 of + them will never return, since `__libunwind_Registers_arm64_jumpto` returns + back to the landing pad, so we need to subtract 1 to the number of + `_EXTRA_LIBUNWIND_FRAMES_WALKED`. + */ + + static constexpr size_t _EXTRA_LIBUNWIND_FRAMES_WALKED = 5 - 1; + _registers.returnto(_walkedFrames + _EXTRA_LIBUNWIND_FRAMES_WALKED); +#else _registers.jumpto(); +#endif } #ifdef __arm__ @@ -1410,6 +1479,13 @@ template void UnwindCursor::saveVFPAsX() { } #endif +#ifdef _LIBUNWIND_TRACE_RET_INJECT +template +void UnwindCursor::setWalkedFrames(unsigned walkedFrames) { + _walkedFrames = walkedFrames; +} +#endif + #ifdef _AIX template uintptr_t UnwindCursor::getDataRelBase() { @@ -1658,11 +1734,11 @@ bool UnwindCursor::getInfoFromEHABISection( template bool UnwindCursor::getInfoFromFdeCie( const typename CFI_Parser::FDE_Info &fdeInfo, - const typename CFI_Parser::CIE_Info &cieInfo, pint_t pc, - uintptr_t dso_base) { + const typename CFI_Parser::CIE_Info &cieInfo, + typename R::link_hardened_reg_arg_t pc, uintptr_t dso_base) { typename CFI_Parser::PrologInfo prolog; - if (CFI_Parser::parseFDEInstructions(_addressSpace, fdeInfo, cieInfo, pc, - R::getArch(), &prolog)) { + if (CFI_Parser::template parseFDEInstructions( + _addressSpace, fdeInfo, cieInfo, pc, R::getArch(), &prolog)) { // Save off parsed FDE info _info.start_ip = fdeInfo.pcStart; _info.end_ip = fdeInfo.pcEnd; @@ -1682,43 +1758,42 @@ bool UnwindCursor::getInfoFromFdeCie( } template -bool UnwindCursor::getInfoFromDwarfSection(pint_t pc, - const UnwindInfoSections §s, - uint32_t fdeSectionOffsetHint) { +bool UnwindCursor::getInfoFromDwarfSection( + typename R::link_hardened_reg_arg_t pc, const UnwindInfoSections §s, + uint32_t fdeSectionOffsetHint) { typename CFI_Parser::FDE_Info fdeInfo; typename CFI_Parser::CIE_Info cieInfo; bool foundFDE = false; bool foundInCache = false; // If compact encoding table gave offset into dwarf section, go directly there if (fdeSectionOffsetHint != 0) { - foundFDE = CFI_Parser::findFDE(_addressSpace, pc, sects.dwarf_section, - sects.dwarf_section_length, - sects.dwarf_section + fdeSectionOffsetHint, - &fdeInfo, &cieInfo); + foundFDE = CFI_Parser::template findFDE( + _addressSpace, pc, sects.dwarf_section, sects.dwarf_section_length, + sects.dwarf_section + fdeSectionOffsetHint, &fdeInfo, &cieInfo); } #if defined(_LIBUNWIND_SUPPORT_DWARF_INDEX) if (!foundFDE && (sects.dwarf_index_section != 0)) { - foundFDE = EHHeaderParser::findFDE( + foundFDE = EHHeaderParser::template findFDE( _addressSpace, pc, sects.dwarf_index_section, (uint32_t)sects.dwarf_index_section_length, &fdeInfo, &cieInfo); } #endif if (!foundFDE) { // otherwise, search cache of previously found FDEs. - pint_t cachedFDE = DwarfFDECache::findFDE(sects.dso_base, pc); + pint_t cachedFDE = + DwarfFDECache::template findFDE(sects.dso_base, pc); if (cachedFDE != 0) { - foundFDE = - CFI_Parser::findFDE(_addressSpace, pc, sects.dwarf_section, - sects.dwarf_section_length, - cachedFDE, &fdeInfo, &cieInfo); + foundFDE = CFI_Parser::template findFDE( + _addressSpace, pc, sects.dwarf_section, sects.dwarf_section_length, + cachedFDE, &fdeInfo, &cieInfo); foundInCache = foundFDE; } } if (!foundFDE) { // Still not found, do full scan of __eh_frame section. - foundFDE = CFI_Parser::findFDE(_addressSpace, pc, sects.dwarf_section, - sects.dwarf_section_length, 0, - &fdeInfo, &cieInfo); + foundFDE = CFI_Parser::template findFDE( + _addressSpace, pc, sects.dwarf_section, sects.dwarf_section_length, 0, + &fdeInfo, &cieInfo); } if (foundFDE) { if (getInfoFromFdeCie(fdeInfo, cieInfo, pc, sects.dso_base)) { @@ -1742,8 +1817,8 @@ bool UnwindCursor::getInfoFromDwarfSection(pint_t pc, #if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) template -bool UnwindCursor::getInfoFromCompactEncodingSection(pint_t pc, - const UnwindInfoSections §s) { +bool UnwindCursor::getInfoFromCompactEncodingSection( + typename R::link_hardened_reg_arg_t pc, const UnwindInfoSections §s) { const bool log = false; if (log) fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX, mh=0x%llX)\n", @@ -1974,6 +2049,16 @@ bool UnwindCursor::getInfoFromCompactEncodingSection(pint_t pc, personalityIndex * sizeof(uint32_t)); pint_t personalityPointer = sects.dso_base + (pint_t)personalityDelta; personality = _addressSpace.getP(personalityPointer); +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + // The GOT for the personality function was signed address authenticated. + // Resign it as a regular function pointer. + const auto discriminator = ptrauth_blend_discriminator( + &_info.handler, __ptrauth_unwind_upi_handler_disc); + void *signedPtr = ptrauth_auth_and_resign( + (void *)personality, ptrauth_key_function_pointer, personalityPointer, + ptrauth_key_function_pointer, discriminator); + personality = (__typeof(personality))signedPtr; +#endif if (log) fprintf(stderr, "getInfoFromCompactEncodingSection(pc=0x%llX), " "personalityDelta=0x%08X, personality=0x%08llX\n", @@ -1987,7 +2072,11 @@ bool UnwindCursor::getInfoFromCompactEncodingSection(pint_t pc, _info.start_ip = funcStart; _info.end_ip = funcEnd; _info.lsda = lsda; - _info.handler = personality; + // We use memmove to copy the personality function as we have already manually + // re-signed the pointer, and assigning directly will attempt to incorrectly + // sign the already signed value. + memmove(reinterpret_cast(&_info.handler), + reinterpret_cast(&personality), sizeof(personality)); _info.gp = 0; _info.flags = 0; _info.format = encoding; @@ -2640,11 +2729,19 @@ void UnwindCursor::setInfoBasedOnIPRegister(bool isReturnAddress) { _isSigReturn = false; #endif - pint_t pc = static_cast(this->getReg(UNW_REG_IP)); + typename R::reg_t rawPC = this->getReg(UNW_REG_IP); + #if defined(_LIBUNWIND_ARM_EHABI) // Remove the thumb bit so the IP represents the actual instruction address. // This matches the behaviour of _Unwind_GetIP on arm. - pc &= (pint_t)~0x1; + rawPC &= (pint_t)~0x1; +#endif + + typename R::link_reg_t pc; +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + _registers.loadAndAuthenticateLinkRegister(rawPC, &pc); +#else + pc = rawPC; #endif // Exit early if at the top of the stack. @@ -2679,7 +2776,7 @@ void UnwindCursor::setInfoBasedOnIPRegister(bool isReturnAddress) { // Ask address space object to find unwind sections for this pc. UnwindInfoSections sects; - if (_addressSpace.findUnwindSections(pc, sects)) { + if (_addressSpace.template findUnwindSections(pc, sects)) { #if defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) // If there is a compact unwind encoding table, look there first. if (sects.compact_unwind_section != 0) { @@ -2735,8 +2832,8 @@ void UnwindCursor::setInfoBasedOnIPRegister(bool isReturnAddress) { #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) // There is no static unwind info for this pc. Look to see if an FDE was // dynamically registered for it. - pint_t cachedFDE = DwarfFDECache::findFDE(DwarfFDECache::kSearchAll, - pc); + pint_t cachedFDE = + DwarfFDECache::template findFDE(DwarfFDECache::kSearchAll, pc); if (cachedFDE != 0) { typename CFI_Parser::FDE_Info fdeInfo; typename CFI_Parser::CIE_Info cieInfo; @@ -2748,7 +2845,7 @@ void UnwindCursor::setInfoBasedOnIPRegister(bool isReturnAddress) { // Lastly, ask AddressSpace object about platform specific ways to locate // other FDEs. pint_t fde; - if (_addressSpace.findOtherFDE(pc, fde)) { + if (_addressSpace.template findOtherFDE(pc, fde)) { typename CFI_Parser::FDE_Info fdeInfo; typename CFI_Parser::CIE_Info cieInfo; if (!CFI_Parser::decodeFDE(_addressSpace, fde, &fdeInfo, &cieInfo)) { @@ -2772,6 +2869,21 @@ void UnwindCursor::setInfoBasedOnIPRegister(bool isReturnAddress) { #if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) && \ defined(_LIBUNWIND_TARGET_AARCH64) + +/* + * The linux sigreturn restorer stub will always have the form: + * + * d2801168 movz x8, #0x8b + * d4000001 svc #0x0 + */ +#if defined(__AARCH64EB__) +#define MOVZ_X8_8B 0x681180d2 +#define SVC_0 0x010000d4 +#else +#define MOVZ_X8_8B 0xd2801168 +#define SVC_0 0xd4000001 +#endif + template bool UnwindCursor::setInfoForSigReturn(Registers_arm64 &) { // Look for the sigreturn trampoline. The trampoline's body is two @@ -2796,7 +2908,7 @@ bool UnwindCursor::setInfoForSigReturn(Registers_arm64 &) { return false; auto *instructions = reinterpret_cast(pc); // Look for instructions: mov x8, #0x8b; svc #0x0 - if (instructions[0] != 0xd2801168 || instructions[1] != 0xd4000001) + if (instructions[0] != MOVZ_X8_8B || instructions[1] != SVC_0) return false; _info = {}; @@ -3188,16 +3300,22 @@ template int UnwindCursor::step(bool stage2) { template void UnwindCursor::getInfo(unw_proc_info_t *info) { if (_unwindInfoMissing) - memset(info, 0, sizeof(*info)); + memset(static_cast(info), 0, sizeof(*info)); else *info = _info; } template bool UnwindCursor::getFunctionName(char *buf, size_t bufLen, - unw_word_t *offset) { - return _addressSpace.findFunctionName((pint_t)this->getReg(UNW_REG_IP), - buf, bufLen, offset); + unw_word_t *offset) { +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + typename R::reg_t rawPC = this->getReg(UNW_REG_IP); + typename R::link_reg_t pc; + _registers.loadAndAuthenticateLinkRegister(rawPC, &pc); +#else + typename R::link_reg_t pc = this->getReg(UNW_REG_IP); +#endif + return _addressSpace.template findFunctionName(pc, buf, bufLen, offset); } #if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) diff --git a/lib/libunwind/src/UnwindLevel1.c b/lib/libunwind/src/UnwindLevel1.c index f3b451ad9b..7368b3cb80 100644 --- a/lib/libunwind/src/UnwindLevel1.c +++ b/lib/libunwind/src/UnwindLevel1.c @@ -48,16 +48,15 @@ // avoided when invoking the `jumpto()` function. To do this, we use inline // assemblies to "goto" the `jumpto()` for these architectures. #if !defined(_LIBUNWIND_USE_CET) && !defined(_LIBUNWIND_USE_GCS) -#define __unw_phase2_resume(cursor, fn) \ +#define __unw_phase2_resume(cursor, payload) \ do { \ - (void)fn; \ - __unw_resume((cursor)); \ + __unw_resume_with_frames_walked((cursor), (payload)); \ } while (0) #elif defined(_LIBUNWIND_TARGET_I386) #define __shstk_step_size (4) -#define __unw_phase2_resume(cursor, fn) \ +#define __unw_phase2_resume(cursor, payload) \ do { \ - _LIBUNWIND_POP_SHSTK_SSP((fn)); \ + _LIBUNWIND_POP_SHSTK_SSP((payload)); \ void *shstkRegContext = __libunwind_shstk_get_registers((cursor)); \ void *shstkJumpAddress = __libunwind_shstk_get_jump_target(); \ __asm__ volatile("push %%edi\n\t" \ @@ -67,9 +66,9 @@ } while (0) #elif defined(_LIBUNWIND_TARGET_X86_64) #define __shstk_step_size (8) -#define __unw_phase2_resume(cursor, fn) \ +#define __unw_phase2_resume(cursor, payload) \ do { \ - _LIBUNWIND_POP_SHSTK_SSP((fn)); \ + _LIBUNWIND_POP_SHSTK_SSP((payload)); \ void *shstkRegContext = __libunwind_shstk_get_registers((cursor)); \ void *shstkJumpAddress = __libunwind_shstk_get_jump_target(); \ __asm__ volatile("jmpq *%%rdx\n\t" ::"D"(shstkRegContext), \ @@ -77,19 +76,37 @@ } while (0) #elif defined(_LIBUNWIND_TARGET_AARCH64) #define __shstk_step_size (8) -#define __unw_phase2_resume(cursor, fn) \ +#define __unw_phase2_resume(cursor, payload) \ do { \ - _LIBUNWIND_POP_SHSTK_SSP((fn)); \ + _LIBUNWIND_POP_SHSTK_SSP((payload)); \ void *shstkRegContext = __libunwind_shstk_get_registers((cursor)); \ void *shstkJumpAddress = __libunwind_shstk_get_jump_target(); \ __asm__ volatile("mov x0, %0\n\t" \ + "mov x1, #0\n\t" \ "br %1\n\t" \ : \ : "r"(shstkRegContext), "r"(shstkJumpAddress) \ - : "x0"); \ + : "x0", "x1"); \ } while (0) #endif +// We need this helper function as the semantics of casting between integers and +// function pointers mean that we end up with a function pointer without the +// correct signature. Instead we assign to an integer with a matching schema, +// and then memmove the result into a variable of the correct type. This memmove +// is possible as `_Unwind_Personality_Fn` is a standard function pointer, and +// as such is not address diversified. +static _Unwind_Personality_Fn get_handler_function(unw_proc_info_t *frameInfo) { + uintptr_t __unwind_ptrauth_restricted_intptr(ptrauth_key_function_pointer, + 0, + ptrauth_function_pointer_type_discriminator(_Unwind_Personality_Fn)) + reauthenticatedIntegerHandler = frameInfo->handler; + _Unwind_Personality_Fn handler; + memmove(&handler, (void *)&reauthenticatedIntegerHandler, + sizeof(_Unwind_Personality_Fn)); + return handler; +} + static _Unwind_Reason_Code unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *exception_object) { __unw_init_local(cursor, uc); @@ -147,8 +164,7 @@ unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *except // If there is a personality routine, ask it if it will want to stop at // this frame. if (frameInfo.handler != 0) { - _Unwind_Personality_Fn p = - (_Unwind_Personality_Fn)(uintptr_t)(frameInfo.handler); + _Unwind_Personality_Fn p = get_handler_function(&frameInfo); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase1(ex_obj=%p): calling personality function %p", (void *)exception_object, (void *)(uintptr_t)p); @@ -184,11 +200,12 @@ unwind_phase1(unw_context_t *uc, unw_cursor_t *cursor, _Unwind_Exception *except } return _URC_NO_REASON; } -extern int __unw_step_stage2(unw_cursor_t *); #if defined(_LIBUNWIND_USE_GCS) // Enable the GCS target feature to permit gcspop instructions to be used. __attribute__((target("+gcs"))) +#else +_LIBUNWIND_TRACE_NO_INLINE #endif static _Unwind_Reason_Code unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, @@ -276,8 +293,7 @@ unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, ++framesWalked; // If there is a personality routine, tell it we are unwinding. if (frameInfo.handler != 0) { - _Unwind_Personality_Fn p = - (_Unwind_Personality_Fn)(uintptr_t)(frameInfo.handler); + _Unwind_Personality_Fn p = get_handler_function(&frameInfo); _Unwind_Action action = _UA_CLEANUP_PHASE; if (sp == exception_object->private_2) { // Tell personality this was the frame it marked in phase 1. @@ -334,6 +350,8 @@ unwind_phase2(unw_context_t *uc, unw_cursor_t *cursor, #if defined(_LIBUNWIND_USE_GCS) // Enable the GCS target feature to permit gcspop instructions to be used. __attribute__((target("+gcs"))) +#else +_LIBUNWIND_TRACE_NO_INLINE #endif static _Unwind_Reason_Code unwind_phase2_forced(unw_context_t *uc, unw_cursor_t *cursor, @@ -394,8 +412,7 @@ unwind_phase2_forced(unw_context_t *uc, unw_cursor_t *cursor, ++framesWalked; // If there is a personality routine, tell it we are unwinding. if (frameInfo.handler != 0) { - _Unwind_Personality_Fn p = - (_Unwind_Personality_Fn)(intptr_t)(frameInfo.handler); + _Unwind_Personality_Fn p = get_handler_function(&frameInfo); _LIBUNWIND_TRACE_UNWINDING( "unwind_phase2_forced(ex_obj=%p): calling personality function %p", (void *)exception_object, (void *)(uintptr_t)p); @@ -597,6 +614,18 @@ _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIP(struct _Unwind_Context *context) { unw_cursor_t *cursor = (unw_cursor_t *)context; unw_word_t result; __unw_get_reg(cursor, UNW_REG_IP, &result); + +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + // If we are in an arm64e frame, then the PC should have been signed with the + // sp + { + unw_word_t sp; + __unw_get_reg(cursor, UNW_REG_SP, &sp); + result = (unw_word_t)ptrauth_auth_data((void *)result, + ptrauth_key_return_address, sp); + } +#endif + _LIBUNWIND_TRACE_API("_Unwind_GetIP(context=%p) => 0x%" PRIxPTR, (void *)context, result); return (uintptr_t)result; diff --git a/lib/libunwind/src/UnwindRegistersRestore.S b/lib/libunwind/src/UnwindRegistersRestore.S index 1bcd205be2..76a8034403 100644 --- a/lib/libunwind/src/UnwindRegistersRestore.S +++ b/lib/libunwind/src/UnwindRegistersRestore.S @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#if !defined(__wasm__) + #include "assembly.h" #define FROM_0_TO_15 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 @@ -16,13 +18,17 @@ #if defined(_AIX) .toc +#elif defined(__aarch64__) && defined(__ELF__) && defined(_LIBUNWIND_EXECUTE_ONLY_CODE) + .section .text,"axy",@progbits,unique,0 #else .text #endif -#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) +#if !defined(__USING_SJLJ_EXCEPTIONS__) #if defined(__i386__) +.att_syntax + DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_jumpto) # # extern "C" void __libunwind_Registers_x86_jumpto(Registers_x86 *); @@ -67,6 +73,7 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_jumpto) # skip gs #elif defined(__x86_64__) && !defined(__arm64ec__) +.att_syntax DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_x86_64_jumpto) # @@ -629,18 +636,35 @@ Lnovec: #elif defined(__aarch64__) +#ifndef __has_feature +#define __has_feature(__feature) 0 +#endif + #if defined(__ARM_FEATURE_GCS_DEFAULT) .arch_extension gcs #endif // -// extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *); +// extern "C" void __libunwind_Registers_arm64_jumpto(Registers_arm64 *, unsigned); // // On entry: // thread_state pointer is in x0 +// walked_frames counter is in x1 // .p2align 2 DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto) + + #if defined(_LIBUNWIND_TRACE_RET_INJECT) + cbz w1, 1f + 0: + subs w1, w1, #1 + adr x16, #8 + ret x16 + + b.ne 0b + 1: + #endif + // skip restore of x0,x1 for now ldp x2, x3, [x0, #0x010] ldp x4, x5, [x0, #0x020] @@ -657,7 +681,7 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto) ldp x24,x25, [x0, #0x0C0] ldp x26,x27, [x0, #0x0D0] ldp x28,x29, [x0, #0x0E0] - ldr x30, [x0, #0x100] // restore pc into lr + #if defined(__ARM_FP) && __ARM_FP != 0 ldp d0, d1, [x0, #0x110] ldp d2, d3, [x0, #0x120] @@ -681,7 +705,18 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto) // context struct, because it is allocated on the stack, and an exception // could clobber the de-allocated portion of the stack after sp has been // restored. - ldr x16, [x0, #0x0F8] + + ldr x16, [x0, #0x0F8] // load sp into scratch + ldr lr, [x0, #0x100] // restore pc into lr + +#if __has_feature(ptrauth_calls) + // The LR is signed with its address inside the register state. Time + // to resign to be a regular ROP protected signed pointer + add x1, x0, #0x100 + autib lr, x1 + pacib lr, x16 // signed the scratch register for sp +#endif + ldp x0, x1, [x0, #0x000] // restore x0,x1 mov sp,x16 // restore sp #if defined(__ARM_FEATURE_GCS_DEFAULT) @@ -694,7 +729,12 @@ DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_jumpto) gcspushm x30 Lnogcs: #endif + +#if __has_feature(ptrauth_calls) + retab +#else ret x30 // jump to pc +#endif #elif defined(__arm__) && !defined(__APPLE__) @@ -1253,7 +1293,8 @@ DEFINE_LIBUNWIND_FUNCTION(_ZN9libunwind19Registers_loongarch6jumptoEv) #endif -#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) */ +#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) */ NO_EXEC_STACK_DIRECTIVE +#endif /* !defined(__wasm__) */ diff --git a/lib/libunwind/src/UnwindRegistersSave.S b/lib/libunwind/src/UnwindRegistersSave.S index 5139a551ad..f988fd461d 100644 --- a/lib/libunwind/src/UnwindRegistersSave.S +++ b/lib/libunwind/src/UnwindRegistersSave.S @@ -6,6 +6,8 @@ // //===----------------------------------------------------------------------===// +#if !defined(__wasm__) + #include "assembly.h" #define FROM_0_TO_15 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 @@ -16,13 +18,16 @@ #if defined(_AIX) .toc +#elif defined(__aarch64__) && defined(__ELF__) && defined(_LIBUNWIND_EXECUTE_ONLY_CODE) + .section .text,"axy",@progbits,unique,0 #else .text #endif -#if !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) +#if !defined(__USING_SJLJ_EXCEPTIONS__) #if defined(__i386__) +.att_syntax # # extern int __unw_getcontext(unw_context_t* thread_state) @@ -107,6 +112,7 @@ DEFINE_LIBUNWIND_FUNCTION("#__unw_getcontext") .text #elif defined(__x86_64__) +.att_syntax # # extern int __unw_getcontext(unw_context_t* thread_state) @@ -759,6 +765,10 @@ LnoR2Fix: #elif defined(__aarch64__) +#ifndef __has_feature +#define __has_feature(__feature) 0 +#endif + // // extern int __unw_getcontext(unw_context_t* thread_state) // @@ -767,6 +777,11 @@ LnoR2Fix: // .p2align 2 DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext) + +#if __has_feature(ptrauth_calls) + pacibsp +#endif + stp x0, x1, [x0, #0x000] stp x2, x3, [x0, #0x010] stp x4, x5, [x0, #0x020] @@ -807,7 +822,74 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext) str d31, [x0, #0x208] #endif mov x0, #0 // return UNW_ESUCCESS + +#if __has_feature(ptrauth_calls) + retab +#else ret +#endif + +// +// extern "C" int64_t __libunwind_Registers_arm64_za_disable() +// +// This function implements the requirements of the __arm_za_disable ABI +// routine, except that it will not abort; it will return a non-zero value +// to signify the routine failed. +// +// Note: This function uses SME instructions. It must only be called if SME +// has been confirmed to be available. +// +// On return: +// +// A status is placed in x0. A zero value indicates success; any non-zero +// value indicates failure. +// + .p2align 2 +DEFINE_LIBUNWIND_FUNCTION(__libunwind_Registers_arm64_za_disable) + .variant_pcs __libunwind_Registers_arm64_za_disable +#if __has_feature(ptrauth_calls) + pacibsp +#endif + // If TPIDR2_EL0 is null, the subroutine just disables ZA. + .inst 0xd53bd0b0 // mrs x16, TPIDR2_EL0 + cbz x16, 1f + + // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are + // nonzero, return a non-zero value (libunwind will then abort). + ldrh w0, [x16, #10] + cbnz w0, 2f + ldr w0, [x16, #12] + cbnz w0, 2f + + // If num_za_save_slices is zero, the subroutine just disables ZA. + ldrh w0, [x16, #8] + cbz x0, 1f + + // If za_save_buffer is NULL, the subroutine just disables ZA. + ldr x16, [x16] + cbz x16, 1f + + // Store ZA to za_save_buffer. + mov x15, xzr +0: + .inst 0xe1206200 // str za[w15,0], [x16] + .inst 0x04305830 // addsvl x16, x16, #1 + add x15, x15, #1 + cmp x0, x15 + b.ne 0b +1: + // * Set TPIDR2_EL0 to null. + .inst 0xd51bd0bf // msr TPIDR2_EL0, xzr + // * Set PSTATE.ZA to 0. + .inst 0xd503447f // smstop za + // * Return zero (success) + mov x0, xzr +2: +#if __has_feature(ptrauth_calls) + retab +#else + ret +#endif #elif defined(__arm__) && !defined(__APPLE__) @@ -1232,6 +1314,8 @@ DEFINE_LIBUNWIND_FUNCTION(__unw_getcontext) WEAK_ALIAS(__unw_getcontext, unw_getcontext) #endif -#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) */ +#endif /* !defined(__USING_SJLJ_EXCEPTIONS__) */ NO_EXEC_STACK_DIRECTIVE + +#endif /* !defined(__wasm__) */ diff --git a/lib/libunwind/src/assembly.h b/lib/libunwind/src/assembly.h index f8e83e138e..84c9d526f1 100644 --- a/lib/libunwind/src/assembly.h +++ b/lib/libunwind/src/assembly.h @@ -15,7 +15,7 @@ #ifndef UNWIND_ASSEMBLY_H #define UNWIND_ASSEMBLY_H -#if defined(__linux__) && defined(__CET__) +#if defined(__CET__) #include #define _LIBUNWIND_CET_ENDBR _CET_ENDBR #else @@ -132,6 +132,10 @@ #if defined(__APPLE__) +#if defined(__aarch64__) || defined(__arm64__) || defined(__arm64e__) +#define _LIBUNWIND_TRACE_RET_INJECT 1 +#endif + #define SYMBOL_IS_FUNC(name) #define HIDDEN_SYMBOL(name) .private_extern name #if defined(_LIBUNWIND_HIDE_SYMBOLS) diff --git a/lib/libunwind/src/config.h b/lib/libunwind/src/config.h index deb5a4d4d7..f017403fa2 100644 --- a/lib/libunwind/src/config.h +++ b/lib/libunwind/src/config.h @@ -28,6 +28,9 @@ #define _LIBUNWIND_SUPPORT_COMPACT_UNWIND 1 #define _LIBUNWIND_SUPPORT_DWARF_UNWIND 1 #endif + #if defined(__aarch64__) || defined(__arm64__) || defined(__arm64e__) + #define _LIBUNWIND_TRACE_RET_INJECT 1 + #endif #elif defined(_WIN32) #ifdef __SEH__ #define _LIBUNWIND_SUPPORT_SEH_UNWIND 1 @@ -61,6 +64,12 @@ #endif #endif +#ifdef _LIBUNWIND_TRACE_RET_INJECT +#define _LIBUNWIND_TRACE_NO_INLINE __attribute__((noinline, disable_tail_calls)) +#else +#define _LIBUNWIND_TRACE_NO_INLINE +#endif + #if defined(_LIBUNWIND_HIDE_SYMBOLS) // The CMake file passes -fvisibility=hidden to control ELF/Mach-O visibility. #define _LIBUNWIND_EXPORT diff --git a/lib/libunwind/src/gcc_personality_v0.c b/lib/libunwind/src/gcc_personality_v0.c index 18f5a9cf89..63003e3b31 100644 --- a/lib/libunwind/src/gcc_personality_v0.c +++ b/lib/libunwind/src/gcc_personality_v0.c @@ -31,6 +31,58 @@ EXCEPTION_DISPOSITION _GCC_specific_handler(PEXCEPTION_RECORD, void *, PCONTEXT, _Unwind_Personality_Fn); #endif +#ifndef __has_feature +#define __has_feature(__feature) 0 +#endif + +#if __has_feature(ptrauth_calls) +#include + +// `__ptrauth_restricted_intptr` is a feature of apple clang that predates +// support for direct application of `__ptrauth` to integer types. This +// guard is necessary to support compilation with those compiler. +#if __has_feature(ptrauth_restricted_intptr_qualifier) +#define __ptrauth_gcc_personality_intptr(key, addressDiscriminated, \ + discriminator) \ + __ptrauth_restricted_intptr(key, addressDiscriminated, discriminator) +#else +#define __ptrauth_gcc_personality_intptr(key, addressDiscriminated, \ + discriminator) \ + __ptrauth(key, addressDiscriminated, discriminator) +#endif +#else +#define __ptrauth_gcc_personality_intptr(...) +#endif + +#define __ptrauth_gcc_personality_func_key ptrauth_key_function_pointer + +// ptrauth_string_discriminator("__gcc_personality_v0'funcStart") == 0xDFEB +#define __ptrauth_gcc_personality_func_start \ + __ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \ + 0xDFEB) + +// ptrauth_string_discriminator("__gcc_personality_v0'start") == 0x52DC +#define __ptrauth_gcc_personality_start \ + __ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \ + 0x52DC) + +// ptrauth_string_discriminator("__gcc_personality_v0'length") == 0xFFF7 +#define __ptrauth_gcc_personality_length \ + __ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \ + 0xFFF7) + +// ptrauth_string_discriminator("__gcc_personality_v0'landingPadOffset") == +// 0x6498 +#define __ptrauth_gcc_personality_lpoffset \ + __ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \ + 0x6498) + +// ptrauth_string_discriminator("__gcc_personality_v0'landingPad") == 0xA134 +#define __ptrauth_gcc_personality_lpad_disc 0xA134 +#define __ptrauth_gcc_personality_lpad \ + __ptrauth_gcc_personality_intptr(__ptrauth_gcc_personality_func_key, 1, \ + __ptrauth_gcc_personality_lpad_disc) + // Pointer encodings documented at: // http://refspecs.freestandards.org/LSB_1.3.0/gLSB/gLSB/ehframehdr.html @@ -206,7 +258,8 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( return continueUnwind(exceptionObject, context); uintptr_t pc = (uintptr_t)_Unwind_GetIP(context) - 1; - uintptr_t funcStart = (uintptr_t)_Unwind_GetRegionStart(context); + uintptr_t __ptrauth_gcc_personality_func_start funcStart = + (uintptr_t)_Unwind_GetRegionStart(context); uintptr_t pcOffset = pc - funcStart; // Parse LSDA header. @@ -225,11 +278,14 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( const uint8_t *callSiteTableEnd = callSiteTableStart + callSiteTableLength; const uint8_t *p = callSiteTableStart; while (p < callSiteTableEnd) { - uintptr_t start = readEncodedPointer(&p, callSiteEncoding); - size_t length = readEncodedPointer(&p, callSiteEncoding); - size_t landingPad = readEncodedPointer(&p, callSiteEncoding); + uintptr_t __ptrauth_gcc_personality_start start = + readEncodedPointer(&p, callSiteEncoding); + size_t __ptrauth_gcc_personality_length length = + readEncodedPointer(&p, callSiteEncoding); + size_t __ptrauth_gcc_personality_lpoffset landingPadOffset = + readEncodedPointer(&p, callSiteEncoding); readULEB128(&p); // action value not used for C code - if (landingPad == 0) + if (landingPadOffset == 0) continue; // no landing pad for this entry if ((start <= pcOffset) && (pcOffset < (start + length))) { // Found landing pad for the PC. @@ -239,7 +295,24 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), (uintptr_t)exceptionObject); _Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0); - _Unwind_SetIP(context, (funcStart + landingPad)); + size_t __ptrauth_gcc_personality_lpad landingPad = + funcStart + landingPadOffset; +#if __has_feature(ptrauth_calls) + uintptr_t stackPointer = _Unwind_GetGR(context, -2); + const uintptr_t existingDiscriminator = ptrauth_blend_discriminator( + &landingPad, __ptrauth_gcc_personality_lpad_disc); + // newIP is authenticated as if it were qualified with a pseudo qualifier + // along the lines of: + // __ptrauth(ptrauth_key_return_address, , 0) + // where the stack pointer is used in place of the strict storage + // address. + uintptr_t newIP = (uintptr_t)ptrauth_auth_and_resign( + *(void **)&landingPad, __ptrauth_gcc_personality_func_key, + existingDiscriminator, ptrauth_key_return_address, stackPointer); + _Unwind_SetIP(context, newIP); +#else + _Unwind_SetIP(context, landingPad); +#endif return _URC_INSTALL_CONTEXT; } } diff --git a/lib/libunwind/src/libunwind.cpp b/lib/libunwind/src/libunwind.cpp index cf39ec5f7d..7ffffc2a30 100644 --- a/lib/libunwind/src/libunwind.cpp +++ b/lib/libunwind/src/libunwind.cpp @@ -118,14 +118,55 @@ _LIBUNWIND_HIDDEN int __unw_set_reg(unw_cursor_t *cursor, unw_regnum_t regNum, typedef LocalAddressSpace::pint_t pint_t; AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; if (co->validReg(regNum)) { - co->setReg(regNum, (pint_t)value); // special case altering IP to re-find info (being called by personality // function) if (regNum == UNW_REG_IP) { unw_proc_info_t info; // First, get the FDE for the old location and then update it. co->getInfo(&info); - co->setInfoBasedOnIPRegister(false); + + pint_t sp = (pint_t)co->getReg(UNW_REG_SP); + +#if defined(_LIBUNWIND_TARGET_AARCH64_AUTHENTICATED_UNWINDING) + { + // It is only valid to set the IP within the current function. This is + // important for ptrauth, otherwise the IP cannot be correctly signed. + // The current signature of `value` is via the schema: + // __ptrauth(ptrauth_key_return_address, <>, 0) + // For this to be generally usable we manually re-sign it to the + // directly supported schema: + // __ptrauth(ptrauth_key_return_address, 1, 0) + unw_word_t + __unwind_ptrauth_restricted_intptr(ptrauth_key_return_address, 1, + 0) authenticated_value; + unw_word_t opaque_value = (uint64_t)ptrauth_auth_and_resign( + (void *)value, ptrauth_key_return_address, sp, + ptrauth_key_return_address, &authenticated_value); + memmove(reinterpret_cast(&authenticated_value), + reinterpret_cast(&opaque_value), + sizeof(authenticated_value)); + if (authenticated_value < info.start_ip || + authenticated_value > info.end_ip) + _LIBUNWIND_ABORT("PC vs frame info mismatch"); + + // PC should have been signed with the sp, so we verify that + // roundtripping does not fail. The `ptrauth_auth_and_resign` is + // guaranteed to trap on authentication failure even without FPAC + // feature. + pint_t pc = (pint_t)co->getReg(UNW_REG_IP); + if (ptrauth_auth_and_resign((void *)pc, ptrauth_key_return_address, sp, + ptrauth_key_return_address, + sp) != (void *)pc) { + _LIBUNWIND_LOG( + "Bad unwind with PAuth-enabled ABI (0x%zX, 0x%zX)->0x%zX\n", pc, + sp, + (pint_t)ptrauth_auth_data((void *)pc, ptrauth_key_return_address, + sp)); + _LIBUNWIND_ABORT("Bad unwind with PAuth-enabled ABI"); + } + } +#endif + // If the original call expects stack adjustment, perform this now. // Normal frame unwinding would have included the offset already in the // CFA computation. @@ -133,7 +174,11 @@ _LIBUNWIND_HIDDEN int __unw_set_reg(unw_cursor_t *cursor, unw_regnum_t regNum, // this should actually be - info.gp. LLVM doesn't currently support // any such platforms and Clang doesn't export a macro for them. if (info.gp) - co->setReg(UNW_REG_SP, co->getReg(UNW_REG_SP) + info.gp); + co->setReg(UNW_REG_SP, sp + info.gp); + co->setReg(UNW_REG_IP, value); + co->setInfoBasedOnIPRegister(false); + } else { + co->setReg(regNum, (pint_t)value); } return UNW_ESUCCESS; } @@ -205,7 +250,27 @@ _LIBUNWIND_HIDDEN int __unw_get_proc_info(unw_cursor_t *cursor, } _LIBUNWIND_WEAK_ALIAS(__unw_get_proc_info, unw_get_proc_info) -/// Resume execution at cursor position (aka longjump). +/// Rebalance the execution flow by injecting the right amount of `ret` +/// instruction relatively to the amount of `walkedFrames` then resume execution +/// at cursor position (aka longjump). +_LIBUNWIND_HIDDEN int __unw_resume_with_frames_walked(unw_cursor_t *cursor, + unsigned walkedFrames) { + _LIBUNWIND_TRACE_API("__unw_resume(cursor=%p, walkedFrames=%u)", + static_cast(cursor), walkedFrames); +#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) + // Inform the ASan runtime that now might be a good time to clean stuff up. + __asan_handle_no_return(); +#endif +#ifdef _LIBUNWIND_TRACE_RET_INJECT + AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; + co->setWalkedFrames(walkedFrames); +#endif + return __unw_resume(cursor); +} +_LIBUNWIND_WEAK_ALIAS(__unw_resume_with_frames_walked, + unw_resume_with_frames_walked) + +/// Legacy function. Resume execution at cursor position (aka longjump). _LIBUNWIND_HIDDEN int __unw_resume(unw_cursor_t *cursor) { _LIBUNWIND_TRACE_API("__unw_resume(cursor=%p)", static_cast(cursor)); #if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) @@ -347,6 +412,41 @@ void __unw_remove_dynamic_eh_frame_section(unw_word_t eh_frame_start) { } #endif // defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) + +/// Maps the UNW_* error code to a textual representation +_LIBUNWIND_HIDDEN const char *__unw_strerror(int error_code) { + switch (error_code) { + case UNW_ESUCCESS: + return "no error"; + case UNW_EUNSPEC: + return "unspecified (general) error"; + case UNW_ENOMEM: + return "out of memory"; + case UNW_EBADREG: + return "bad register number"; + case UNW_EREADONLYREG: + return "attempt to write read-only register"; + case UNW_ESTOPUNWIND: + return "stop unwinding"; + case UNW_EINVALIDIP: + return "invalid IP"; + case UNW_EBADFRAME: + return "bad frame"; + case UNW_EINVAL: + return "unsupported operation or bad value"; + case UNW_EBADVERSION: + return "unwind info has unsupported version"; + case UNW_ENOINFO: + return "no unwind info found"; +#if defined(_LIBUNWIND_TARGET_AARCH64) && !defined(_LIBUNWIND_IS_NATIVE_ONLY) + case UNW_ECROSSRASIGNING: + return "cross unwind with return address signing"; +#endif + } + return "invalid error code"; +} +_LIBUNWIND_WEAK_ALIAS(__unw_strerror, unw_strerror) + #endif // !defined(__USING_SJLJ_EXCEPTIONS__) && !defined(__wasm__) #ifdef __APPLE__ diff --git a/lib/libunwind/src/libunwind_ext.h b/lib/libunwind/src/libunwind_ext.h index 28db43a4f6..b3762c24d7 100644 --- a/lib/libunwind/src/libunwind_ext.h +++ b/lib/libunwind/src/libunwind_ext.h @@ -26,11 +26,16 @@ extern "C" { extern int __unw_getcontext(unw_context_t *); extern int __unw_init_local(unw_cursor_t *, unw_context_t *); extern int __unw_step(unw_cursor_t *); +extern int __unw_step_stage2(unw_cursor_t *); extern int __unw_get_reg(unw_cursor_t *, unw_regnum_t, unw_word_t *); extern int __unw_get_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t *); extern int __unw_set_reg(unw_cursor_t *, unw_regnum_t, unw_word_t); extern int __unw_set_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t); -extern int __unw_resume(unw_cursor_t *); +_LIBUNWIND_TRACE_NO_INLINE + extern int __unw_resume_with_frames_walked(unw_cursor_t *, unsigned); +// `__unw_resume` is a legacy function. Use `__unw_resume_with_frames_walked` instead. +_LIBUNWIND_TRACE_NO_INLINE + extern int __unw_resume(unw_cursor_t *); #ifdef __arm__ /* Save VFP registers in FSTMX format (instead of FSTMD). */ @@ -42,6 +47,7 @@ extern int __unw_get_proc_info(unw_cursor_t *, unw_proc_info_t *); extern int __unw_is_fpreg(unw_cursor_t *, unw_regnum_t); extern int __unw_is_signal_frame(unw_cursor_t *); extern int __unw_get_proc_name(unw_cursor_t *, char *, size_t, unw_word_t *); +extern const char *__unw_strerror(int); #if defined(_AIX) extern uintptr_t __unw_get_data_rel_base(unw_cursor_t *); diff --git a/lib/libunwind/src/shadow_stack_unwind.h b/lib/libunwind/src/shadow_stack_unwind.h index 1f229d8317..b00ca2c932 100644 --- a/lib/libunwind/src/shadow_stack_unwind.h +++ b/lib/libunwind/src/shadow_stack_unwind.h @@ -12,8 +12,8 @@ #include "libunwind.h" -// Currently, CET is implemented on Linux x86 platforms. -#if defined(_LIBUNWIND_TARGET_LINUX) && defined(__CET__) && defined(__SHSTK__) +// Currently, CET is implemented on some ELF x86 platforms. +#if defined(__CET__) && defined(__SHSTK__) #define _LIBUNWIND_USE_CET 1 #endif