diff --git a/coroutine/amd64/Context.h b/coroutine/amd64/Context.h index 65aa6383044..0deb9b6dd4f 100644 --- a/coroutine/amd64/Context.h +++ b/coroutine/amd64/Context.h @@ -32,6 +32,22 @@ enum {COROUTINE_REGISTERS = 6}; #include #endif +#if defined(__SANITIZE_THREAD__) + #define COROUTINE_SANITIZE_THREAD +#elif defined(__has_feature) + #if __has_feature(thread_sanitizer) + #define COROUTINE_SANITIZE_THREAD + #endif +#endif + +#if defined(COROUTINE_SANITIZE_THREAD) +/* ThreadSanitizer cannot follow a userspace stack switch on its own: its + * per-OS-thread shadow stack must be handed to the destination fiber on every + * coroutine switch via the fiber API, otherwise it leaks the shadow stack + * across switches and eventually faults inside libtsan. */ +#include +#endif + struct coroutine_context { void **stack_pointer; @@ -42,12 +58,26 @@ struct coroutine_context void *stack_base; size_t stack_size; #endif + +#if defined(COROUTINE_SANITIZE_THREAD) + void *tsan_fiber; + /* Whether we created tsan_fiber (via __tsan_create_fiber, must be + * destroyed) or borrowed it from __tsan_get_current_fiber (the OS thread's + * implicit fiber, owned by TSan; must not be destroyed). */ + int tsan_fiber_owned; +#endif }; typedef COROUTINE(* coroutine_start)(struct coroutine_context *from, struct coroutine_context *self); static inline void coroutine_initialize_main(struct coroutine_context * context) { context->stack_pointer = NULL; + +#if defined(COROUTINE_SANITIZE_THREAD) + /* The OS thread's implicit (already running) fiber, owned by TSan. */ + context->tsan_fiber = __tsan_get_current_fiber(); + context->tsan_fiber_owned = 0; +#endif } static inline void coroutine_initialize( @@ -64,6 +94,11 @@ static inline void coroutine_initialize( context->stack_size = size; #endif +#if defined(COROUTINE_SANITIZE_THREAD) + context->tsan_fiber = __tsan_create_fiber(0); + context->tsan_fiber_owned = 1; +#endif + // Stack grows down. Force 16-byte alignment. char * top = (char*)stack + size; context->stack_pointer = (void**)((uintptr_t)top & ~0xF); @@ -80,6 +115,17 @@ struct coroutine_context * coroutine_transfer(struct coroutine_context * current static inline void coroutine_destroy(struct coroutine_context * context) { context->stack_pointer = NULL; + +#if defined(COROUTINE_SANITIZE_THREAD) + /* Only destroy fibers we created. The borrowed __tsan_get_current_fiber() + * handle (the OS thread's implicit fiber) is owned by TSan; destroying it + * aborts libtsan (FiberDestroy -> ProcWire CheckFailed). */ + if (context->tsan_fiber && context->tsan_fiber_owned) { + __tsan_destroy_fiber(context->tsan_fiber); + context->tsan_fiber = NULL; + context->tsan_fiber_owned = 0; + } +#endif } #endif /* COROUTINE_AMD64_CONTEXT_H */ diff --git a/thread_pthread.c b/thread_pthread.c index 3214e25560c..f819a0030e2 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -1237,6 +1237,13 @@ coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_co __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size); #endif +#if defined(COROUTINE_SANITIZE_THREAD) + /* Tell TSan we are switching to transfer_to's fiber before the stack + * switch, so its per-thread shadow stack stays bound to the right + * coroutine. */ + __tsan_switch_to_fiber(transfer_to->tsan_fiber, 0); +#endif + RBIMPL_ATTR_MAYBE_UNUSED() struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to);