1391824ddim//===-- tsan_rtl_report.cpp -----------------------------------------------===//
2cfeab00andrew//
38312957dim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
48312957dim// See https://llvm.org/LICENSE.txt for license information.
58312957dim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6cfeab00andrew//
7cfeab00andrew//===----------------------------------------------------------------------===//
8cfeab00andrew//
9cfeab00andrew// This file is a part of ThreadSanitizer (TSan), a race detector.
10cfeab00andrew//
11cfeab00andrew//===----------------------------------------------------------------------===//
12cfeab00andrew
13cfeab00andrew#include "sanitizer_common/sanitizer_libc.h"
14cfeab00andrew#include "sanitizer_common/sanitizer_placement_new.h"
15254a83bandrew#include "sanitizer_common/sanitizer_stackdepot.h"
16254a83bandrew#include "sanitizer_common/sanitizer_common.h"
172f7fa77ed#include "sanitizer_common/sanitizer_stacktrace.h"
18cfeab00andrew#include "tsan_platform.h"
19cfeab00andrew#include "tsan_rtl.h"
20cfeab00andrew#include "tsan_suppressions.h"
21cfeab00andrew#include "tsan_symbolize.h"
22cfeab00andrew#include "tsan_report.h"
23cfeab00andrew#include "tsan_sync.h"
24cfeab00andrew#include "tsan_mman.h"
25cfeab00andrew#include "tsan_flags.h"
26254a83bandrew#include "tsan_fd.h"
27cfeab00andrew
28254a83bandrewnamespace __tsan {
29254a83bandrew
30391824ddimusing namespace __sanitizer;
31cfeab00andrew
322f1c5ccdimstatic ReportStack *SymbolizeStack(StackTrace trace);
332f7fa77ed
34254a83bandrewvoid TsanCheckFailed(const char *file, int line, const char *cond,
35254a83bandrew                     u64 v1, u64 v2) {
362f1c5ccdim  // There is high probability that interceptors will check-fail as well,
372f1c5ccdim  // on the other hand there is no sense in processing interceptors
382f1c5ccdim  // since we are going to die soon.
392f1c5ccdim  ScopedIgnoreInterceptors ignore;
4024528c5dim#if !SANITIZER_GO
4124528c5dim  cur_thread()->ignore_sync++;
4224528c5dim  cur_thread()->ignore_reads_and_writes++;
4324528c5dim#endif
44254a83bandrew  Printf("FATAL: ThreadSanitizer CHECK failed: "
45254a83bandrew         "%s:%d \"%s\" (0x%zx, 0x%zx)\n",
46254a83bandrew         file, line, cond, (uptr)v1, (uptr)v2);
472f1c5ccdim  PrintCurrentStackSlow(StackTrace::GetCurrentPc());
48cfeab00andrew  Die();
49cfeab00andrew}
50cfeab00andrew
51cfeab00andrew// Can be overriden by an application/test to intercept reports.
52cfeab00andrew#ifdef TSAN_EXTERNAL_HOOKS
53cfeab00andrewbool OnReport(const ReportDesc *rep, bool suppressed);
54cfeab00andrew#else
550d16d37dimSANITIZER_WEAK_CXX_DEFAULT_IMPL
560d16d37dimbool OnReport(const ReportDesc *rep, bool suppressed) {
57cfeab00andrew  (void)rep;
58cfeab00andrew  return suppressed;
59cfeab00andrew}
60cfeab00andrew#endif
61cfeab00andrew
62906f157dimSANITIZER_WEAK_DEFAULT_IMPL
63906f157dimvoid __tsan_on_report(const ReportDesc *rep) {
64906f157dim  (void)rep;
65906f157dim}
66906f157dim
672f1c5ccdimstatic void StackStripMain(SymbolizedStack *frames) {
682f1c5ccdim  SymbolizedStack *last_frame = nullptr;
692f1c5ccdim  SymbolizedStack *last_frame2 = nullptr;
702f1c5ccdim  for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
71cfeab00andrew    last_frame2 = last_frame;
722f1c5ccdim    last_frame = cur;
73cfeab00andrew  }
74cfeab00andrew
75cfeab00andrew  if (last_frame2 == 0)
76cfeab00andrew    return;
7724528c5dim#if !SANITIZER_GO
782f1c5ccdim  const char *last = last_frame->info.function;
792f1c5ccdim  const char *last2 = last_frame2->info.function;
80cfeab00andrew  // Strip frame above 'main'
81cfeab00andrew  if (last2 && 0 == internal_strcmp(last2, "main")) {
822f1c5ccdim    last_frame->ClearAll();
832f1c5ccdim    last_frame2->next = nullptr;
84cfeab00andrew  // Strip our internal thread start routine.
85cfeab00andrew  } else if (last && 0 == internal_strcmp(last, "__tsan_thread_start_func")) {
862f1c5ccdim    last_frame->ClearAll();
872f1c5ccdim    last_frame2->next = nullptr;
88cfeab00andrew  // Strip global ctors init.
89cfeab00andrew  } else if (last && 0 == internal_strcmp(last, "__do_global_ctors_aux")) {
902f1c5ccdim    last_frame->ClearAll();
912f1c5ccdim    last_frame2->next = nullptr;
92cfeab00andrew  // If both are 0, then we probably just failed to symbolize.
93cfeab00andrew  } else if (last || last2) {
94cfeab00andrew    // Ensure that we recovered stack completely. Trimmed stack
95cfeab00andrew    // can actually happen if we do not instrument some code,
96254a83bandrew    // so it's only a debug print. However we must try hard to not miss it
97cfeab00andrew    // due to our fault.
985c652b1dim    DPrintf("Bottom stack frame is missed\n");
99cfeab00andrew  }
100cfeab00andrew#else
101c3f67e1dim  // The last frame always point into runtime (gosched0, goexit0, runtime.main).
1022f1c5ccdim  last_frame->ClearAll();
1032f1c5ccdim  last_frame2->next = nullptr;
104cfeab00andrew#endif
105cfeab00andrew}
106cfeab00andrew
1072f1c5ccdimReportStack *SymbolizeStackId(u32 stack_id) {
1082f1c5ccdim  if (stack_id == 0)
109cfeab00andrew    return 0;
1102f1c5ccdim  StackTrace stack = StackDepotGet(stack_id);
1112f1c5ccdim  if (stack.trace == nullptr)
1122f1c5ccdim    return nullptr;
1132f1c5ccdim  return SymbolizeStack(stack);
1142f1c5ccdim}
1152f1c5ccdim
1162f1c5ccdimstatic ReportStack *SymbolizeStack(StackTrace trace) {
1172f1c5ccdim  if (trace.size == 0)
1182f1c5ccdim    return 0;
1192f1c5ccdim  SymbolizedStack *top = nullptr;
1202f1c5ccdim  for (uptr si = 0; si < trace.size; si++) {
1212f1c5ccdim    const uptr pc = trace.trace[si];
1222f1c5ccdim    uptr pc1 = pc;
1232f1c5ccdim    // We obtain the return address, but we're interested in the previous
1242f1c5ccdim    // instruction.
1252f1c5ccdim    if ((pc & kExternalPCBit) == 0)
1262f1c5ccdim      pc1 = StackTrace::GetPreviousInstructionPc(pc);
1272f1c5ccdim    SymbolizedStack *ent = SymbolizeCode(pc1);
128cfeab00andrew    CHECK_NE(ent, 0);
1292f1c5ccdim    SymbolizedStack *last = ent;
130cfeab00andrew    while (last->next) {
1312f1c5ccdim      last->info.address = pc;  // restore original pc for report
132cfeab00andrew      last = last->next;
133cfeab00andrew    }
1342f1c5ccdim    last->info.address = pc;  // restore original pc for report
1352f1c5ccdim    last->next = top;
1362f1c5ccdim    top = ent;
137cfeab00andrew  }
1382f1c5ccdim  StackStripMain(top);
1392f1c5ccdim
1402f1c5ccdim  ReportStack *stack = ReportStack::New();
1412f1c5ccdim  stack->frames = top;
142cfeab00andrew  return stack;
143cfeab00andrew}
144cfeab00andrew
145a14b3e1dimScopedReportBase::ScopedReportBase(ReportType typ, uptr tag) {
1462f1c5ccdim  ctx->thread_registry->CheckLocked();
147cfeab00andrew  void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc));
148cfeab00andrew  rep_ = new(mem) ReportDesc;
149cfeab00andrew  rep_->typ = typ;
1503c744a5dim  rep_->tag = tag;
1512f1c5ccdim  ctx->report_mtx.Lock();
152cfeab00andrew}
153cfeab00andrew
154a14b3e1dimScopedReportBase::~ScopedReportBase() {
1552f1c5ccdim  ctx->report_mtx.Unlock();
156254a83bandrew  DestroyAndFree(rep_);
157391824ddim  rep_ = nullptr;
158cfeab00andrew}
159cfeab00andrew
160a14b3e1dimvoid ScopedReportBase::AddStack(StackTrace stack, bool suppressable) {
161cfeab00andrew  ReportStack **rs = rep_->stacks.PushBack();
1622f1c5ccdim  *rs = SymbolizeStack(stack);
1632f1c5ccdim  (*rs)->suppressable = suppressable;
164cfeab00andrew}
165cfeab00andrew
166a14b3e1dimvoid ScopedReportBase::AddMemoryAccess(uptr addr, uptr external_tag, Shadow s,
167a14b3e1dim                                       StackTrace stack, const MutexSet *mset) {
168cfeab00andrew  void *mem = internal_alloc(MBlockReportMop, sizeof(ReportMop));
169cfeab00andrew  ReportMop *mop = new(mem) ReportMop;
170cfeab00andrew  rep_->mops.PushBack(mop);
171cfeab00andrew  mop->tid = s.tid();
172cfeab00andrew  mop->addr = addr + s.addr0();
173cfeab00andrew  mop->size = s.size();
1742f7fa77ed  mop->write = s.IsWrite();
1752f7fa77ed  mop->atomic = s.IsAtomic();
1762f1c5ccdim  mop->stack = SymbolizeStack(stack);
1776070792dim  mop->external_tag = external_tag;
1782f1c5ccdim  if (mop->stack)
1792f1c5ccdim    mop->stack->suppressable = true;
180254a83bandrew  for (uptr i = 0; i < mset->Size(); i++) {
181254a83bandrew    MutexSet::Desc d = mset->Get(i);
1822f1c5ccdim    u64 mid = this->AddMutex(d.id);
1832f1c5ccdim    ReportMopMutex mtx = {mid, d.write};
1842f1c5ccdim    mop->mset.PushBack(mtx);
185254a83bandrew  }
186cfeab00andrew}
187cfeab00andrew
188a14b3e1dimvoid ScopedReportBase::AddUniqueTid(int unique_tid) {
1892f1c5ccdim  rep_->unique_tids.PushBack(unique_tid);
1902f1c5ccdim}
1912f1c5ccdim
192a14b3e1dimvoid ScopedReportBase::AddThread(const ThreadContext *tctx, bool suppressable) {
193254a83bandrew  for (uptr i = 0; i < rep_->threads.Size(); i++) {
1942f7fa77ed    if ((u32)rep_->threads[i]->id == tctx->tid)
195254a83bandrew      return;
196254a83bandrew  }
197cfeab00andrew  void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread));
1980d16d37dim  ReportThread *rt = new(mem) ReportThread;
199cfeab00andrew  rep_->threads.PushBack(rt);
200cfeab00andrew  rt->id = tctx->tid;
201906f157dim  rt->os_id = tctx->os_id;
202cfeab00andrew  rt->running = (tctx->status == ThreadStatusRunning);
2032f1c5ccdim  rt->name = internal_strdup(tctx->name);
2042f7fa77ed  rt->parent_tid = tctx->parent_tid;
2058312957dim  rt->thread_type = tctx->thread_type;
2062f7fa77ed  rt->stack = 0;
2072f1c5ccdim  rt->stack = SymbolizeStackId(tctx->creation_stack_id);
2082f1c5ccdim  if (rt->stack)
2092f1c5ccdim    rt->stack->suppressable = suppressable;
210cfeab00andrew}
211cfeab00andrew
21224528c5dim#if !SANITIZER_GO
2130d16d37dimstatic bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) {
2140d16d37dim  int unique_id = *(int *)arg;
2150d16d37dim  return tctx->unique_id == (u32)unique_id;
2160d16d37dim}
2170d16d37dim
2182f7fa77edstatic ThreadContext *FindThreadByUidLocked(int unique_id) {
2192f7fa77ed  ctx->thread_registry->CheckLocked();
2200d16d37dim  return static_cast<ThreadContext *>(
2210d16d37dim      ctx->thread_registry->FindThreadContextLocked(
2220d16d37dim          FindThreadByUidLockedCallback, &unique_id));
223254a83bandrew}
224254a83bandrew
2252f7fa77edstatic ThreadContext *FindThreadByTidLocked(int tid) {
2262f7fa77ed  ctx->thread_registry->CheckLocked();
2272f7fa77ed  return static_cast<ThreadContext*>(
2282f7fa77ed      ctx->thread_registry->GetThreadLocked(tid));
2292f7fa77ed}
2302f7fa77ed
2312f7fa77edstatic bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) {
2322f7fa77ed  uptr addr = (uptr)arg;
2332f7fa77ed  ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
2342f7fa77ed  if (tctx->status != ThreadStatusRunning)
2352f7fa77ed    return false;
2362f7fa77ed  ThreadState *thr = tctx->thr;
2372f7fa77ed  CHECK(thr);
2382f7fa77ed  return ((addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size) ||
2392f7fa77ed          (addr >= thr->tls_addr && addr < thr->tls_addr + thr->tls_size));
2402f7fa77ed}
2412f7fa77ed
242254a83bandrewThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
2432f7fa77ed  ctx->thread_registry->CheckLocked();
2442f7fa77ed  ThreadContext *tctx = static_cast<ThreadContext*>(
2452f7fa77ed      ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls,
2462f7fa77ed                                                    (void*)addr));
2472f7fa77ed  if (!tctx)
2482f7fa77ed    return 0;
2492f7fa77ed  ThreadState *thr = tctx->thr;
2502f7fa77ed  CHECK(thr);
2512f7fa77ed  *is_stack = (addr >= thr->stk_addr && addr < thr->stk_addr + thr->stk_size);
2522f7fa77ed  return tctx;
253254a83bandrew}
254254a83bandrew#endif
255254a83bandrew
256a14b3e1dimvoid ScopedReportBase::AddThread(int unique_tid, bool suppressable) {
25724528c5dim#if !SANITIZER_GO
2585c652b1dim  if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
2595c652b1dim    AddThread(tctx, suppressable);
2602f1c5ccdim#endif
2612f1c5ccdim}
2622f1c5ccdim
263a14b3e1dimvoid ScopedReportBase::AddMutex(const SyncVar *s) {
264254a83bandrew  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
265254a83bandrew    if (rep_->mutexes[i]->id == s->uid)
266254a83bandrew      return;
267254a83bandrew  }
268cfeab00andrew  void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
2690d16d37dim  ReportMutex *rm = new(mem) ReportMutex;
270cfeab00andrew  rep_->mutexes.PushBack(rm);
271254a83bandrew  rm->id = s->uid;
2722f1c5ccdim  rm->addr = s->addr;
273254a83bandrew  rm->destroyed = false;
2742f1c5ccdim  rm->stack = SymbolizeStackId(s->creation_stack_id);
275cfeab00andrew}
276cfeab00andrew
277a14b3e1dimu64 ScopedReportBase::AddMutex(u64 id) {
2782f1c5ccdim  u64 uid = 0;
2792f1c5ccdim  u64 mid = id;
2802f1c5ccdim  uptr addr = SyncVar::SplitId(id, &uid);
281906f157dim  SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true);
2822f1c5ccdim  // Check that the mutex is still alive.
2832f1c5ccdim  // Another mutex can be created at the same address,
2842f1c5ccdim  // so check uid as well.
2852f1c5ccdim  if (s && s->CheckId(uid)) {
2862f1c5ccdim    mid = s->uid;
2872f1c5ccdim    AddMutex(s);
2882f1c5ccdim  } else {
2892f1c5ccdim    AddDeadMutex(id);
2902f1c5ccdim  }
2912f1c5ccdim  if (s)
2922f1c5ccdim    s->mtx.Unlock();
2932f1c5ccdim  return mid;
2942f1c5ccdim}
2952f1c5ccdim
296a14b3e1dimvoid ScopedReportBase::AddDeadMutex(u64 id) {
297254a83bandrew  for (uptr i = 0; i < rep_->mutexes.Size(); i++) {
298254a83bandrew    if (rep_->mutexes[i]->id == id)
299254a83bandrew      return;
300254a83bandrew  }
301254a83bandrew  void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex));
3020d16d37dim  ReportMutex *rm = new(mem) ReportMutex;
303254a83bandrew  rep_->mutexes.PushBack(rm);
304254a83bandrew  rm->id = id;
3052f1c5ccdim  rm->addr = 0;
306254a83bandrew  rm->destroyed = true;
307254a83bandrew  rm->stack = 0;
308254a83bandrew}
309254a83bandrew
310a14b3e1dimvoid ScopedReportBase::AddLocation(uptr addr, uptr size) {
311254a83bandrew  if (addr == 0)
312254a83bandrew    return;
31324528c5dim#if !SANITIZER_GO
314254a83bandrew  int fd = -1;
315118d2afdim  int creat_tid = kInvalidTid;
316254a83bandrew  u32 creat_stack = 0;
3172f1c5ccdim  if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
3182f1c5ccdim    ReportLocation *loc = ReportLocation::New(ReportLocationFD);
319254a83bandrew    loc->fd = fd;
320254a83bandrew    loc->tid = creat_tid;
3212f1c5ccdim    loc->stack = SymbolizeStackId(creat_stack);
3222f1c5ccdim    rep_->locs.PushBack(loc);
3232f7fa77ed    ThreadContext *tctx = FindThreadByUidLocked(creat_tid);
324254a83bandrew    if (tctx)
325254a83bandrew      AddThread(tctx);
326254a83bandrew    return;
327254a83bandrew  }
3282f7fa77ed  MBlock *b = 0;
3292f1c5ccdim  Allocator *a = allocator();
3302f1c5ccdim  if (a->PointerIsMine((void*)addr)) {
3312f1c5ccdim    void *block_begin = a->GetBlockBegin((void*)addr);
3322f1c5ccdim    if (block_begin)
3332f1c5ccdim      b = ctx->metamap.GetBlock((uptr)block_begin);
3342f1c5ccdim  }
3352f1c5ccdim  if (b != 0) {
3362f1c5ccdim    ThreadContext *tctx = FindThreadByTidLocked(b->tid);
3372f1c5ccdim    ReportLocation *loc = ReportLocation::New(ReportLocationHeap);
3382f1c5ccdim    loc->heap_chunk_start = (uptr)allocator()->GetBlockBegin((void *)addr);
3392f1c5ccdim    loc->heap_chunk_size = b->siz;
3406070792dim    loc->external_tag = b->tag;
3412f1c5ccdim    loc->tid = tctx ? tctx->tid : b->tid;
3422f1c5ccdim    loc->stack = SymbolizeStackId(b->stk);
343cfeab00andrew    rep_->locs.PushBack(loc);
344254a83bandrew    if (tctx)
345254a83bandrew      AddThread(tctx);
346254a83bandrew    return;
347cfeab00andrew  }
348254a83bandrew  bool is_stack = false;
349254a83bandrew  if (ThreadContext *tctx = IsThreadStackOrTls(addr, &is_stack)) {
3502f1c5ccdim    ReportLocation *loc =
3512f1c5ccdim        ReportLocation::New(is_stack ? ReportLocationStack : ReportLocationTLS);
352254a83bandrew    loc->tid = tctx->tid;
3532f1c5ccdim    rep_->locs.PushBack(loc);
354254a83bandrew    AddThread(tctx);
355254a83bandrew  }
356906f157dim#endif
3572f1c5ccdim  if (ReportLocation *loc = SymbolizeData(addr)) {
3582f1c5ccdim    loc->suppressable = true;
359254a83bandrew    rep_->locs.PushBack(loc);
360254a83bandrew    return;
361254a83bandrew  }
362cfeab00andrew}
363cfeab00andrew
36424528c5dim#if !SANITIZER_GO
365a14b3e1dimvoid ScopedReportBase::AddSleep(u32 stack_id) {
3662f1c5ccdim  rep_->sleep = SymbolizeStackId(stack_id);
367254a83bandrew}
368254a83bandrew#endif
369254a83bandrew
370a14b3e1dimvoid ScopedReportBase::SetCount(int count) { rep_->count = count; }
3712f7fa77ed
372a14b3e1dimconst ReportDesc *ScopedReportBase::GetReport() const { return rep_; }
373a14b3e1dim
374a14b3e1dimScopedReport::ScopedReport(ReportType typ, uptr tag)
375a14b3e1dim    : ScopedReportBase(typ, tag) {}
376a14b3e1dim
377a14b3e1dimScopedReport::~ScopedReport() {}
378cfeab00andrew
3792f1c5ccdimvoid RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
380635f333dim                  MutexSet *mset, uptr *tag) {
381254a83bandrew  // This function restores stack trace and mutex set for the thread/epoch.
382254a83bandrew  // It does so by getting stack trace and mutex set at the beginning of
383254a83bandrew  // trace part, and then replaying the trace till the given epoch.
3840d16d37dim  Trace* trace = ThreadTrace(tid);
3850d16d37dim  ReadLock l(&trace->mtx);
386254a83bandrew  const int partidx = (epoch / kTracePartSize) % TraceParts();
387cfeab00andrew  TraceHeader* hdr = &trace->headers[partidx];
3880d16d37dim  if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize)
389cfeab00andrew    return;
3900d16d37dim  CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0);
391254a83bandrew  const u64 epoch0 = RoundDown(epoch, TraceSize());
392254a83bandrew  const u64 eend = epoch % TraceSize();
393254a83bandrew  const u64 ebegin = RoundDown(eend, kTracePartSize);
394cfeab00andrew  DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n",
395cfeab00andrew          tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx);
396a14b3e1dim  Vector<uptr> stack;
3970d16d37dim  stack.Resize(hdr->stack0.size + 64);
3982f1c5ccdim  for (uptr i = 0; i < hdr->stack0.size; i++) {
3992f1c5ccdim    stack[i] = hdr->stack0.trace[i];
4005c652b1dim    DPrintf2("  #%02zu: pc=%zx\n", i, stack[i]);
401cfeab00andrew  }
402254a83bandrew  if (mset)
403254a83bandrew    *mset = hdr->mset0;
4042f1c5ccdim  uptr pos = hdr->stack0.size;
405254a83bandrew  Event *events = (Event*)GetThreadTrace(tid);
406cfeab00andrew  for (uptr i = ebegin; i <= eend; i++) {
407254a83bandrew    Event ev = events[i];
408a14b3e1dim    EventType typ = (EventType)(ev >> kEventPCBits);
409a14b3e1dim    uptr pc = (uptr)(ev & ((1ull << kEventPCBits) - 1));
410cfeab00andrew    DPrintf2("  %zu typ=%d pc=%zx\n", i, typ, pc);
411cfeab00andrew    if (typ == EventTypeMop) {
412cfeab00andrew      stack[pos] = pc;
413cfeab00andrew    } else if (typ == EventTypeFuncEnter) {
4140d16d37dim      if (stack.Size() < pos + 2)
4150d16d37dim        stack.Resize(pos + 2);
416cfeab00andrew      stack[pos++] = pc;
417cfeab00andrew    } else if (typ == EventTypeFuncExit) {
418cfeab00andrew      if (pos > 0)
419cfeab00andrew        pos--;
420cfeab00andrew    }
421254a83bandrew    if (mset) {
422254a83bandrew      if (typ == EventTypeLock) {
423254a83bandrew        mset->Add(pc, true, epoch0 + i);
424254a83bandrew      } else if (typ == EventTypeUnlock) {
425254a83bandrew        mset->Del(pc, true);
426254a83bandrew      } else if (typ == EventTypeRLock) {
427254a83bandrew        mset->Add(pc, false, epoch0 + i);
428254a83bandrew      } else if (typ == EventTypeRUnlock) {
429254a83bandrew        mset->Del(pc, false);
430254a83bandrew      }
431254a83bandrew    }
432cfeab00andrew    for (uptr j = 0; j <= pos; j++)
433cfeab00andrew      DPrintf2("      #%zu: %zx\n", j, stack[j]);
434cfeab00andrew  }
435cfeab00andrew  if (pos == 0 && stack[0] == 0)
436cfeab00andrew    return;
437cfeab00andrew  pos++;
4380d16d37dim  stk->Init(&stack[0], pos);
439635f333dim  ExtractTagFromStack(stk, tag);
440cfeab00andrew}
441cfeab00andrew
4422f1c5ccdimstatic bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2],
4432f1c5ccdim                             uptr addr_min, uptr addr_max) {
444cfeab00andrew  bool equal_stack = false;
445254a83bandrew  RacyStacks hash;
446cfeab00andrew  bool equal_address = false;
447cfeab00andrew  RacyAddress ra0 = {addr_min, addr_max};
4480d16d37dim  {
4490d16d37dim    ReadLock lock(&ctx->racy_mtx);
4500d16d37dim    if (flags()->suppress_equal_stacks) {
4510d16d37dim      hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr));
4520d16d37dim      hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr));
4530d16d37dim      for (uptr i = 0; i < ctx->racy_stacks.Size(); i++) {
4540d16d37dim        if (hash == ctx->racy_stacks[i]) {
4550d16d37dim          VPrintf(2,
4560d16d37dim              "ThreadSanitizer: suppressing report as doubled (stack)\n");
4570d16d37dim          equal_stack = true;
4580d16d37dim          break;
4590d16d37dim        }
4600d16d37dim      }
4610d16d37dim    }
4620d16d37dim    if (flags()->suppress_equal_addresses) {
4630d16d37dim      for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) {
4640d16d37dim        RacyAddress ra2 = ctx->racy_addresses[i];
4650d16d37dim        uptr maxbeg = max(ra0.addr_min, ra2.addr_min);
4660d16d37dim        uptr minend = min(ra0.addr_max, ra2.addr_max);
4670d16d37dim        if (maxbeg < minend) {
4680d16d37dim          VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n");
4690d16d37dim          equal_address = true;
4700d16d37dim          break;
4710d16d37dim        }
472cfeab00andrew      }
473cfeab00andrew    }
474cfeab00andrew  }
4750d16d37dim  if (!equal_stack && !equal_address)
4760d16d37dim    return false;
4770d16d37dim  if (!equal_stack) {
4780d16d37dim    Lock lock(&ctx->racy_mtx);
4790d16d37dim    ctx->racy_stacks.PushBack(hash);
480cfeab00andrew  }
4810d16d37dim  if (!equal_address) {
4820d16d37dim    Lock lock(&ctx->racy_mtx);
4830d16d37dim    ctx->racy_addresses.PushBack(ra0);
4840d16d37dim  }
4850d16d37dim  return true;
486cfeab00andrew}
487cfeab00andrew
4882f1c5ccdimstatic void AddRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2],
4892f1c5ccdim                          uptr addr_min, uptr addr_max) {
4900d16d37dim  Lock lock(&ctx->racy_mtx);
491cfeab00andrew  if (flags()->suppress_equal_stacks) {
492cfeab00andrew    RacyStacks hash;
4932f1c5ccdim    hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr));
4942f1c5ccdim    hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr));
495cfeab00andrew    ctx->racy_stacks.PushBack(hash);
496cfeab00andrew  }
497cfeab00andrew  if (flags()->suppress_equal_addresses) {
498cfeab00andrew    RacyAddress ra0 = {addr_min, addr_max};
499cfeab00andrew    ctx->racy_addresses.PushBack(ra0);
500cfeab00andrew  }
501cfeab00andrew}
502cfeab00andrew
5032f1c5ccdimbool OutputReport(ThreadState *thr, const ScopedReport &srep) {
5044ce3a8ddim  if (!flags()->report_bugs || thr->suppress_reports)
5050d16d37dim    return false;
5060d16d37dim  atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime());
507cfeab00andrew  const ReportDesc *rep = srep.GetReport();
508906f157dim  CHECK_EQ(thr->current_report, nullptr);
509906f157dim  thr->current_report = rep;
5102f7fa77ed  Suppression *supp = 0;
5110d16d37dim  uptr pc_or_addr = 0;
5120d16d37dim  for (uptr i = 0; pc_or_addr == 0 && i < rep->mops.Size(); i++)
5130d16d37dim    pc_or_addr = IsSuppressed(rep->typ, rep->mops[i]->stack, &supp);
5140d16d37dim  for (uptr i = 0; pc_or_addr == 0 && i < rep->stacks.Size(); i++)
5150d16d37dim    pc_or_addr = IsSuppressed(rep->typ, rep->stacks[i], &supp);
5160d16d37dim  for (uptr i = 0; pc_or_addr == 0 && i < rep->threads.Size(); i++)
5170d16d37dim    pc_or_addr = IsSuppressed(rep->typ, rep->threads[i]->stack, &supp);
5180d16d37dim  for (uptr i = 0; pc_or_addr == 0 && i < rep->locs.Size(); i++)
5190d16d37dim    pc_or_addr = IsSuppressed(rep->typ, rep->locs[i], &supp);
5200d16d37dim  if (pc_or_addr != 0) {
5210d16d37dim    Lock lock(&ctx->fired_suppressions_mtx);
5220d16d37dim    FiredSuppression s = {srep.GetReport()->typ, pc_or_addr, supp};
523c3f67e1dim    ctx->fired_suppressions.push_back(s);
524254a83bandrew  }
5252f1c5ccdim  {
5262f1c5ccdim    bool old_is_freeing = thr->is_freeing;
5272f1c5ccdim    thr->is_freeing = false;
5280d16d37dim    bool suppressed = OnReport(rep, pc_or_addr != 0);
5292f1c5ccdim    thr->is_freeing = old_is_freeing;
530906f157dim    if (suppressed) {
531906f157dim      thr->current_report = nullptr;
5322f1c5ccdim      return false;
533906f157dim    }
5342f1c5ccdim  }
535cfeab00andrew  PrintReport(rep);
536906f157dim  __tsan_on_report(rep);
537c3f67e1dim  ctx->nreported++;
538c3f67e1dim  if (flags()->halt_on_error)
5390d16d37dim    Die();
540906f157dim  thr->current_report = nullptr;
541cfeab00andrew  return true;
542cfeab00andrew}
543cfeab00andrew
5440d16d37dimbool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace) {
5450d16d37dim  ReadLock lock(&ctx->fired_suppressions_mtx);
546c3f67e1dim  for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) {
5470d16d37dim    if (ctx->fired_suppressions[k].type != type)
548254a83bandrew      continue;
5492f1c5ccdim    for (uptr j = 0; j < trace.size; j++) {
5502f7fa77ed      FiredSuppression *s = &ctx->fired_suppressions[k];
5510d16d37dim      if (trace.trace[j] == s->pc_or_addr) {
5522f7fa77ed        if (s->supp)
5530d16d37dim          atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed);
554254a83bandrew        return true;
5552f7fa77ed      }
556254a83bandrew    }
557254a83bandrew  }
558254a83bandrew  return false;
559254a83bandrew}
560254a83bandrew
5610d16d37dimstatic bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) {
5620d16d37dim  ReadLock lock(&ctx->fired_suppressions_mtx);
563c3f67e1dim  for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) {
5640d16d37dim    if (ctx->fired_suppressions[k].type != type)
565c3f67e1dim      continue;
566c3f67e1dim    FiredSuppression *s = &ctx->fired_suppressions[k];
5670d16d37dim    if (addr == s->pc_or_addr) {
568c3f67e1dim      if (s->supp)
5690d16d37dim        atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed);
570c3f67e1dim      return true;
571c3f67e1dim    }
572c3f67e1dim  }
573c3f67e1dim  return false;
574c3f67e1dim}
575c3f67e1dim
5762f7fa77edstatic bool RaceBetweenAtomicAndFree(ThreadState *thr) {
5772f7fa77ed  Shadow s0(thr->racy_state[0]);
5782f7fa77ed  Shadow s1(thr->racy_state[1]);
5792f7fa77ed  CHECK(!(s0.IsAtomic() && s1.IsAtomic()));
5802f7fa77ed  if (!s0.IsAtomic() && !s1.IsAtomic())
5812f7fa77ed    return true;
5822f7fa77ed  if (s0.IsAtomic() && s1.IsFreed())
5832f7fa77ed    return true;
5842f7fa77ed  if (s1.IsAtomic() && thr->is_freeing)
5852f7fa77ed    return true;
586254a83bandrew  return false;
587254a83bandrew}
588254a83bandrew
589cfeab00andrewvoid ReportRace(ThreadState *thr) {
5902f1c5ccdim  CheckNoLocks(thr);
5912f1c5ccdim
5922f1c5ccdim  // Symbolizer makes lots of intercepted calls. If we try to process them,
5932f1c5ccdim  // at best it will cause deadlocks on internal mutexes.
5942f1c5ccdim  ScopedIgnoreInterceptors ignore;
5952f1c5ccdim
596254a83bandrew  if (!flags()->report_bugs)
597254a83bandrew    return;
5982f7fa77ed  if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr))
5992f7fa77ed    return;
600254a83bandrew
601cfeab00andrew  bool freed = false;
602cfeab00andrew  {
603cfeab00andrew    Shadow s(thr->racy_state[1]);
604cfeab00andrew    freed = s.GetFreedAndReset();
605cfeab00andrew    thr->racy_state[1] = s.raw();
606cfeab00andrew  }
607cfeab00andrew
608cfeab00andrew  uptr addr = ShadowToMem((uptr)thr->racy_shadow_addr);
609cfeab00andrew  uptr addr_min = 0;
610cfeab00andrew  uptr addr_max = 0;
611cfeab00andrew  {
612cfeab00andrew    uptr a0 = addr + Shadow(thr->racy_state[0]).addr0();
613cfeab00andrew    uptr a1 = addr + Shadow(thr->racy_state[1]).addr0();
614cfeab00andrew    uptr e0 = a0 + Shadow(thr->racy_state[0]).size();
615cfeab00andrew    uptr e1 = a1 + Shadow(thr->racy_state[1]).size();
616cfeab00andrew    addr_min = min(a0, a1);
617cfeab00andrew    addr_max = max(e0, e1);
618cfeab00andrew    if (IsExpectedReport(addr_min, addr_max - addr_min))
619cfeab00andrew      return;
620cfeab00andrew  }
621cfeab00andrew
6222f7fa77ed  ReportType typ = ReportTypeRace;
6232f1c5ccdim  if (thr->is_vptr_access && freed)
6242f1c5ccdim    typ = ReportTypeVptrUseAfterFree;
6252f1c5ccdim  else if (thr->is_vptr_access)
6262f7fa77ed    typ = ReportTypeVptrRace;
6272f7fa77ed  else if (freed)
6282f7fa77ed    typ = ReportTypeUseAfterFree;
6290d16d37dim
6300d16d37dim  if (IsFiredSuppression(ctx, typ, addr))
631c3f67e1dim    return;
6320d16d37dim
633cfeab00andrew  const uptr kMop = 2;
6342f1c5ccdim  VarSizeStackTrace traces[kMop];
635635f333dim  uptr tags[kMop] = {kExternalTagNone};
636a14b3e1dim  uptr toppc = TraceTopPC(thr);
637a14b3e1dim  if (toppc >> kEventPCBits) {
638a14b3e1dim    // This is a work-around for a known issue.
639a14b3e1dim    // The scenario where this happens is rather elaborate and requires
640a14b3e1dim    // an instrumented __sanitizer_report_error_summary callback and
641a14b3e1dim    // a __tsan_symbolize_external callback and a race during a range memory
642a14b3e1dim    // access larger than 8 bytes. MemoryAccessRange adds the current PC to
643a14b3e1dim    // the trace and starts processing memory accesses. A first memory access
644a14b3e1dim    // triggers a race, we report it and call the instrumented
645a14b3e1dim    // __sanitizer_report_error_summary, which adds more stuff to the trace
646a14b3e1dim    // since it is intrumented. Then a second memory access in MemoryAccessRange
647a14b3e1dim    // also triggers a race and we get here and call TraceTopPC to get the
648a14b3e1dim    // current PC, however now it contains some unrelated events from the
649a14b3e1dim    // callback. Most likely, TraceTopPC will now return a EventTypeFuncExit
650a14b3e1dim    // event. Later we subtract -1 from it (in GetPreviousInstructionPc)
651a14b3e1dim    // and the resulting PC has kExternalPCBit set, so we pass it to
6529419380dim    // __tsan_symbolize_external_ex. __tsan_symbolize_external_ex is within its
6539419380dim    // rights to crash since the PC is completely bogus.
654391824ddim    // test/tsan/double_race.cpp contains a test case for this.
655a14b3e1dim    toppc = 0;
656a14b3e1dim  }
657635f333dim  ObtainCurrentStack(thr, toppc, &traces[0], &tags[0]);
6580d16d37dim  if (IsFiredSuppression(ctx, typ, traces[0]))
659254a83bandrew    return;
6600d16d37dim
6610d16d37dim  // MutexSet is too large to live on stack.
662a14b3e1dim  Vector<u64> mset_buffer;
6630d16d37dim  mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1);
6640d16d37dim  MutexSet *mset2 = new(&mset_buffer[0]) MutexSet();
6650d16d37dim
666254a83bandrew  Shadow s2(thr->racy_state[1]);
667635f333dim  RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2, &tags[1]);
6680d16d37dim  if (IsFiredSuppression(ctx, typ, traces[1]))
669c3f67e1dim    return;
670cfeab00andrew
671cfeab00andrew  if (HandleRacyStacks(thr, traces, addr_min, addr_max))
672cfeab00andrew    return;
673cfeab00andrew
6743c744a5dim  // If any of the accesses has a tag, treat this as an "external" race.
6753c744a5dim  uptr tag = kExternalTagNone;
6763c744a5dim  for (uptr i = 0; i < kMop; i++) {
6773c744a5dim    if (tags[i] != kExternalTagNone) {
6783c744a5dim      typ = ReportTypeExternalRace;
6793c744a5dim      tag = tags[i];
6803c744a5dim      break;
6813c744a5dim    }
6823c744a5dim  }
683635f333dim
6840d16d37dim  ThreadRegistryLock l0(ctx->thread_registry);
6853c744a5dim  ScopedReport rep(typ, tag);
686cfeab00andrew  for (uptr i = 0; i < kMop; i++) {
687cfeab00andrew    Shadow s(thr->racy_state[i]);
688635f333dim    rep.AddMemoryAccess(addr, tags[i], s, traces[i],
6896070792dim                        i == 0 ? &thr->mset : mset2);
690cfeab00andrew  }
691cfeab00andrew
692cfeab00andrew  for (uptr i = 0; i < kMop; i++) {
693cfeab00andrew    FastState s(thr->racy_state[i]);
6942f7fa77ed    ThreadContext *tctx = static_cast<ThreadContext*>(
6952f7fa77ed        ctx->thread_registry->GetThreadLocked(s.tid()));
696cfeab00andrew    if (s.epoch() < tctx->epoch0 || s.epoch() > tctx->epoch1)
697cfeab00andrew      continue;
698cfeab00andrew    rep.AddThread(tctx);
699cfeab00andrew  }
700cfeab00andrew
701254a83bandrew  rep.AddLocation(addr_min, addr_max - addr_min);
702254a83bandrew
70324528c5dim#if !SANITIZER_GO
704391824ddim  {
705254a83bandrew    Shadow s(thr->racy_state[1]);
706254a83bandrew    if (s.epoch() <= thr->last_sleep_clock.get(s.tid()))
707254a83bandrew      rep.AddSleep(thr->last_sleep_stack_id);
708254a83bandrew  }
709254a83bandrew#endif
710254a83bandrew
7112f1c5ccdim  if (!OutputReport(thr, rep))
712cfeab00andrew    return;
713cfeab00andrew
714cfeab00andrew  AddRacyStacks(thr, traces, addr_min, addr_max);
715cfeab00andrew}
716cfeab00andrew
717254a83bandrewvoid PrintCurrentStack(ThreadState *thr, uptr pc) {
7182f1c5ccdim  VarSizeStackTrace trace;
7192f1c5ccdim  ObtainCurrentStack(thr, pc, &trace);
720254a83bandrew  PrintStack(SymbolizeStack(trace));
721254a83bandrew}
722254a83bandrew
723906f157dim// Always inlining PrintCurrentStackSlow, because LocatePcInTrace assumes
724906f157dim// __sanitizer_print_stack_trace exists in the actual unwinded stack, but
725906f157dim// tail-call to PrintCurrentStackSlow breaks this assumption because
726906f157dim// __sanitizer_print_stack_trace disappears after tail-call.
727906f157dim// However, this solution is not reliable enough, please see dvyukov's comment
728906f157dim// http://reviews.llvm.org/D19148#406208
729906f157dim// Also see PR27280 comment 2 and 3 for breaking examples and analysis.
730906f157dimALWAYS_INLINE
7312f1c5ccdimvoid PrintCurrentStackSlow(uptr pc) {
73224528c5dim#if !SANITIZER_GO
7338312957dim  uptr bp = GET_CURRENT_FRAME();
7342f1c5ccdim  BufferedStackTrace *ptrace =
7352f1c5ccdim      new(internal_alloc(MBlockStackTrace, sizeof(BufferedStackTrace)))
7362f1c5ccdim          BufferedStackTrace();
7378312957dim  ptrace->Unwind(pc, bp, nullptr, false);
7388312957dim
7392f7fa77ed  for (uptr i = 0; i < ptrace->size / 2; i++) {
7402f1c5ccdim    uptr tmp = ptrace->trace_buffer[i];
7412f1c5ccdim    ptrace->trace_buffer[i] = ptrace->trace_buffer[ptrace->size - i - 1];
7422f1c5ccdim    ptrace->trace_buffer[ptrace->size - i - 1] = tmp;
7432f7fa77ed  }
7442f1c5ccdim  PrintStack(SymbolizeStack(*ptrace));
7452f7fa77ed#endif
7462f7fa77ed}
7472f7fa77ed
748cfeab00andrew}  // namespace __tsan
7492f1c5ccdim
7502f1c5ccdimusing namespace __tsan;
7512f1c5ccdim
7522f1c5ccdimextern "C" {
7532f1c5ccdimSANITIZER_INTERFACE_ATTRIBUTE
7542f1c5ccdimvoid __sanitizer_print_stack_trace() {
7552f1c5ccdim  PrintCurrentStackSlow(StackTrace::GetCurrentPc());
7562f1c5ccdim}
7572f1c5ccdim}  // extern "C"
758