1/*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\
2|*
3|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4|* See https://llvm.org/LICENSE.txt for license information.
5|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6|*
7|*===----------------------------------------------------------------------===*|
8|*
9|* This file implements the call back routines for the gcov profiling
10|* instrumentation pass. Link against this library when running code through
11|* the -insert-gcov-profiling LLVM pass.
12|*
13|* We emit files in a corrupt version of GCOV's "gcda" file format. These files
14|* are only close enough that LCOV will happily parse them. Anything that lcov
15|* ignores is missing.
16|*
17|* TODO: gcov is multi-process safe by having each exit open the existing file
18|* and append to it. We'd like to achieve that and be thread-safe too.
19|*
20\*===----------------------------------------------------------------------===*/
21
22#if !defined(__Fuchsia__)
23
24#include <errno.h>
25#include <fcntl.h>
26#include <stdio.h>
27#include <stdlib.h>
28#include <string.h>
29
30#if defined(_WIN32)
31#define WIN32_LEAN_AND_MEAN
32#include <windows.h>
33#include "WindowsMMap.h"
34#else
35#include <sys/file.h>
36#include <sys/mman.h>
37#include <sys/types.h>
38#include <unistd.h>
39#endif
40
41#if defined(__FreeBSD__) && defined(__i386__)
42#define I386_FREEBSD 1
43#else
44#define I386_FREEBSD 0
45#endif
46
47#if !defined(_MSC_VER) && !I386_FREEBSD
48#include <stdint.h>
49#endif
50
51#if defined(_MSC_VER)
52typedef unsigned char uint8_t;
53typedef unsigned int uint32_t;
54typedef unsigned long long uint64_t;
55#elif I386_FREEBSD
56/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to
57 * FreeBSD 10, r232261) when compiled in 32-bit mode.
58 */
59typedef unsigned char uint8_t;
60typedef unsigned int uint32_t;
61typedef unsigned long long uint64_t;
62#endif
63
64#include "InstrProfiling.h"
65#include "InstrProfilingUtil.h"
66
67/* #define DEBUG_GCDAPROFILING */
68
69enum {
70  GCOV_DATA_MAGIC = 0x67636461, // "gcda"
71
72  GCOV_TAG_FUNCTION = 0x01000000,
73  GCOV_TAG_COUNTER_ARCS = 0x01a10000,
74  // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9.
75  GCOV_TAG_OBJECT_SUMMARY = 0xa1000000,
76  GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000,
77};
78
79/*
80 * --- GCOV file format I/O primitives ---
81 */
82
83/*
84 * The current file name we're outputting. Used primarily for error logging.
85 */
86static char *filename = NULL;
87
88/*
89 * The current file we're outputting.
90 */
91static FILE *output_file = NULL;
92
93/*
94 * Buffer that we write things into.
95 */
96#define WRITE_BUFFER_SIZE (128 * 1024)
97static unsigned char *write_buffer = NULL;
98static uint64_t cur_buffer_size = 0;
99static uint64_t cur_pos = 0;
100static uint64_t file_size = 0;
101static int new_file = 0;
102static int gcov_version;
103#if defined(_WIN32)
104static HANDLE mmap_handle = NULL;
105#endif
106static int fd = -1;
107
108typedef void (*fn_ptr)();
109
110typedef void* dynamic_object_id;
111// The address of this variable identifies a given dynamic object.
112static dynamic_object_id current_id;
113#define CURRENT_ID (&current_id)
114
115struct fn_node {
116  dynamic_object_id id;
117  fn_ptr fn;
118  struct fn_node* next;
119};
120
121struct fn_list {
122  struct fn_node *head, *tail;
123};
124
125/*
126 * A list of functions to write out the data, shared between all dynamic objects.
127 */
128struct fn_list writeout_fn_list;
129
130/*
131 *  A list of flush functions that our __gcov_flush() function should call, shared between all dynamic objects.
132 */
133struct fn_list flush_fn_list;
134
135/*
136 *  A list of reset functions, shared between all dynamic objects.
137 */
138struct fn_list reset_fn_list;
139
140static void fn_list_insert(struct fn_list* list, fn_ptr fn) {
141  struct fn_node* new_node = malloc(sizeof(struct fn_node));
142  new_node->fn = fn;
143  new_node->next = NULL;
144  new_node->id = CURRENT_ID;
145
146  if (!list->head) {
147    list->head = list->tail = new_node;
148  } else {
149    list->tail->next = new_node;
150    list->tail = new_node;
151  }
152}
153
154static void fn_list_remove(struct fn_list* list) {
155  struct fn_node* curr = list->head;
156  struct fn_node* prev = NULL;
157  struct fn_node* next = NULL;
158
159  while (curr) {
160    next = curr->next;
161
162    if (curr->id == CURRENT_ID) {
163      if (curr == list->head) {
164        list->head = next;
165      }
166
167      if (curr == list->tail) {
168        list->tail = prev;
169      }
170
171      if (prev) {
172        prev->next = next;
173      }
174
175      free(curr);
176    } else {
177      prev = curr;
178    }
179
180    curr = next;
181  }
182}
183
184static void resize_write_buffer(uint64_t size) {
185  if (!new_file) return;
186  size += cur_pos;
187  if (size <= cur_buffer_size) return;
188  size = (size - 1) / WRITE_BUFFER_SIZE + 1;
189  size *= WRITE_BUFFER_SIZE;
190  write_buffer = realloc(write_buffer, size);
191  cur_buffer_size = size;
192}
193
194static void write_bytes(const char *s, size_t len) {
195  resize_write_buffer(len);
196  memcpy(&write_buffer[cur_pos], s, len);
197  cur_pos += len;
198}
199
200static void write_32bit_value(uint32_t i) {
201  write_bytes((char*)&i, 4);
202}
203
204static void write_64bit_value(uint64_t i) {
205  // GCOV uses a lo-/hi-word format even on big-endian systems.
206  // See also GCOVBuffer::readInt64 in LLVM.
207  uint32_t lo = (uint32_t) i;
208  uint32_t hi = (uint32_t) (i >> 32);
209  write_32bit_value(lo);
210  write_32bit_value(hi);
211}
212
213static uint32_t read_32bit_value() {
214  uint32_t val;
215
216  if (new_file)
217    return (uint32_t)-1;
218
219  val = *(uint32_t*)&write_buffer[cur_pos];
220  cur_pos += 4;
221  return val;
222}
223
224static uint64_t read_64bit_value() {
225  // GCOV uses a lo-/hi-word format even on big-endian systems.
226  // See also GCOVBuffer::readInt64 in LLVM.
227  uint32_t lo = read_32bit_value();
228  uint32_t hi = read_32bit_value();
229  return ((uint64_t)hi << 32) | ((uint64_t)lo);
230}
231
232static char *mangle_filename(const char *orig_filename) {
233  char *new_filename;
234  size_t prefix_len;
235  int prefix_strip;
236  const char *prefix = lprofGetPathPrefix(&prefix_strip, &prefix_len);
237
238  if (prefix == NULL)
239    return strdup(orig_filename);
240
241  new_filename = malloc(prefix_len + 1 + strlen(orig_filename) + 1);
242  lprofApplyPathPrefix(new_filename, orig_filename, prefix, prefix_len,
243                       prefix_strip);
244
245  return new_filename;
246}
247
248static int map_file() {
249  fseek(output_file, 0L, SEEK_END);
250  file_size = ftell(output_file);
251
252  /* A size of 0 means the file has been created just now (possibly by another
253   * process in lock-after-open race condition). No need to mmap. */
254  if (file_size == 0)
255    return -1;
256
257#if defined(_WIN32)
258  HANDLE mmap_fd;
259  if (fd == -1)
260    mmap_fd = INVALID_HANDLE_VALUE;
261  else
262    mmap_fd = (HANDLE)_get_osfhandle(fd);
263
264  mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL);
265  if (mmap_handle == NULL) {
266    fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n",
267            filename, GetLastError());
268    return -1;
269  }
270
271  write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size);
272  if (write_buffer == NULL) {
273    fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename,
274            GetLastError());
275    CloseHandle(mmap_handle);
276    return -1;
277  }
278#else
279  write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE,
280                      MAP_FILE | MAP_SHARED, fd, 0);
281  if (write_buffer == (void *)-1) {
282    int errnum = errno;
283    fprintf(stderr, "profiling: %s: cannot map: %s\n", filename,
284            strerror(errnum));
285    return -1;
286  }
287#endif
288
289  return 0;
290}
291
292static void unmap_file() {
293#if defined(_WIN32)
294  if (!FlushViewOfFile(write_buffer, file_size)) {
295    fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename,
296            GetLastError());
297  }
298
299  if (!UnmapViewOfFile(write_buffer)) {
300    fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename,
301            GetLastError());
302  }
303
304  if (!CloseHandle(mmap_handle)) {
305    fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n",
306            filename, GetLastError());
307  }
308
309  mmap_handle = NULL;
310#else
311  if (msync(write_buffer, file_size, MS_SYNC) == -1) {
312    int errnum = errno;
313    fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename,
314            strerror(errnum));
315  }
316
317  /* We explicitly ignore errors from unmapping because at this point the data
318   * is written and we don't care.
319   */
320  (void)munmap(write_buffer, file_size);
321#endif
322
323  write_buffer = NULL;
324  file_size = 0;
325}
326
327/*
328 * --- LLVM line counter API ---
329 */
330
331/* A file in this case is a translation unit. Each .o file built with line
332 * profiling enabled will emit to a different file. Only one file may be
333 * started at a time.
334 */
335COMPILER_RT_VISIBILITY
336void llvm_gcda_start_file(const char *orig_filename, uint32_t version,
337                          uint32_t checksum) {
338  const char *mode = "r+b";
339  filename = mangle_filename(orig_filename);
340
341  /* Try just opening the file. */
342  fd = open(filename, O_RDWR | O_BINARY);
343
344  if (fd == -1) {
345    /* Try creating the file. */
346    fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
347    if (fd != -1) {
348      mode = "w+b";
349    } else {
350      /* Try creating the directories first then opening the file. */
351      __llvm_profile_recursive_mkdir(filename);
352      fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
353      if (fd != -1) {
354        mode = "w+b";
355      } else {
356        /* Another process may have created the file just now.
357         * Try opening it without O_CREAT and O_EXCL. */
358        fd = open(filename, O_RDWR | O_BINARY);
359        if (fd == -1) {
360          /* Bah! It's hopeless. */
361          int errnum = errno;
362          fprintf(stderr, "profiling: %s: cannot open: %s\n", filename,
363                  strerror(errnum));
364          return;
365        }
366      }
367    }
368  }
369
370  /* Try to flock the file to serialize concurrent processes writing out to the
371   * same GCDA. This can fail if the filesystem doesn't support it, but in that
372   * case we'll just carry on with the old racy behaviour and hope for the best.
373   */
374  lprofLockFd(fd);
375  output_file = fdopen(fd, mode);
376
377  /* Initialize the write buffer. */
378  new_file = 0;
379  write_buffer = NULL;
380  cur_buffer_size = 0;
381  cur_pos = 0;
382
383  if (map_file() == -1) {
384    /* The file has been created just now (file_size == 0) or mmap failed
385     * unexpectedly. In the latter case, try to recover by clobbering. */
386    new_file = 1;
387    write_buffer = NULL;
388    resize_write_buffer(WRITE_BUFFER_SIZE);
389    memset(write_buffer, 0, WRITE_BUFFER_SIZE);
390  }
391
392  /* gcda file, version, stamp checksum. */
393  {
394    uint8_t c3 = version >> 24;
395    uint8_t c2 = (version >> 16) & 255;
396    uint8_t c1 = (version >> 8) & 255;
397    gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0'
398                             : (c3 - '0') * 10 + c1 - '0';
399  }
400  write_32bit_value(GCOV_DATA_MAGIC);
401  write_32bit_value(version);
402  write_32bit_value(checksum);
403
404#ifdef DEBUG_GCDAPROFILING
405  fprintf(stderr, "llvmgcda: [%s]\n", orig_filename);
406#endif
407}
408
409/* Given an array of pointers to counters (counters), increment the n-th one,
410 * where we're also given a pointer to n (predecessor).
411 */
412COMPILER_RT_VISIBILITY
413void llvm_gcda_increment_indirect_counter(uint32_t *predecessor,
414                                          uint64_t **counters) {
415  uint64_t *counter;
416  uint32_t pred;
417
418  pred = *predecessor;
419  if (pred == 0xffffffff)
420    return;
421  counter = counters[pred];
422
423  /* Don't crash if the pred# is out of sync. This can happen due to threads,
424     or because of a TODO in GCOVProfiling.cpp buildEdgeLookupTable(). */
425  if (counter)
426    ++*counter;
427#ifdef DEBUG_GCDAPROFILING
428  else
429    fprintf(stderr,
430            "llvmgcda: increment_indirect_counter counters=%08llx, pred=%u\n",
431            *counter, *predecessor);
432#endif
433}
434
435COMPILER_RT_VISIBILITY
436void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum,
437                             uint32_t cfg_checksum) {
438  uint32_t len = 2;
439  int use_extra_checksum = gcov_version >= 47;
440
441  if (use_extra_checksum)
442    len++;
443#ifdef DEBUG_GCDAPROFILING
444  fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident);
445#endif
446  if (!output_file) return;
447
448  /* function tag */
449  write_32bit_value(GCOV_TAG_FUNCTION);
450  write_32bit_value(len);
451  write_32bit_value(ident);
452  write_32bit_value(func_checksum);
453  if (use_extra_checksum)
454    write_32bit_value(cfg_checksum);
455}
456
457COMPILER_RT_VISIBILITY
458void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
459  uint32_t i;
460  uint64_t *old_ctrs = NULL;
461  uint32_t val = 0;
462  uint64_t save_cur_pos = cur_pos;
463
464  if (!output_file) return;
465
466  val = read_32bit_value();
467
468  if (val != (uint32_t)-1) {
469    /* There are counters present in the file. Merge them. */
470    if (val != GCOV_TAG_COUNTER_ARCS) {
471      fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
472                      "corrupt arc tag (0x%08x)\n",
473              filename, val);
474      return;
475    }
476
477    val = read_32bit_value();
478    if (val == (uint32_t)-1 || val / 2 != num_counters) {
479      fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
480                      "mismatched number of counters (%d)\n",
481              filename, val);
482      return;
483    }
484
485    old_ctrs = malloc(sizeof(uint64_t) * num_counters);
486    for (i = 0; i < num_counters; ++i)
487      old_ctrs[i] = read_64bit_value();
488  }
489
490  cur_pos = save_cur_pos;
491
492  /* Counter #1 (arcs) tag */
493  write_32bit_value(GCOV_TAG_COUNTER_ARCS);
494  write_32bit_value(num_counters * 2);
495  for (i = 0; i < num_counters; ++i) {
496    counters[i] += (old_ctrs ? old_ctrs[i] : 0);
497    write_64bit_value(counters[i]);
498  }
499
500  free(old_ctrs);
501
502#ifdef DEBUG_GCDAPROFILING
503  fprintf(stderr, "llvmgcda:   %u arcs\n", num_counters);
504  for (i = 0; i < num_counters; ++i)
505    fprintf(stderr, "llvmgcda:   %llu\n", (unsigned long long)counters[i]);
506#endif
507}
508
509COMPILER_RT_VISIBILITY
510void llvm_gcda_summary_info() {
511  uint32_t runs = 1;
512  static uint32_t run_counted = 0; // We only want to increase the run count once.
513  uint32_t val = 0;
514  uint64_t save_cur_pos = cur_pos;
515
516  if (!output_file) return;
517
518  val = read_32bit_value();
519
520  if (val != (uint32_t)-1) {
521    /* There are counters present in the file. Merge them. */
522    if (val != (gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY
523                                   : GCOV_TAG_PROGRAM_SUMMARY)) {
524      fprintf(stderr,
525              "profiling: %s: cannot merge previous run count: "
526              "corrupt object tag (0x%08x)\n",
527              filename, val);
528      return;
529    }
530
531    val = read_32bit_value(); /* length */
532    uint32_t prev_runs;
533    if (gcov_version < 90) {
534      read_32bit_value();
535      read_32bit_value();
536      prev_runs = read_32bit_value();
537    } else {
538      prev_runs = read_32bit_value();
539      read_32bit_value();
540    }
541    for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i)
542      read_32bit_value();
543    /* Add previous run count to new counter, if not already counted before. */
544    runs = run_counted ? prev_runs : prev_runs + 1;
545  }
546
547  cur_pos = save_cur_pos;
548
549  if (gcov_version >= 90) {
550    write_32bit_value(GCOV_TAG_OBJECT_SUMMARY);
551    write_32bit_value(2);
552    write_32bit_value(runs);
553    write_32bit_value(0); // sum_max
554  } else {
555    // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set
556    // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which
557    // can make gcov read "Runs:".
558    write_32bit_value(GCOV_TAG_PROGRAM_SUMMARY);
559    write_32bit_value(3);
560    write_32bit_value(0);
561    write_32bit_value(0);
562    write_32bit_value(runs);
563  }
564
565  run_counted = 1;
566
567#ifdef DEBUG_GCDAPROFILING
568  fprintf(stderr, "llvmgcda:   %u runs\n", runs);
569#endif
570}
571
572COMPILER_RT_VISIBILITY
573void llvm_gcda_end_file() {
574  /* Write out EOF record. */
575  if (output_file) {
576    write_bytes("\0\0\0\0\0\0\0\0", 8);
577
578    if (new_file) {
579      fwrite(write_buffer, cur_pos, 1, output_file);
580      free(write_buffer);
581    } else {
582      unmap_file();
583    }
584
585    fflush(output_file);
586    lprofUnlockFd(fd);
587    fclose(output_file);
588    output_file = NULL;
589    write_buffer = NULL;
590  }
591  free(filename);
592
593#ifdef DEBUG_GCDAPROFILING
594  fprintf(stderr, "llvmgcda: -----\n");
595#endif
596}
597
598COMPILER_RT_VISIBILITY
599void llvm_register_writeout_function(fn_ptr fn) {
600  fn_list_insert(&writeout_fn_list, fn);
601}
602
603COMPILER_RT_VISIBILITY
604void llvm_writeout_files(void) {
605  struct fn_node *curr = writeout_fn_list.head;
606
607  while (curr) {
608    if (curr->id == CURRENT_ID) {
609      curr->fn();
610    }
611    curr = curr->next;
612  }
613}
614
615#ifndef _WIN32
616// __attribute__((destructor)) and destructors whose priorities are greater than
617// 100 run before this function and can thus be tracked. The priority is
618// compatible with GCC 7 onwards.
619#if __GNUC__ >= 9
620#pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
621#endif
622__attribute__((destructor(100)))
623#endif
624static void llvm_writeout_and_clear(void) {
625  llvm_writeout_files();
626  fn_list_remove(&writeout_fn_list);
627}
628
629COMPILER_RT_VISIBILITY
630void llvm_register_flush_function(fn_ptr fn) {
631  fn_list_insert(&flush_fn_list, fn);
632}
633
634void __gcov_flush() {
635  struct fn_node* curr = flush_fn_list.head;
636
637  while (curr) {
638    curr->fn();
639    curr = curr->next;
640  }
641}
642
643COMPILER_RT_VISIBILITY
644void llvm_delete_flush_function_list(void) {
645  fn_list_remove(&flush_fn_list);
646}
647
648COMPILER_RT_VISIBILITY
649void llvm_register_reset_function(fn_ptr fn) {
650  fn_list_insert(&reset_fn_list, fn);
651}
652
653COMPILER_RT_VISIBILITY
654void llvm_delete_reset_function_list(void) { fn_list_remove(&reset_fn_list); }
655
656COMPILER_RT_VISIBILITY
657void llvm_reset_counters(void) {
658  struct fn_node *curr = reset_fn_list.head;
659
660  while (curr) {
661    if (curr->id == CURRENT_ID) {
662      curr->fn();
663    }
664    curr = curr->next;
665  }
666}
667
668#if !defined(_WIN32)
669COMPILER_RT_VISIBILITY
670pid_t __gcov_fork() {
671  pid_t parent_pid = getpid();
672  pid_t pid = fork();
673
674  if (pid == 0) {
675    pid_t child_pid = getpid();
676    if (child_pid != parent_pid) {
677      // The pid changed so we've a fork (one could have its own fork function)
678      // Just reset the counters for this child process
679      // threads.
680      llvm_reset_counters();
681    }
682  }
683  return pid;
684}
685#endif
686
687COMPILER_RT_VISIBILITY
688void llvm_gcov_init(fn_ptr wfn, fn_ptr ffn, fn_ptr rfn) {
689  static int atexit_ran = 0;
690
691  if (wfn)
692    llvm_register_writeout_function(wfn);
693
694  if (ffn)
695    llvm_register_flush_function(ffn);
696
697  if (rfn)
698    llvm_register_reset_function(rfn);
699
700  if (atexit_ran == 0) {
701    atexit_ran = 1;
702
703    /* Make sure we write out the data and delete the data structures. */
704    atexit(llvm_delete_reset_function_list);
705    atexit(llvm_delete_flush_function_list);
706#ifdef _WIN32
707    atexit(llvm_writeout_and_clear);
708#endif
709  }
710}
711
712#endif
713