InstrProfilingMerge.c

/*===- InstrProfilingMerge.c - Profile in-process Merging  ---------------===*\
|*
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|* See https://llvm.org/LICENSE.txt for license information.
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|*
|*===----------------------------------------------------------------------===*
|* This file defines the API needed for in-process merging of profile data
|* stored in memory buffer.
\*===---------------------------------------------------------------------===*/

#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
#include "InstrProfilingUtil.h"

#define INSTR_PROF_VALUE_PROF_DATA
#include "profile/InstrProfData.inc"

COMPILER_RT_VISIBILITY
void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);

COMPILER_RT_VISIBILITY
uint64_t lprofGetLoadModuleSignature(void) {
  /* A very fast way to compute a module signature.  */
  uint64_t Version = __llvm_profile_get_version();
  uint64_t NumCounters = __llvm_profile_get_num_counters(
      __llvm_profile_begin_counters(), __llvm_profile_end_counters());
  uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
                                                 __llvm_profile_end_data());
  uint64_t NamesSize =
      (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
  uint64_t NumVnodes =
      (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
  const __llvm_profile_data *FirstD = __llvm_profile_begin_data();

  return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
         (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
         __llvm_profile_get_magic();
}

#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wcast-qual"
#elif defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wcast-qual"
#endif

/* Returns 1 if profile is not structurally compatible.  */
COMPILER_RT_VISIBILITY
int __llvm_profile_check_compatibility(const char *ProfileData,
                                       uint64_t ProfileSize) {
  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
  SrcDataStart =
      (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
                              Header->BinaryIdsSize);
  SrcDataEnd = SrcDataStart + Header->NumData;

  if (ProfileSize < sizeof(__llvm_profile_header))
    return 1;

  /* Check the header first.  */
  if (Header->Magic != __llvm_profile_get_magic() ||
      Header->Version != __llvm_profile_get_version() ||
      Header->NumData !=
          __llvm_profile_get_num_data(__llvm_profile_begin_data(),
                                      __llvm_profile_end_data()) ||
      Header->NumCounters !=
          __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
                                          __llvm_profile_end_counters()) ||
      Header->NumBitmapBytes !=
          __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
                                              __llvm_profile_end_bitmap()) ||
      Header->NamesSize !=
          __llvm_profile_get_name_size(__llvm_profile_begin_names(),
                                       __llvm_profile_end_names()) ||
      Header->ValueKindLast != IPVK_Last)
    return 1;

  if (ProfileSize <
      sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
          Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
          Header->NumCounters * __llvm_profile_counter_entry_size() +
          Header->NumBitmapBytes)
    return 1;

  for (SrcData = SrcDataStart,
       DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
       SrcData < SrcDataEnd; ++SrcData, ++DstData) {
    if (SrcData->NameRef != DstData->NameRef ||
        SrcData->FuncHash != DstData->FuncHash ||
        SrcData->NumCounters != DstData->NumCounters ||
        SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
      return 1;
  }

  /* Matched! */
  return 0;
}

static uintptr_t signextIfWin64(void *V) {
#ifdef _WIN64
  return (uintptr_t)(int32_t)(uintptr_t)V;
#else
  return (uintptr_t)V;
#endif
}

// Skip names section, vtable profile data section and vtable names section
// for runtime profile merge. To merge runtime addresses from multiple
// profiles collected from the same instrumented binary, the binary should be
// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
// disabled). In this set-up these three sections remain unchanged.
static uint64_t
getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
  const uint64_t VTableSectionSize =
      Header->NumVTables * sizeof(VTableProfData);
  const uint64_t PaddingBytesAfterVTableSection =
      __llvm_profile_get_num_padding_bytes(VTableSectionSize);
  const uint64_t VNamesSize = Header->VNamesSize;
  const uint64_t PaddingBytesAfterVNamesSize =
      __llvm_profile_get_num_padding_bytes(VNamesSize);
  return Header->NamesSize +
         __llvm_profile_get_num_padding_bytes(Header->NamesSize) +
         VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
         PaddingBytesAfterVNamesSize;
}

COMPILER_RT_VISIBILITY
int __llvm_profile_merge_from_buffer(const char *ProfileData,
                                     uint64_t ProfileSize) {
  if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
    PROF_ERR("%s\n",
             "Temporal profiles do not support profile merging at runtime. "
             "Instead, merge raw profiles using the llvm-profdata tool.");
    return 1;
  }

  __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
  __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
  char *SrcCountersStart, *DstCounter;
  const char *SrcCountersEnd, *SrcCounter;
  const char *SrcBitmapStart;
  const char *SrcNameStart;
  const char *SrcValueProfDataStart, *SrcValueProfData;
  uintptr_t CountersDelta = Header->CountersDelta;
  uintptr_t BitmapDelta = Header->BitmapDelta;

  SrcDataStart =
      (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
                              Header->BinaryIdsSize);
  SrcDataEnd = SrcDataStart + Header->NumData;
  SrcCountersStart = (char *)SrcDataEnd;
  SrcCountersEnd = SrcCountersStart +
                   Header->NumCounters * __llvm_profile_counter_entry_size();
  SrcBitmapStart = SrcCountersEnd;
  SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
  SrcValueProfDataStart =
      SrcNameStart + getDistanceFromCounterToValueProf(Header);
  if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
    return 1;

  // Merge counters by iterating the entire counter section when data section is
  // empty due to correlation.
  if (Header->NumData == 0) {
    for (SrcCounter = SrcCountersStart,
        DstCounter = __llvm_profile_begin_counters();
         SrcCounter < SrcCountersEnd;) {
      if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
        *DstCounter &= *SrcCounter;
      } else {
        *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
      }
      SrcCounter += __llvm_profile_counter_entry_size();
      DstCounter += __llvm_profile_counter_entry_size();
    }
    return 0;
  }

  for (SrcData = SrcDataStart,
      DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
      SrcValueProfData = SrcValueProfDataStart;
       SrcData < SrcDataEnd; ++SrcData, ++DstData) {
    // For the in-memory destination, CounterPtr is the distance from the start
    // address of the data to the start address of the counter. On WIN64,
    // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
    // extend CounterPtr to get the original value.
    char *DstCounters =
        (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
    char *DstBitmap =
        (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr));
    unsigned NVK = 0;

    // SrcData is a serialized representation of the memory image. We need to
    // compute the in-buffer counter offset from the in-memory address distance.
    // The initial CountersDelta is the in-memory address difference
    // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
    // CountersDelta computes the offset into the in-buffer counter section.
    //
    // On WIN64, CountersDelta is truncated as well, so no need for signext.
    char *SrcCounters =
        SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
    // CountersDelta needs to be decreased as we advance to the next data
    // record.
    CountersDelta -= sizeof(*SrcData);
    unsigned NC = SrcData->NumCounters;
    if (NC == 0)
      return 1;
    if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
        (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
      return 1;
    for (unsigned I = 0; I < NC; I++) {
      if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
        // A value of zero signifies the function is covered.
        DstCounters[I] &= SrcCounters[I];
      } else {
        ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
      }
    }

    const char *SrcBitmap =
        SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
    // BitmapDelta also needs to be decreased as we advance to the next data
    // record.
    BitmapDelta -= sizeof(*SrcData);
    unsigned NB = SrcData->NumBitmapBytes;
    // NumBitmapBytes may legitimately be 0. Just keep going.
    if (NB != 0) {
      if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
        return 1;
      // Merge Src and Dst Bitmap bytes by simply ORing them together.
      for (unsigned I = 0; I < NB; I++)
        DstBitmap[I] |= SrcBitmap[I];
    }

    /* Now merge value profile data. */
    if (!VPMergeHook)
      continue;

    for (unsigned I = 0; I <= IPVK_Last; I++)
      NVK += (SrcData->NumValueSites[I] != 0);

    if (!NVK)
      continue;

    if (SrcValueProfData >= ProfileData + ProfileSize)
      return 1;
    VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
    SrcValueProfData =
        SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
  }

  return 0;
}

#ifdef __GNUC__
#pragma GCC diagnostic pop
#elif defined(__clang__)
#pragma clang diagnostic pop
#endif