use crate::llvm;
use crate::abi::Abi;
use crate::builder::Builder;
use crate::common::CodegenCx;
use crate::coverageinfo::ffi::{CounterExpression, CounterMappingRegion};
use crate::coverageinfo::map_data::FunctionCoverage;
use libc::c_uint;
use rustc_codegen_ssa::traits::{
BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, MiscMethods,
StaticMethods,
};
use rustc_data_structures::fx::FxHashMap;
use rustc_hir as hir;
use rustc_hir::def_id::DefId;
use rustc_llvm::RustString;
use rustc_middle::bug;
use rustc_middle::mir::coverage::{CounterId, CoverageKind};
use rustc_middle::mir::Coverage;
use rustc_middle::ty;
use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
use rustc_middle::ty::GenericArgs;
use rustc_middle::ty::Instance;
use rustc_middle::ty::Ty;
use std::cell::RefCell;
pub(crate) mod ffi;
pub(crate) mod map_data;
pub mod mapgen;
const UNUSED_FUNCTION_COUNTER_ID: CounterId = CounterId::START;
const VAR_ALIGN_BYTES: usize = 8;
pub struct CrateCoverageContext<'ll, 'tcx> {
pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>>>,
pub(crate) pgo_func_name_var_map: RefCell<FxHashMap<Instance<'tcx>, &'ll llvm::Value>>,
}
impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> {
pub fn new() -> Self {
Self {
function_coverage_map: Default::default(),
pgo_func_name_var_map: Default::default(),
}
}
pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage<'tcx>> {
self.function_coverage_map.replace(FxHashMap::default())
}
}
impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
pub(crate) fn coverageinfo_finalize(&self) {
mapgen::finalize(self)
}
fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value {
if let Some(coverage_context) = self.coverage_context() {
debug!("getting pgo_func_name_var for instance={:?}", instance);
let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut();
pgo_func_name_var_map
.entry(instance)
.or_insert_with(|| create_pgo_func_name_var(self, instance))
} else {
bug!("Could not get the `coverage_context`");
}
}
fn define_unused_fn(&self, def_id: DefId) {
let instance = declare_unused_fn(self, def_id);
codegen_unused_fn_and_counter(self, instance);
add_unused_function_coverage(self, instance, def_id);
}
}
impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) {
let bx = self;
let Some(coverage_context) = bx.coverage_context() else { return };
let mut coverage_map = coverage_context.function_coverage_map.borrow_mut();
let func_coverage = coverage_map
.entry(instance)
.or_insert_with(|| FunctionCoverage::new(bx.tcx(), instance));
let Coverage { kind, code_regions } = coverage;
match *kind {
CoverageKind::Counter { function_source_hash, id } => {
debug!(
"ensuring function source hash is set for instance={:?}; function_source_hash={}",
instance, function_source_hash,
);
func_coverage.set_function_source_hash(function_source_hash);
func_coverage.add_counter(id, code_regions);
drop(coverage_map);
let coverageinfo = bx.tcx().coverageinfo(instance.def);
let fn_name = bx.get_pgo_func_name_var(instance);
let hash = bx.const_u64(function_source_hash);
let num_counters = bx.const_u32(coverageinfo.num_counters);
let index = bx.const_u32(id.as_u32());
debug!(
"codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
fn_name, hash, num_counters, index,
);
bx.instrprof_increment(fn_name, hash, num_counters, index);
}
CoverageKind::Expression { id, lhs, op, rhs } => {
func_coverage.add_counter_expression(id, lhs, op, rhs, code_regions);
}
CoverageKind::Unreachable => {
func_coverage.add_unreachable_regions(code_regions);
}
}
}
}
fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> {
let tcx = cx.tcx;
let instance = Instance::new(
def_id,
GenericArgs::for_item(tcx, def_id, |param, _| {
if let ty::GenericParamDefKind::Lifetime = param.kind {
tcx.lifetimes.re_erased.into()
} else {
tcx.mk_param_from_def(param)
}
}),
);
let llfn = cx.declare_fn(
tcx.symbol_name(instance).name,
cx.fn_abi_of_fn_ptr(
ty::Binder::dummy(tcx.mk_fn_sig(
[Ty::new_unit(tcx)],
Ty::new_unit(tcx),
false,
hir::Unsafety::Unsafe,
Abi::Rust,
)),
ty::List::empty(),
),
None,
);
llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage);
llvm::set_visibility(llfn, llvm::Visibility::Default);
assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none());
instance
}
fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
let llfn = cx.get_fn(instance);
let llbb = Builder::append_block(cx, llfn, "unused_function");
let mut bx = Builder::build(cx, llbb);
let fn_name = bx.get_pgo_func_name_var(instance);
let hash = bx.const_u64(0);
let num_counters = bx.const_u32(1);
let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID));
debug!(
"codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?},
index={:?}) for unused function: {:?}",
fn_name, hash, num_counters, index, instance
);
bx.instrprof_increment(fn_name, hash, num_counters, index);
bx.ret_void();
}
fn add_unused_function_coverage<'tcx>(
cx: &CodegenCx<'_, 'tcx>,
instance: Instance<'tcx>,
def_id: DefId,
) {
let tcx = cx.tcx;
let mut function_coverage = FunctionCoverage::unused(tcx, instance);
for &code_region in tcx.covered_code_regions(def_id) {
let code_region = std::slice::from_ref(code_region);
function_coverage.add_unreachable_regions(code_region);
}
if let Some(coverage_context) = cx.coverage_context() {
coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage);
} else {
bug!("Could not get the `coverage_context`");
}
}
fn create_pgo_func_name_var<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
instance: Instance<'tcx>,
) -> &'ll llvm::Value {
let mangled_fn_name: &str = cx.tcx.symbol_name(instance).name;
let llfn = cx.get_fn(instance);
unsafe {
llvm::LLVMRustCoverageCreatePGOFuncNameVar(
llfn,
mangled_fn_name.as_ptr().cast(),
mangled_fn_name.len(),
)
}
}
pub(crate) fn write_filenames_section_to_buffer<'a>(
filenames: impl IntoIterator<Item = &'a str>,
buffer: &RustString,
) {
let (pointers, lengths) = filenames
.into_iter()
.map(|s: &str| (s.as_ptr().cast(), s.len()))
.unzip::<_, _, Vec<_>, Vec<_>>();
unsafe {
llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(
pointers.as_ptr(),
pointers.len(),
lengths.as_ptr(),
lengths.len(),
buffer,
);
}
}
pub(crate) fn write_mapping_to_buffer(
virtual_file_mapping: Vec<u32>,
expressions: Vec<CounterExpression>,
mapping_regions: Vec<CounterMappingRegion>,
buffer: &RustString,
) {
unsafe {
llvm::LLVMRustCoverageWriteMappingToBuffer(
virtual_file_mapping.as_ptr(),
virtual_file_mapping.len() as c_uint,
expressions.as_ptr(),
expressions.len() as c_uint,
mapping_regions.as_ptr(),
mapping_regions.len() as c_uint,
buffer,
);
}
}
pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 {
unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) }
}
pub(crate) fn mapping_version() -> u32 {
unsafe { llvm::LLVMRustCoverageMappingVersion() }
}
pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
cov_data_val: &'ll llvm::Value,
) {
let covmap_var_name = llvm::build_string(|s| unsafe {
llvm::LLVMRustCoverageWriteMappingVarNameToString(s);
})
.expect("Rust Coverage Mapping var name failed UTF-8 conversion");
debug!("covmap var name: {:?}", covmap_var_name);
let covmap_section_name = llvm::build_string(|s| unsafe {
llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s);
})
.expect("Rust Coverage section name failed UTF-8 conversion");
debug!("covmap section name: {:?}", covmap_section_name);
let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name);
llvm::set_initializer(llglobal, cov_data_val);
llvm::set_global_constant(llglobal, true);
llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage);
llvm::set_section(llglobal, &covmap_section_name);
llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
cx.add_used_global(llglobal);
}
pub(crate) fn save_func_record_to_mod<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
covfun_section_name: &str,
func_name_hash: u64,
func_record_val: &'ll llvm::Value,
is_used: bool,
) {
let func_record_var_name =
format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" });
debug!("function record var name: {:?}", func_record_var_name);
debug!("function record section name: {:?}", covfun_section_name);
let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name);
llvm::set_initializer(llglobal, func_record_val);
llvm::set_global_constant(llglobal, true);
llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage);
llvm::set_visibility(llglobal, llvm::Visibility::Hidden);
llvm::set_section(llglobal, covfun_section_name);
llvm::set_alignment(llglobal, VAR_ALIGN_BYTES);
llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name);
cx.add_used_global(llglobal);
}
pub(crate) fn covfun_section_name(cx: &CodegenCx<'_, '_>) -> String {
llvm::build_string(|s| unsafe {
llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s);
})
.expect("Rust Coverage function record section name failed UTF-8 conversion")
}