use std::borrow::Cow;
use std::cell::RefCell;
use std::fmt;
use std::path::Path;
use std::process;
use either::Either;
use rand::rngs::StdRng;
use rand::SeedableRng;
use rustc_ast::ast::Mutability;
use rustc_const_eval::const_eval::CheckAlignment;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
#[allow(unused)]
use rustc_data_structures::static_assert_size;
use rustc_middle::{
mir,
ty::{
self,
layout::{LayoutCx, LayoutError, LayoutOf, TyAndLayout},
Instance, Ty, TyCtxt, TypeAndMut,
},
};
use rustc_span::def_id::{CrateNum, DefId};
use rustc_span::{Span, SpanData, Symbol};
use rustc_target::abi::{Align, Size};
use rustc_target::spec::abi::Abi;
use crate::{
concurrency::{data_race, weak_memory},
shims::unix::FileHandler,
*,
};
pub const SIGRTMIN: i32 = 34;
pub const SIGRTMAX: i32 = 42;
pub struct FrameExtra<'tcx> {
pub borrow_tracker: Option<borrow_tracker::FrameState>,
pub catch_unwind: Option<CatchUnwindData<'tcx>>,
pub timing: Option<measureme::DetachedTiming>,
pub is_user_relevant: bool,
}
impl<'tcx> std::fmt::Debug for FrameExtra<'tcx> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let FrameExtra { borrow_tracker, catch_unwind, timing: _, is_user_relevant: _ } = self;
f.debug_struct("FrameData")
.field("borrow_tracker", borrow_tracker)
.field("catch_unwind", catch_unwind)
.finish()
}
}
impl VisitTags for FrameExtra<'_> {
fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
let FrameExtra { catch_unwind, borrow_tracker, timing: _, is_user_relevant: _ } = self;
catch_unwind.visit_tags(visit);
borrow_tracker.visit_tags(visit);
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum MiriMemoryKind {
Rust,
Miri,
C,
WinHeap,
Machine,
Runtime,
Global,
ExternStatic,
Tls,
Mmap,
}
impl From<MiriMemoryKind> for MemoryKind<MiriMemoryKind> {
#[inline(always)]
fn from(kind: MiriMemoryKind) -> MemoryKind<MiriMemoryKind> {
MemoryKind::Machine(kind)
}
}
impl MayLeak for MiriMemoryKind {
#[inline(always)]
fn may_leak(self) -> bool {
use self::MiriMemoryKind::*;
match self {
Rust | Miri | C | WinHeap | Runtime => false,
Machine | Global | ExternStatic | Tls | Mmap => true,
}
}
}
impl MiriMemoryKind {
fn should_save_allocation_span(self) -> bool {
use self::MiriMemoryKind::*;
match self {
Rust | Miri | C | WinHeap | Mmap => true,
Machine | Global | ExternStatic | Tls | Runtime => false,
}
}
}
impl fmt::Display for MiriMemoryKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use self::MiriMemoryKind::*;
match self {
Rust => write!(f, "Rust heap"),
Miri => write!(f, "Miri bare-metal heap"),
C => write!(f, "C heap"),
WinHeap => write!(f, "Windows heap"),
Machine => write!(f, "machine-managed memory"),
Runtime => write!(f, "language runtime memory"),
Global => write!(f, "global (static or const)"),
ExternStatic => write!(f, "extern static"),
Tls => write!(f, "thread-local static"),
Mmap => write!(f, "mmap"),
}
}
}
#[derive(Clone, Copy)]
pub enum Provenance {
Concrete {
alloc_id: AllocId,
tag: BorTag,
},
Wildcard,
}
impl PartialEq for Provenance {
fn eq(&self, _other: &Self) -> bool {
panic!("Provenance must not be compared")
}
}
impl Eq for Provenance {}
impl std::hash::Hash for Provenance {
fn hash<H: std::hash::Hasher>(&self, _state: &mut H) {
panic!("Provenance must not be hashed")
}
}
#[derive(Copy, Clone, PartialEq)]
pub enum ProvenanceExtra {
Concrete(BorTag),
Wildcard,
}
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
static_assert_size!(Pointer<Provenance>, 24);
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
static_assert_size!(Scalar<Provenance>, 32);
impl fmt::Debug for Provenance {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Provenance::Concrete { alloc_id, tag } => {
if f.alternate() {
write!(f, "[{alloc_id:#?}]")?;
} else {
write!(f, "[{alloc_id:?}]")?;
}
write!(f, "{tag:?}")?;
}
Provenance::Wildcard => {
write!(f, "[wildcard]")?;
}
}
Ok(())
}
}
impl interpret::Provenance for Provenance {
const OFFSET_IS_ADDR: bool = true;
fn get_alloc_id(self) -> Option<AllocId> {
match self {
Provenance::Concrete { alloc_id, .. } => Some(alloc_id),
Provenance::Wildcard => None,
}
}
fn join(left: Option<Self>, right: Option<Self>) -> Option<Self> {
match (left, right) {
(
Some(Provenance::Concrete { alloc_id: left_alloc, tag: left_tag }),
Some(Provenance::Concrete { alloc_id: right_alloc, tag: right_tag }),
) if left_alloc == right_alloc && left_tag == right_tag => left,
(Some(Provenance::Wildcard), o) | (o, Some(Provenance::Wildcard)) => o,
_ => None,
}
}
}
impl fmt::Debug for ProvenanceExtra {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ProvenanceExtra::Concrete(pid) => write!(f, "{pid:?}"),
ProvenanceExtra::Wildcard => write!(f, "<wildcard>"),
}
}
}
impl ProvenanceExtra {
pub fn and_then<T>(self, f: impl FnOnce(BorTag) -> Option<T>) -> Option<T> {
match self {
ProvenanceExtra::Concrete(pid) => f(pid),
ProvenanceExtra::Wildcard => None,
}
}
}
#[derive(Debug, Clone)]
pub struct AllocExtra<'tcx> {
pub borrow_tracker: Option<borrow_tracker::AllocState>,
pub data_race: Option<data_race::AllocState>,
pub weak_memory: Option<weak_memory::AllocState>,
pub backtrace: Option<Vec<FrameInfo<'tcx>>>,
}
impl VisitTags for AllocExtra<'_> {
fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
let AllocExtra { borrow_tracker, data_race, weak_memory, backtrace: _ } = self;
borrow_tracker.visit_tags(visit);
data_race.visit_tags(visit);
weak_memory.visit_tags(visit);
}
}
pub struct PrimitiveLayouts<'tcx> {
pub unit: TyAndLayout<'tcx>,
pub i8: TyAndLayout<'tcx>,
pub i16: TyAndLayout<'tcx>,
pub i32: TyAndLayout<'tcx>,
pub i64: TyAndLayout<'tcx>,
pub i128: TyAndLayout<'tcx>,
pub isize: TyAndLayout<'tcx>,
pub u8: TyAndLayout<'tcx>,
pub u16: TyAndLayout<'tcx>,
pub u32: TyAndLayout<'tcx>,
pub u64: TyAndLayout<'tcx>,
pub u128: TyAndLayout<'tcx>,
pub usize: TyAndLayout<'tcx>,
pub bool: TyAndLayout<'tcx>,
pub mut_raw_ptr: TyAndLayout<'tcx>, pub const_raw_ptr: TyAndLayout<'tcx>, }
impl<'mir, 'tcx: 'mir> PrimitiveLayouts<'tcx> {
fn new(layout_cx: LayoutCx<'tcx, TyCtxt<'tcx>>) -> Result<Self, &'tcx LayoutError<'tcx>> {
let tcx = layout_cx.tcx;
let mut_raw_ptr =
Ty::new_ptr(tcx, TypeAndMut { ty: tcx.types.unit, mutbl: Mutability::Mut });
let const_raw_ptr =
Ty::new_ptr(tcx, TypeAndMut { ty: tcx.types.unit, mutbl: Mutability::Not });
Ok(Self {
unit: layout_cx.layout_of(Ty::new_unit(tcx))?,
i8: layout_cx.layout_of(tcx.types.i8)?,
i16: layout_cx.layout_of(tcx.types.i16)?,
i32: layout_cx.layout_of(tcx.types.i32)?,
i64: layout_cx.layout_of(tcx.types.i64)?,
i128: layout_cx.layout_of(tcx.types.i128)?,
isize: layout_cx.layout_of(tcx.types.isize)?,
u8: layout_cx.layout_of(tcx.types.u8)?,
u16: layout_cx.layout_of(tcx.types.u16)?,
u32: layout_cx.layout_of(tcx.types.u32)?,
u64: layout_cx.layout_of(tcx.types.u64)?,
u128: layout_cx.layout_of(tcx.types.u128)?,
usize: layout_cx.layout_of(tcx.types.usize)?,
bool: layout_cx.layout_of(tcx.types.bool)?,
mut_raw_ptr: layout_cx.layout_of(mut_raw_ptr)?,
const_raw_ptr: layout_cx.layout_of(const_raw_ptr)?,
})
}
pub fn uint(&self, size: Size) -> Option<TyAndLayout<'tcx>> {
match size.bits() {
8 => Some(self.u8),
16 => Some(self.u16),
32 => Some(self.u32),
64 => Some(self.u64),
128 => Some(self.u128),
_ => None,
}
}
pub fn int(&self, size: Size) -> Option<TyAndLayout<'tcx>> {
match size.bits() {
8 => Some(self.i8),
16 => Some(self.i16),
32 => Some(self.i32),
64 => Some(self.i64),
128 => Some(self.i128),
_ => None,
}
}
}
pub struct MiriMachine<'mir, 'tcx> {
pub tcx: TyCtxt<'tcx>,
pub borrow_tracker: Option<borrow_tracker::GlobalState>,
pub data_race: Option<data_race::GlobalState>,
pub intptrcast: intptrcast::GlobalState,
pub(crate) env_vars: EnvVars<'tcx>,
pub(crate) main_fn_ret_place: Option<MPlaceTy<'tcx, Provenance>>,
pub(crate) argc: Option<Pointer<Option<Provenance>>>,
pub(crate) argv: Option<Pointer<Option<Provenance>>>,
pub(crate) cmd_line: Option<Pointer<Option<Provenance>>>,
pub(crate) tls: TlsData<'tcx>,
pub(crate) isolated_op: IsolatedOp,
pub(crate) validate: bool,
pub(crate) enforce_abi: bool,
pub(crate) file_handler: shims::unix::FileHandler,
pub(crate) dir_handler: shims::unix::DirHandler,
pub(crate) clock: Clock,
pub(crate) threads: ThreadManager<'mir, 'tcx>,
pub(crate) layouts: PrimitiveLayouts<'tcx>,
pub(crate) static_roots: Vec<AllocId>,
profiler: Option<measureme::Profiler>,
string_cache: FxHashMap<String, measureme::StringId>,
pub(crate) exported_symbols_cache: FxHashMap<Symbol, Option<Instance<'tcx>>>,
pub(crate) panic_on_unsupported: bool,
pub(crate) backtrace_style: BacktraceStyle,
pub(crate) local_crates: Vec<CrateNum>,
extern_statics: FxHashMap<Symbol, Pointer<Provenance>>,
pub(crate) rng: RefCell<StdRng>,
tracked_alloc_ids: FxHashSet<AllocId>,
pub(crate) check_alignment: AlignmentCheck,
pub(crate) cmpxchg_weak_failure_rate: f64,
pub(crate) mute_stdout_stderr: bool,
pub(crate) weak_memory: bool,
pub(crate) preemption_rate: f64,
pub(crate) report_progress: Option<u32>,
pub(crate) basic_block_count: u64,
#[cfg(target_os = "linux")]
pub external_so_lib: Option<(libloading::Library, std::path::PathBuf)>,
#[cfg(not(target_os = "linux"))]
pub external_so_lib: Option<!>,
pub(crate) gc_interval: u32,
pub(crate) since_gc: u32,
pub(crate) num_cpus: u32,
pub(crate) page_size: u64,
pub(crate) stack_addr: u64,
pub(crate) stack_size: u64,
pub(crate) collect_leak_backtraces: bool,
pub(crate) allocation_spans: RefCell<FxHashMap<AllocId, (Span, Option<Span>)>>,
}
impl<'mir, 'tcx> MiriMachine<'mir, 'tcx> {
pub(crate) fn new(config: &MiriConfig, layout_cx: LayoutCx<'tcx, TyCtxt<'tcx>>) -> Self {
let tcx = layout_cx.tcx;
let local_crates = helpers::get_local_crates(tcx);
let layouts =
PrimitiveLayouts::new(layout_cx).expect("Couldn't get layouts of primitive types");
let profiler = config.measureme_out.as_ref().map(|out| {
let crate_name = layout_cx
.tcx
.sess
.opts
.crate_name
.clone()
.unwrap_or_else(|| "unknown-crate".to_string());
let pid = process::id();
let filename = format!("{crate_name}-{pid:07}");
let path = Path::new(out).join(filename);
measureme::Profiler::new(path).expect("Couldn't create `measureme` profiler")
});
let rng = StdRng::seed_from_u64(config.seed.unwrap_or(0));
let borrow_tracker = config.borrow_tracker.map(|bt| bt.instantiate_global_state(config));
let data_race = config.data_race_detector.then(|| data_race::GlobalState::new(config));
let page_size = if let Some(page_size) = config.page_size {
page_size
} else {
let target = &tcx.sess.target;
match target.arch.as_ref() {
"wasm32" | "wasm64" => 64 * 1024, "aarch64" => {
if target.options.vendor.as_ref() == "apple" {
16 * 1024
} else {
4 * 1024
}
}
_ => 4 * 1024,
}
};
let stack_addr = if tcx.pointer_size().bits() < 32 { page_size } else { page_size * 32 };
let stack_size =
if tcx.pointer_size().bits() < 32 { page_size * 4 } else { page_size * 16 };
MiriMachine {
tcx,
borrow_tracker,
data_race,
intptrcast: RefCell::new(intptrcast::GlobalStateInner::new(config, stack_addr)),
env_vars: EnvVars::default(),
main_fn_ret_place: None,
argc: None,
argv: None,
cmd_line: None,
tls: TlsData::default(),
isolated_op: config.isolated_op,
validate: config.validate,
enforce_abi: config.check_abi,
file_handler: FileHandler::new(config.mute_stdout_stderr),
dir_handler: Default::default(),
layouts,
threads: ThreadManager::default(),
static_roots: Vec::new(),
profiler,
string_cache: Default::default(),
exported_symbols_cache: FxHashMap::default(),
panic_on_unsupported: config.panic_on_unsupported,
backtrace_style: config.backtrace_style,
local_crates,
extern_statics: FxHashMap::default(),
rng: RefCell::new(rng),
tracked_alloc_ids: config.tracked_alloc_ids.clone(),
check_alignment: config.check_alignment,
cmpxchg_weak_failure_rate: config.cmpxchg_weak_failure_rate,
mute_stdout_stderr: config.mute_stdout_stderr,
weak_memory: config.weak_memory_emulation,
preemption_rate: config.preemption_rate,
report_progress: config.report_progress,
basic_block_count: 0,
clock: Clock::new(config.isolated_op == IsolatedOp::Allow),
#[cfg(target_os = "linux")]
external_so_lib: config.external_so_file.as_ref().map(|lib_file_path| {
let target_triple = layout_cx.tcx.sess.opts.target_triple.triple();
if env!("TARGET") != target_triple {
panic!(
"calling external C functions in linked .so file requires host and target to be the same: host={}, target={}",
env!("TARGET"),
target_triple,
);
}
(
unsafe {
libloading::Library::new(lib_file_path)
.expect("failed to read specified extern shared object file")
},
lib_file_path.clone(),
)
}),
#[cfg(not(target_os = "linux"))]
external_so_lib: config.external_so_file.as_ref().map(|_| {
panic!("loading external .so files is only supported on Linux")
}),
gc_interval: config.gc_interval,
since_gc: 0,
num_cpus: config.num_cpus,
page_size,
stack_addr,
stack_size,
collect_leak_backtraces: config.collect_leak_backtraces,
allocation_spans: RefCell::new(FxHashMap::default()),
}
}
pub(crate) fn late_init(
this: &mut MiriInterpCx<'mir, 'tcx>,
config: &MiriConfig,
on_main_stack_empty: StackEmptyCallback<'mir, 'tcx>,
) -> InterpResult<'tcx> {
EnvVars::init(this, config)?;
MiriMachine::init_extern_statics(this)?;
ThreadManager::init(this, on_main_stack_empty);
Ok(())
}
fn add_extern_static(
this: &mut MiriInterpCx<'mir, 'tcx>,
name: &str,
ptr: Pointer<Option<Provenance>>,
) {
let ptr = ptr.into_pointer_or_addr().unwrap();
this.machine.extern_statics.try_insert(Symbol::intern(name), ptr).unwrap();
}
fn alloc_extern_static(
this: &mut MiriInterpCx<'mir, 'tcx>,
name: &str,
val: ImmTy<'tcx, Provenance>,
) -> InterpResult<'tcx> {
let place = this.allocate(val.layout, MiriMemoryKind::ExternStatic.into())?;
this.write_immediate(*val, &place)?;
Self::add_extern_static(this, name, place.ptr());
Ok(())
}
fn init_extern_statics(this: &mut MiriInterpCx<'mir, 'tcx>) -> InterpResult<'tcx> {
let val = ImmTy::from_int(0, this.machine.layouts.u8);
Self::alloc_extern_static(this, "__rust_no_alloc_shim_is_unstable", val)?;
match this.tcx.sess.target.os.as_ref() {
"linux" => {
Self::add_extern_static(
this,
"environ",
this.machine.env_vars.environ.as_ref().unwrap().ptr(),
);
for name in &["__cxa_thread_atexit_impl", "getrandom", "statx", "__clock_gettime64"]
{
let val = ImmTy::from_int(0, this.machine.layouts.usize);
Self::alloc_extern_static(this, name, val)?;
}
}
"freebsd" => {
Self::add_extern_static(
this,
"environ",
this.machine.env_vars.environ.as_ref().unwrap().ptr(),
);
}
"android" => {
let layout = this.machine.layouts.const_raw_ptr;
let dlsym = Dlsym::from_str("signal".as_bytes(), &this.tcx.sess.target.os)?
.expect("`signal` must be an actual dlsym on android");
let ptr = this.fn_ptr(FnVal::Other(dlsym));
let val = ImmTy::from_scalar(Scalar::from_pointer(ptr, this), layout);
Self::alloc_extern_static(this, "signal", val)?;
for name in &["bsd_signal"] {
let val = ImmTy::from_int(0, this.machine.layouts.usize);
Self::alloc_extern_static(this, name, val)?;
}
}
"windows" => {
let val = ImmTy::from_int(0, this.machine.layouts.u8);
Self::alloc_extern_static(this, "_tls_used", val)?;
}
_ => {} }
Ok(())
}
pub(crate) fn communicate(&self) -> bool {
self.isolated_op == IsolatedOp::Allow
}
pub(crate) fn is_local(&self, frame: &FrameInfo<'_>) -> bool {
let def_id = frame.instance.def_id();
def_id.is_local() || self.local_crates.contains(&def_id.krate)
}
pub(crate) fn handle_abnormal_termination(&mut self) {
drop(self.profiler.take());
}
pub(crate) fn round_up_to_multiple_of_page_size(&self, length: u64) -> Option<u64> {
#[allow(clippy::arithmetic_side_effects)] (length.checked_add(self.page_size - 1)? / self.page_size).checked_mul(self.page_size)
}
pub(crate) fn page_align(&self) -> Align {
Align::from_bytes(self.page_size).unwrap()
}
pub(crate) fn allocated_span(&self, alloc_id: AllocId) -> Option<SpanData> {
self.allocation_spans
.borrow()
.get(&alloc_id)
.map(|(allocated, _deallocated)| allocated.data())
}
pub(crate) fn deallocated_span(&self, alloc_id: AllocId) -> Option<SpanData> {
self.allocation_spans
.borrow()
.get(&alloc_id)
.and_then(|(_allocated, deallocated)| *deallocated)
.map(Span::data)
}
}
impl VisitTags for MiriMachine<'_, '_> {
fn visit_tags(&self, visit: &mut dyn FnMut(BorTag)) {
#[rustfmt::skip]
let MiriMachine {
threads,
tls,
env_vars,
main_fn_ret_place,
argc,
argv,
cmd_line,
extern_statics,
dir_handler,
borrow_tracker,
data_race,
intptrcast,
file_handler,
tcx: _,
isolated_op: _,
validate: _,
enforce_abi: _,
clock: _,
layouts: _,
static_roots: _,
profiler: _,
string_cache: _,
exported_symbols_cache: _,
panic_on_unsupported: _,
backtrace_style: _,
local_crates: _,
rng: _,
tracked_alloc_ids: _,
check_alignment: _,
cmpxchg_weak_failure_rate: _,
mute_stdout_stderr: _,
weak_memory: _,
preemption_rate: _,
report_progress: _,
basic_block_count: _,
external_so_lib: _,
gc_interval: _,
since_gc: _,
num_cpus: _,
page_size: _,
stack_addr: _,
stack_size: _,
collect_leak_backtraces: _,
allocation_spans: _,
} = self;
threads.visit_tags(visit);
tls.visit_tags(visit);
env_vars.visit_tags(visit);
dir_handler.visit_tags(visit);
file_handler.visit_tags(visit);
data_race.visit_tags(visit);
borrow_tracker.visit_tags(visit);
intptrcast.visit_tags(visit);
main_fn_ret_place.visit_tags(visit);
argc.visit_tags(visit);
argv.visit_tags(visit);
cmd_line.visit_tags(visit);
for ptr in extern_statics.values() {
ptr.visit_tags(visit);
}
}
}
pub type MiriInterpCx<'mir, 'tcx> = InterpCx<'mir, 'tcx, MiriMachine<'mir, 'tcx>>;
pub trait MiriInterpCxExt<'mir, 'tcx> {
fn eval_context_ref<'a>(&'a self) -> &'a MiriInterpCx<'mir, 'tcx>;
fn eval_context_mut<'a>(&'a mut self) -> &'a mut MiriInterpCx<'mir, 'tcx>;
}
impl<'mir, 'tcx> MiriInterpCxExt<'mir, 'tcx> for MiriInterpCx<'mir, 'tcx> {
#[inline(always)]
fn eval_context_ref(&self) -> &MiriInterpCx<'mir, 'tcx> {
self
}
#[inline(always)]
fn eval_context_mut(&mut self) -> &mut MiriInterpCx<'mir, 'tcx> {
self
}
}
impl<'mir, 'tcx> Machine<'mir, 'tcx> for MiriMachine<'mir, 'tcx> {
type MemoryKind = MiriMemoryKind;
type ExtraFnVal = Dlsym;
type FrameExtra = FrameExtra<'tcx>;
type AllocExtra = AllocExtra<'tcx>;
type Provenance = Provenance;
type ProvenanceExtra = ProvenanceExtra;
type Bytes = Box<[u8]>;
type MemoryMap = MonoHashMap<
AllocId,
(MemoryKind<MiriMemoryKind>, Allocation<Provenance, Self::AllocExtra, Self::Bytes>),
>;
const GLOBAL_KIND: Option<MiriMemoryKind> = Some(MiriMemoryKind::Global);
const PANIC_ON_ALLOC_FAIL: bool = false;
#[inline(always)]
fn enforce_alignment(ecx: &MiriInterpCx<'mir, 'tcx>) -> CheckAlignment {
if ecx.machine.check_alignment == AlignmentCheck::None {
CheckAlignment::No
} else {
CheckAlignment::Error
}
}
#[inline(always)]
fn use_addr_for_alignment_check(ecx: &MiriInterpCx<'mir, 'tcx>) -> bool {
ecx.machine.check_alignment == AlignmentCheck::Int
}
fn alignment_check_failed(
_ecx: &InterpCx<'mir, 'tcx, Self>,
has: Align,
required: Align,
_check: CheckAlignment,
) -> InterpResult<'tcx, ()> {
throw_ub!(AlignmentCheckFailed { has, required })
}
#[inline(always)]
fn enforce_validity(ecx: &MiriInterpCx<'mir, 'tcx>, _layout: TyAndLayout<'tcx>) -> bool {
ecx.machine.validate
}
#[inline(always)]
fn enforce_abi(ecx: &MiriInterpCx<'mir, 'tcx>) -> bool {
ecx.machine.enforce_abi
}
#[inline(always)]
fn ignore_optional_overflow_checks(ecx: &MiriInterpCx<'mir, 'tcx>) -> bool {
!ecx.tcx.sess.overflow_checks()
}
#[inline(always)]
fn find_mir_or_eval_fn(
ecx: &mut MiriInterpCx<'mir, 'tcx>,
instance: ty::Instance<'tcx>,
abi: Abi,
args: &[FnArg<'tcx, Provenance>],
dest: &PlaceTy<'tcx, Provenance>,
ret: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> {
ecx.find_mir_or_eval_fn(instance, abi, args, dest, ret, unwind)
}
#[inline(always)]
fn call_extra_fn(
ecx: &mut MiriInterpCx<'mir, 'tcx>,
fn_val: Dlsym,
abi: Abi,
args: &[FnArg<'tcx, Provenance>],
dest: &PlaceTy<'tcx, Provenance>,
ret: Option<mir::BasicBlock>,
_unwind: mir::UnwindAction,
) -> InterpResult<'tcx> {
let args = ecx.copy_fn_args(args)?; ecx.call_dlsym(fn_val, abi, &args, dest, ret)
}
#[inline(always)]
fn call_intrinsic(
ecx: &mut MiriInterpCx<'mir, 'tcx>,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx, Provenance>],
dest: &PlaceTy<'tcx, Provenance>,
ret: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx> {
ecx.call_intrinsic(instance, args, dest, ret, unwind)
}
#[inline(always)]
fn assert_panic(
ecx: &mut MiriInterpCx<'mir, 'tcx>,
msg: &mir::AssertMessage<'tcx>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx> {
ecx.assert_panic(msg, unwind)
}
fn panic_nounwind(ecx: &mut InterpCx<'mir, 'tcx, Self>, msg: &str) -> InterpResult<'tcx> {
ecx.start_panic_nounwind(msg)
}
fn unwind_terminate(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
reason: mir::UnwindTerminateReason,
) -> InterpResult<'tcx> {
let panic = ecx.tcx.lang_items().get(reason.lang_item()).unwrap();
let panic = ty::Instance::mono(ecx.tcx.tcx, panic);
ecx.call_function(
panic,
Abi::Rust,
&[],
None,
StackPopCleanup::Goto { ret: None, unwind: mir::UnwindAction::Unreachable },
)?;
Ok(())
}
#[inline(always)]
fn binary_ptr_op(
ecx: &MiriInterpCx<'mir, 'tcx>,
bin_op: mir::BinOp,
left: &ImmTy<'tcx, Provenance>,
right: &ImmTy<'tcx, Provenance>,
) -> InterpResult<'tcx, (ImmTy<'tcx, Provenance>, bool)> {
ecx.binary_ptr_op(bin_op, left, right)
}
fn thread_local_static_base_pointer(
ecx: &mut MiriInterpCx<'mir, 'tcx>,
def_id: DefId,
) -> InterpResult<'tcx, Pointer<Provenance>> {
ecx.get_or_create_thread_local_alloc(def_id)
}
fn extern_static_base_pointer(
ecx: &MiriInterpCx<'mir, 'tcx>,
def_id: DefId,
) -> InterpResult<'tcx, Pointer<Provenance>> {
let link_name = ecx.item_link_name(def_id);
if let Some(&ptr) = ecx.machine.extern_statics.get(&link_name) {
let Provenance::Concrete { alloc_id, .. } = ptr.provenance else {
panic!("extern_statics cannot contain wildcards")
};
let (shim_size, shim_align, _kind) = ecx.get_alloc_info(alloc_id);
let def_ty = ecx.tcx.type_of(def_id).instantiate_identity();
let extern_decl_layout = ecx.tcx.layout_of(ty::ParamEnv::empty().and(def_ty)).unwrap();
if extern_decl_layout.size != shim_size || extern_decl_layout.align.abi != shim_align {
throw_unsup_format!(
"`extern` static `{name}` from crate `{krate}` has been declared \
with a size of {decl_size} bytes and alignment of {decl_align} bytes, \
but Miri emulates it via an extern static shim \
with a size of {shim_size} bytes and alignment of {shim_align} bytes",
name = ecx.tcx.def_path_str(def_id),
krate = ecx.tcx.crate_name(def_id.krate),
decl_size = extern_decl_layout.size.bytes(),
decl_align = extern_decl_layout.align.abi.bytes(),
shim_size = shim_size.bytes(),
shim_align = shim_align.bytes(),
)
}
Ok(ptr)
} else {
throw_unsup_format!(
"`extern` static `{name}` from crate `{krate}` is not supported by Miri",
name = ecx.tcx.def_path_str(def_id),
krate = ecx.tcx.crate_name(def_id.krate),
)
}
}
fn adjust_allocation<'b>(
ecx: &MiriInterpCx<'mir, 'tcx>,
id: AllocId,
alloc: Cow<'b, Allocation>,
kind: Option<MemoryKind<Self::MemoryKind>>,
) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra>>> {
let kind = kind.expect("we set our STATIC_KIND so this cannot be None");
if ecx.machine.tracked_alloc_ids.contains(&id) {
ecx.emit_diagnostic(NonHaltingDiagnostic::CreatedAlloc(
id,
alloc.size(),
alloc.align,
kind,
));
}
let alloc = alloc.into_owned();
let borrow_tracker = ecx
.machine
.borrow_tracker
.as_ref()
.map(|bt| bt.borrow_mut().new_allocation(id, alloc.size(), kind, &ecx.machine));
let race_alloc = ecx.machine.data_race.as_ref().map(|data_race| {
data_race::AllocState::new_allocation(
data_race,
&ecx.machine.threads,
alloc.size(),
kind,
ecx.machine.current_span(),
)
});
let buffer_alloc = ecx.machine.weak_memory.then(weak_memory::AllocState::new_allocation);
let backtrace = if kind.may_leak() || !ecx.machine.collect_leak_backtraces {
None
} else {
Some(ecx.generate_stacktrace())
};
let alloc: Allocation<Provenance, Self::AllocExtra> = alloc.adjust_from_tcx(
&ecx.tcx,
AllocExtra {
borrow_tracker,
data_race: race_alloc,
weak_memory: buffer_alloc,
backtrace,
},
|ptr| ecx.global_base_pointer(ptr),
)?;
if matches!(kind, MemoryKind::Machine(kind) if kind.should_save_allocation_span()) {
ecx.machine
.allocation_spans
.borrow_mut()
.insert(id, (ecx.machine.current_span(), None));
}
Ok(Cow::Owned(alloc))
}
fn adjust_alloc_base_pointer(
ecx: &MiriInterpCx<'mir, 'tcx>,
ptr: Pointer<AllocId>,
) -> InterpResult<'tcx, Pointer<Provenance>> {
if cfg!(debug_assertions) {
let alloc_id = ptr.provenance;
match ecx.tcx.try_get_global_alloc(alloc_id) {
Some(GlobalAlloc::Static(def_id)) if ecx.tcx.is_thread_local_static(def_id) => {
panic!("adjust_alloc_base_pointer called on thread-local static")
}
Some(GlobalAlloc::Static(def_id)) if ecx.tcx.is_foreign_item(def_id) => {
panic!("adjust_alloc_base_pointer called on extern static")
}
_ => {}
}
}
let absolute_addr = intptrcast::GlobalStateInner::rel_ptr_to_addr(ecx, ptr)?;
let tag = if let Some(borrow_tracker) = &ecx.machine.borrow_tracker {
borrow_tracker.borrow_mut().base_ptr_tag(ptr.provenance, &ecx.machine)
} else {
BorTag::default()
};
Ok(Pointer::new(
Provenance::Concrete { alloc_id: ptr.provenance, tag },
Size::from_bytes(absolute_addr),
))
}
#[inline(always)]
fn ptr_from_addr_cast(
ecx: &MiriInterpCx<'mir, 'tcx>,
addr: u64,
) -> InterpResult<'tcx, Pointer<Option<Self::Provenance>>> {
intptrcast::GlobalStateInner::ptr_from_addr_cast(ecx, addr)
}
fn expose_ptr(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
ptr: Pointer<Self::Provenance>,
) -> InterpResult<'tcx> {
match ptr.provenance {
Provenance::Concrete { alloc_id, tag } =>
intptrcast::GlobalStateInner::expose_ptr(ecx, alloc_id, tag),
Provenance::Wildcard => {
Ok(())
}
}
}
fn ptr_get_alloc(
ecx: &MiriInterpCx<'mir, 'tcx>,
ptr: Pointer<Self::Provenance>,
) -> Option<(AllocId, Size, Self::ProvenanceExtra)> {
let rel = intptrcast::GlobalStateInner::abs_ptr_to_rel(ecx, ptr);
rel.map(|(alloc_id, size)| {
let tag = match ptr.provenance {
Provenance::Concrete { tag, .. } => ProvenanceExtra::Concrete(tag),
Provenance::Wildcard => ProvenanceExtra::Wildcard,
};
(alloc_id, size, tag)
})
}
#[inline(always)]
fn before_memory_read(
_tcx: TyCtxt<'tcx>,
machine: &Self,
alloc_extra: &AllocExtra<'tcx>,
(alloc_id, prov_extra): (AllocId, Self::ProvenanceExtra),
range: AllocRange,
) -> InterpResult<'tcx> {
if let Some(data_race) = &alloc_extra.data_race {
data_race.read(alloc_id, range, machine)?;
}
if let Some(borrow_tracker) = &alloc_extra.borrow_tracker {
borrow_tracker.before_memory_read(alloc_id, prov_extra, range, machine)?;
}
if let Some(weak_memory) = &alloc_extra.weak_memory {
weak_memory.memory_accessed(range, machine.data_race.as_ref().unwrap());
}
Ok(())
}
#[inline(always)]
fn before_memory_write(
_tcx: TyCtxt<'tcx>,
machine: &mut Self,
alloc_extra: &mut AllocExtra<'tcx>,
(alloc_id, prov_extra): (AllocId, Self::ProvenanceExtra),
range: AllocRange,
) -> InterpResult<'tcx> {
if let Some(data_race) = &mut alloc_extra.data_race {
data_race.write(alloc_id, range, machine)?;
}
if let Some(borrow_tracker) = &mut alloc_extra.borrow_tracker {
borrow_tracker.before_memory_write(alloc_id, prov_extra, range, machine)?;
}
if let Some(weak_memory) = &alloc_extra.weak_memory {
weak_memory.memory_accessed(range, machine.data_race.as_ref().unwrap());
}
Ok(())
}
#[inline(always)]
fn before_memory_deallocation(
_tcx: TyCtxt<'tcx>,
machine: &mut Self,
alloc_extra: &mut AllocExtra<'tcx>,
(alloc_id, prove_extra): (AllocId, Self::ProvenanceExtra),
range: AllocRange,
) -> InterpResult<'tcx> {
if machine.tracked_alloc_ids.contains(&alloc_id) {
machine.emit_diagnostic(NonHaltingDiagnostic::FreedAlloc(alloc_id));
}
if let Some(data_race) = &mut alloc_extra.data_race {
data_race.deallocate(alloc_id, range, machine)?;
}
if let Some(borrow_tracker) = &mut alloc_extra.borrow_tracker {
borrow_tracker.before_memory_deallocation(alloc_id, prove_extra, range, machine)?;
}
if let Some((_, deallocated_at)) = machine.allocation_spans.borrow_mut().get_mut(&alloc_id)
{
*deallocated_at = Some(machine.current_span());
}
Ok(())
}
#[inline(always)]
fn retag_ptr_value(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
kind: mir::RetagKind,
val: &ImmTy<'tcx, Provenance>,
) -> InterpResult<'tcx, ImmTy<'tcx, Provenance>> {
if ecx.machine.borrow_tracker.is_some() {
ecx.retag_ptr_value(kind, val)
} else {
Ok(val.clone())
}
}
#[inline(always)]
fn retag_place_contents(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
kind: mir::RetagKind,
place: &PlaceTy<'tcx, Provenance>,
) -> InterpResult<'tcx> {
if ecx.machine.borrow_tracker.is_some() {
ecx.retag_place_contents(kind, place)?;
}
Ok(())
}
fn protect_in_place_function_argument(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
place: &PlaceTy<'tcx, Provenance>,
) -> InterpResult<'tcx> {
ecx.write_uninit(place)?;
if ecx.machine.borrow_tracker.is_some() {
if let Either::Left(place) = ecx.place_to_op(place)?.as_mplace_or_imm() {
ecx.protect_place(&place)?;
} else {
}
}
Ok(())
}
#[inline(always)]
fn init_frame_extra(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
frame: Frame<'mir, 'tcx, Provenance>,
) -> InterpResult<'tcx, Frame<'mir, 'tcx, Provenance, FrameExtra<'tcx>>> {
let timing = if let Some(profiler) = ecx.machine.profiler.as_ref() {
let fn_name = frame.instance.to_string();
let entry = ecx.machine.string_cache.entry(fn_name.clone());
let name = entry.or_insert_with(|| profiler.alloc_string(&*fn_name));
Some(profiler.start_recording_interval_event_detached(
*name,
measureme::EventId::from_label(*name),
ecx.get_active_thread().to_u32(),
))
} else {
None
};
let borrow_tracker = ecx.machine.borrow_tracker.as_ref();
let extra = FrameExtra {
borrow_tracker: borrow_tracker.map(|bt| bt.borrow_mut().new_frame(&ecx.machine)),
catch_unwind: None,
timing,
is_user_relevant: ecx.machine.is_user_relevant(&frame),
};
Ok(frame.with_extra(extra))
}
fn stack<'a>(
ecx: &'a InterpCx<'mir, 'tcx, Self>,
) -> &'a [Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>] {
ecx.active_thread_stack()
}
fn stack_mut<'a>(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
) -> &'a mut Vec<Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>> {
ecx.active_thread_stack_mut()
}
fn before_terminator(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
ecx.machine.basic_block_count += 1u64; ecx.machine.since_gc += 1;
if let Some(report_progress) = ecx.machine.report_progress {
if ecx.machine.basic_block_count % u64::from(report_progress) == 0 {
ecx.emit_diagnostic(NonHaltingDiagnostic::ProgressReport {
block_count: ecx.machine.basic_block_count,
});
}
}
if ecx.machine.gc_interval > 0 && ecx.machine.since_gc >= ecx.machine.gc_interval {
ecx.machine.since_gc = 0;
ecx.garbage_collect_tags()?;
}
ecx.maybe_preempt_active_thread();
ecx.machine.clock.tick();
Ok(())
}
#[inline(always)]
fn after_stack_push(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
if ecx.frame().extra.is_user_relevant {
let stack_len = ecx.active_thread_stack().len();
ecx.active_thread_mut().set_top_user_relevant_frame(stack_len - 1);
}
Ok(())
}
fn before_stack_pop(
ecx: &InterpCx<'mir, 'tcx, Self>,
frame: &Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>,
) -> InterpResult<'tcx> {
if let Some(borrow_tracker) = &ecx.machine.borrow_tracker {
borrow_tracker.borrow_mut().end_call(&frame.extra);
}
Ok(())
}
#[inline(always)]
fn after_stack_pop(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
mut frame: Frame<'mir, 'tcx, Provenance, FrameExtra<'tcx>>,
unwinding: bool,
) -> InterpResult<'tcx, StackPopJump> {
if frame.extra.is_user_relevant {
ecx.active_thread_mut().recompute_top_user_relevant_frame();
}
let timing = frame.extra.timing.take();
let res = ecx.handle_stack_pop_unwind(frame.extra, unwinding);
if let Some(profiler) = ecx.machine.profiler.as_ref() {
profiler.finish_recording_interval_event(timing.unwrap());
}
res
}
fn after_local_allocated(
ecx: &mut InterpCx<'mir, 'tcx, Self>,
frame: usize,
local: mir::Local,
mplace: &MPlaceTy<'tcx, Provenance>,
) -> InterpResult<'tcx> {
let Some(Provenance::Concrete { alloc_id, .. }) = mplace.ptr().provenance else {
panic!("after_local_allocated should only be called on fresh allocations");
};
let local_decl = &ecx.active_thread_stack()[frame].body.local_decls[local];
let span = local_decl.source_info.span;
ecx.machine.allocation_spans.borrow_mut().insert(alloc_id, (span, None));
Ok(())
}
}