use std::assert_matches::assert_matches;
use either::{Either, Left, Right};
use rustc_hir::def::Namespace;
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
use rustc_middle::ty::{ConstInt, Ty, TyCtxt};
use rustc_middle::{mir, ty};
use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size};
use super::{
alloc_range, from_known_layout, mir_assign_valid_types, AllocId, Frame, InterpCx, InterpResult,
MPlaceTy, Machine, MemPlace, MemPlaceMeta, PlaceTy, Pointer, Projectable, Provenance, Scalar,
};
#[derive(Copy, Clone, Debug)]
pub enum Immediate<Prov: Provenance = AllocId> {
Scalar(Scalar<Prov>),
ScalarPair(Scalar<Prov>, Scalar<Prov>),
Uninit,
}
impl<Prov: Provenance> From<Scalar<Prov>> for Immediate<Prov> {
#[inline(always)]
fn from(val: Scalar<Prov>) -> Self {
Immediate::Scalar(val)
}
}
impl<Prov: Provenance> Immediate<Prov> {
pub fn from_pointer(ptr: Pointer<Prov>, cx: &impl HasDataLayout) -> Self {
Immediate::Scalar(Scalar::from_pointer(ptr, cx))
}
pub fn from_maybe_pointer(ptr: Pointer<Option<Prov>>, cx: &impl HasDataLayout) -> Self {
Immediate::Scalar(Scalar::from_maybe_pointer(ptr, cx))
}
pub fn new_slice(ptr: Pointer<Option<Prov>>, len: u64, cx: &impl HasDataLayout) -> Self {
Immediate::ScalarPair(
Scalar::from_maybe_pointer(ptr, cx),
Scalar::from_target_usize(len, cx),
)
}
pub fn new_dyn_trait(
val: Pointer<Option<Prov>>,
vtable: Pointer<Option<Prov>>,
cx: &impl HasDataLayout,
) -> Self {
Immediate::ScalarPair(
Scalar::from_maybe_pointer(val, cx),
Scalar::from_maybe_pointer(vtable, cx),
)
}
#[inline]
#[cfg_attr(debug_assertions, track_caller)] pub fn to_scalar(self) -> Scalar<Prov> {
match self {
Immediate::Scalar(val) => val,
Immediate::ScalarPair(..) => bug!("Got a scalar pair where a scalar was expected"),
Immediate::Uninit => bug!("Got uninit where a scalar was expected"),
}
}
#[inline]
#[cfg_attr(debug_assertions, track_caller)] pub fn to_scalar_pair(self) -> (Scalar<Prov>, Scalar<Prov>) {
match self {
Immediate::ScalarPair(val1, val2) => (val1, val2),
Immediate::Scalar(..) => bug!("Got a scalar where a scalar pair was expected"),
Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"),
}
}
}
#[derive(Clone)]
pub struct ImmTy<'tcx, Prov: Provenance = AllocId> {
imm: Immediate<Prov>,
pub layout: TyAndLayout<'tcx>,
}
impl<Prov: Provenance> std::fmt::Display for ImmTy<'_, Prov> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn p<'a, 'tcx, Prov: Provenance>(
cx: FmtPrinter<'a, 'tcx>,
s: Scalar<Prov>,
ty: Ty<'tcx>,
) -> Result<FmtPrinter<'a, 'tcx>, std::fmt::Error> {
match s {
Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true),
Scalar::Ptr(ptr, _sz) => {
cx.pretty_print_const_pointer(ptr, ty)
}
}
}
ty::tls::with(|tcx| {
match self.imm {
Immediate::Scalar(s) => {
if let Some(ty) = tcx.lift(self.layout.ty) {
let cx = FmtPrinter::new(tcx, Namespace::ValueNS);
f.write_str(&p(cx, s, ty)?.into_buffer())?;
return Ok(());
}
write!(f, "{:x}: {}", s, self.layout.ty)
}
Immediate::ScalarPair(a, b) => {
write!(f, "({:x}, {:x}): {}", a, b, self.layout.ty)
}
Immediate::Uninit => {
write!(f, "uninit: {}", self.layout.ty)
}
}
})
}
}
impl<Prov: Provenance> std::fmt::Debug for ImmTy<'_, Prov> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ImmTy")
.field("imm", &self.imm)
.field("ty", &format_args!("{}", self.layout.ty))
.finish()
}
}
impl<'tcx, Prov: Provenance> std::ops::Deref for ImmTy<'tcx, Prov> {
type Target = Immediate<Prov>;
#[inline(always)]
fn deref(&self) -> &Immediate<Prov> {
&self.imm
}
}
impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
#[inline]
pub fn from_scalar(val: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
debug_assert!(layout.abi.is_scalar(), "`ImmTy::from_scalar` on non-scalar layout");
ImmTy { imm: val.into(), layout }
}
#[inline(always)]
pub fn from_immediate(imm: Immediate<Prov>, layout: TyAndLayout<'tcx>) -> Self {
debug_assert!(
match (imm, layout.abi) {
(Immediate::Scalar(..), Abi::Scalar(..)) => true,
(Immediate::ScalarPair(..), Abi::ScalarPair(..)) => true,
(Immediate::Uninit, _) if layout.is_sized() => true,
_ => false,
},
"immediate {imm:?} does not fit to layout {layout:?}",
);
ImmTy { imm, layout }
}
#[inline]
pub fn uninit(layout: TyAndLayout<'tcx>) -> Self {
debug_assert!(layout.is_sized(), "immediates must be sized");
ImmTy { imm: Immediate::Uninit, layout }
}
#[inline]
pub fn try_from_uint(i: impl Into<u128>, layout: TyAndLayout<'tcx>) -> Option<Self> {
Some(Self::from_scalar(Scalar::try_from_uint(i, layout.size)?, layout))
}
#[inline]
pub fn from_uint(i: impl Into<u128>, layout: TyAndLayout<'tcx>) -> Self {
Self::from_scalar(Scalar::from_uint(i, layout.size), layout)
}
#[inline]
pub fn try_from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Option<Self> {
Some(Self::from_scalar(Scalar::try_from_int(i, layout.size)?, layout))
}
#[inline]
pub fn from_int(i: impl Into<i128>, layout: TyAndLayout<'tcx>) -> Self {
Self::from_scalar(Scalar::from_int(i, layout.size), layout)
}
#[inline]
pub fn from_bool(b: bool, tcx: TyCtxt<'tcx>) -> Self {
let layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(tcx.types.bool)).unwrap();
Self::from_scalar(Scalar::from_bool(b), layout)
}
#[inline]
pub fn to_const_int(self) -> ConstInt {
assert!(self.layout.ty.is_integral());
let int = self.to_scalar().assert_int();
ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral())
}
fn offset_(&self, offset: Size, layout: TyAndLayout<'tcx>, cx: &impl HasDataLayout) -> Self {
let inner_val: Immediate<_> = match (**self, self.layout.abi) {
(Immediate::Uninit, _) => Immediate::Uninit,
_ if layout.is_zst() => Immediate::Uninit,
_ if matches!(layout.abi, Abi::Aggregate { .. })
&& matches!(&layout.fields, abi::FieldsShape::Arbitrary { offsets, .. } if offsets.len() == 0) =>
{
Immediate::Uninit
}
_ if layout.size == self.layout.size => {
assert_eq!(offset.bytes(), 0);
assert!(
match (self.layout.abi, layout.abi) {
(Abi::Scalar(..), Abi::Scalar(..)) => true,
(Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
_ => false,
},
"cannot project into {} immediate with equally-sized field {}\nouter ABI: {:#?}\nfield ABI: {:#?}",
self.layout.ty,
layout.ty,
self.layout.abi,
layout.abi,
);
**self
}
(Immediate::ScalarPair(a_val, b_val), Abi::ScalarPair(a, b)) => {
assert!(matches!(layout.abi, Abi::Scalar(..)));
Immediate::from(if offset.bytes() == 0 {
debug_assert_eq!(layout.size, a.size(cx));
a_val
} else {
debug_assert_eq!(offset, a.size(cx).align_to(b.align(cx).abi));
debug_assert_eq!(layout.size, b.size(cx));
b_val
})
}
_ => bug!("invalid field access on immediate {}, layout {:#?}", self, self.layout),
};
ImmTy::from_immediate(inner_val, layout)
}
}
impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
#[inline(always)]
fn meta(&self) -> MemPlaceMeta<Prov> {
debug_assert!(self.layout.is_sized()); MemPlaceMeta::None
}
fn offset_with_meta<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, Self> {
assert_matches!(meta, MemPlaceMeta::None); Ok(self.offset_(offset, layout, ecx))
}
fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone().into())
}
}
#[derive(Copy, Clone, Debug)]
pub(super) enum Operand<Prov: Provenance = AllocId> {
Immediate(Immediate<Prov>),
Indirect(MemPlace<Prov>),
}
#[derive(Clone)]
pub struct OpTy<'tcx, Prov: Provenance = AllocId> {
op: Operand<Prov>, pub layout: TyAndLayout<'tcx>,
pub align: Option<Align>,
}
impl<Prov: Provenance> std::fmt::Debug for OpTy<'_, Prov> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OpTy")
.field("op", &self.op)
.field("ty", &format_args!("{}", self.layout.ty))
.finish()
}
}
impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
#[inline(always)]
fn from(val: ImmTy<'tcx, Prov>) -> Self {
OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None }
}
}
impl<'tcx, Prov: Provenance> From<MPlaceTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
#[inline(always)]
fn from(mplace: MPlaceTy<'tcx, Prov>) -> Self {
OpTy {
op: Operand::Indirect(*mplace.mplace()),
layout: mplace.layout,
align: Some(mplace.align),
}
}
}
impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
#[inline(always)]
pub(super) fn op(&self) -> &Operand<Prov> {
&self.op
}
}
impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
#[inline]
fn meta(&self) -> MemPlaceMeta<Prov> {
match self.as_mplace_or_imm() {
Left(mplace) => mplace.meta(),
Right(_) => {
debug_assert!(self.layout.is_sized(), "unsized immediates are not a thing");
MemPlaceMeta::None
}
}
}
fn offset_with_meta<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, Self> {
match self.as_mplace_or_imm() {
Left(mplace) => Ok(mplace.offset_with_meta(offset, meta, layout, ecx)?.into()),
Right(imm) => {
debug_assert!(layout.is_sized(), "unsized immediates are not a thing");
assert_matches!(meta, MemPlaceMeta::None); Ok(imm.offset_(offset, layout, ecx).into())
}
}
}
fn to_op<'mir, M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone())
}
}
pub trait Readable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>>;
}
impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for OpTy<'tcx, Prov> {
#[inline(always)]
fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
self.as_mplace_or_imm()
}
}
impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
#[inline(always)]
fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
Left(self.clone())
}
}
impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for ImmTy<'tcx, Prov> {
#[inline(always)]
fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
Right(self.clone())
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn read_immediate_from_mplace_raw(
&self,
mplace: &MPlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
if mplace.layout.is_unsized() {
return Ok(None);
}
let Some(alloc) = self.get_place_alloc(mplace)? else {
return Ok(Some(ImmTy::uninit(mplace.layout)));
};
Ok(match mplace.layout.abi {
Abi::Scalar(abi::Scalar::Initialized { value: s, .. }) => {
let size = s.size(self);
assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
let scalar = alloc.read_scalar(
alloc_range(Size::ZERO, size),
matches!(s, abi::Pointer(_)),
)?;
Some(ImmTy::from_scalar(scalar, mplace.layout))
}
Abi::ScalarPair(
abi::Scalar::Initialized { value: a, .. },
abi::Scalar::Initialized { value: b, .. },
) => {
let (a_size, b_size) = (a.size(self), b.size(self));
let b_offset = a_size.align_to(b.align(self).abi);
assert!(b_offset.bytes() > 0); let a_val = alloc.read_scalar(
alloc_range(Size::ZERO, a_size),
matches!(a, abi::Pointer(_)),
)?;
let b_val = alloc.read_scalar(
alloc_range(b_offset, b_size),
matches!(b, abi::Pointer(_)),
)?;
Some(ImmTy::from_immediate(Immediate::ScalarPair(a_val, b_val), mplace.layout))
}
_ => {
None
}
})
}
pub fn read_immediate_raw(
&self,
src: &impl Readable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, Either<MPlaceTy<'tcx, M::Provenance>, ImmTy<'tcx, M::Provenance>>> {
Ok(match src.as_mplace_or_imm() {
Left(ref mplace) => {
if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
Right(val)
} else {
Left(mplace.clone())
}
}
Right(val) => Right(val),
})
}
#[inline(always)]
pub fn read_immediate(
&self,
op: &impl Readable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
if !matches!(
op.layout().abi,
Abi::Scalar(abi::Scalar::Initialized { .. })
| Abi::ScalarPair(abi::Scalar::Initialized { .. }, abi::Scalar::Initialized { .. })
) {
span_bug!(self.cur_span(), "primitive read not possible for type: {}", op.layout().ty);
}
let imm = self.read_immediate_raw(op)?.right().unwrap();
if matches!(*imm, Immediate::Uninit) {
throw_ub!(InvalidUninitBytes(None));
}
Ok(imm)
}
pub fn read_scalar(
&self,
op: &impl Readable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, Scalar<M::Provenance>> {
Ok(self.read_immediate(op)?.to_scalar())
}
pub fn read_pointer(
&self,
op: &impl Readable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
self.read_scalar(op)?.to_pointer(self)
}
pub fn read_target_usize(
&self,
op: &impl Readable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, u64> {
self.read_scalar(op)?.to_target_usize(self)
}
pub fn read_target_isize(
&self,
op: &impl Readable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, i64> {
self.read_scalar(op)?.to_target_isize(self)
}
pub fn read_str(&self, mplace: &MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx, &str> {
let len = mplace.len(self)?;
let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
Ok(str)
}
pub fn operand_to_simd(
&self,
op: &OpTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, u64)> {
assert!(op.layout.ty.is_simd());
match op.as_mplace_or_imm() {
Left(mplace) => self.mplace_to_simd(&mplace),
Right(imm) => match *imm {
Immediate::Uninit => {
throw_ub!(InvalidUninitBytes(None))
}
Immediate::Scalar(..) | Immediate::ScalarPair(..) => {
bug!("arrays/slices can never have Scalar/ScalarPair layout")
}
},
}
}
pub fn local_to_op(
&self,
frame: &Frame<'mir, 'tcx, M::Provenance, M::FrameExtra>,
local: mir::Local,
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let layout = self.layout_of_local(frame, local, layout)?;
let op = *frame.locals[local].access()?;
if matches!(op, Operand::Immediate(_)) {
if layout.is_unsized() {
throw_inval!(ConstPropNonsense);
}
}
Ok(OpTy { op, layout, align: Some(layout.align.abi) })
}
pub fn place_to_op(
&self,
place: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
match place.as_mplace_or_local() {
Left(mplace) => Ok(mplace.into()),
Right((frame, local, offset)) => {
debug_assert!(place.layout.is_sized()); let base = self.local_to_op(&self.stack()[frame], local, None)?;
let mut field = match offset {
Some(offset) => base.offset(offset, place.layout, self)?,
None => {
debug_assert_eq!(place.layout, base.layout);
base
}
};
field.align = Some(place.align);
Ok(field)
}
}
}
pub fn eval_place_to_op(
&self,
mir_place: mir::Place<'tcx>,
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let layout = if mir_place.projection.is_empty() { layout } else { None };
let mut op = self.local_to_op(self.frame(), mir_place.local, layout)?;
for elem in mir_place.projection.iter() {
op = self.project(&op, elem)?
}
trace!("eval_place_to_op: got {:?}", op);
if cfg!(debug_assertions) {
let normalized_place_ty = self.subst_from_current_frame_and_normalize_erasing_regions(
mir_place.ty(&self.frame().body.local_decls, *self.tcx).ty,
)?;
if !mir_assign_valid_types(
*self.tcx,
self.param_env,
self.layout_of(normalized_place_ty)?,
op.layout,
) {
span_bug!(
self.cur_span(),
"eval_place of a MIR place with type {} produced an interpreter operand with type {}",
normalized_place_ty,
op.layout.ty,
)
}
}
Ok(op)
}
#[inline]
pub fn eval_operand(
&self,
mir_op: &mir::Operand<'tcx>,
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
use rustc_middle::mir::Operand::*;
let op = match mir_op {
&Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?,
Constant(constant) => {
let c =
self.subst_from_current_frame_and_normalize_erasing_regions(constant.const_)?;
self.eval_mir_constant(&c, Some(constant.span), layout)?
}
};
trace!("{:?}: {:?}", mir_op, op);
Ok(op)
}
pub(crate) fn const_val_to_op(
&self,
val_val: mir::ConstValue<'tcx>,
ty: Ty<'tcx>,
layout: Option<TyAndLayout<'tcx>>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let adjust_scalar = |scalar| -> InterpResult<'tcx, _> {
Ok(match scalar {
Scalar::Ptr(ptr, size) => Scalar::Ptr(self.global_base_pointer(ptr)?, size),
Scalar::Int(int) => Scalar::Int(int),
})
};
let layout = from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty))?;
let op = match val_val {
mir::ConstValue::Indirect { alloc_id, offset } => {
let ptr = self.global_base_pointer(Pointer::new(alloc_id, offset))?;
Operand::Indirect(MemPlace::from_ptr(ptr.into()))
}
mir::ConstValue::Scalar(x) => Operand::Immediate(adjust_scalar(x)?.into()),
mir::ConstValue::ZeroSized => Operand::Immediate(Immediate::Uninit),
mir::ConstValue::Slice { data, meta } => {
let ptr = Pointer::new(self.tcx.reserve_and_set_memory_alloc(data), Size::ZERO);
Operand::Immediate(Immediate::new_slice(
self.global_base_pointer(ptr)?.into(),
meta,
self,
))
}
};
Ok(OpTy { op, layout, align: Some(layout.align.abi) })
}
}
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
mod size_asserts {
use super::*;
use rustc_data_structures::static_assert_size;
static_assert_size!(Immediate, 48);
static_assert_size!(ImmTy<'_>, 64);
static_assert_size!(Operand, 56);
static_assert_size!(OpTy<'_>, 80);
}