This commit is contained in:
playX18 2025-02-09 09:21:48 +07:00
parent 680261eec4
commit a65a043870
31 changed files with 7724 additions and 0 deletions

3
.cursorignore Normal file
View file

@ -0,0 +1,3 @@
# Add directories or file patterns to ignore during indexing (e.g. foo/ or *.csv)
target/*
*/target/*

1
.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target

1481
Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

4
Cargo.toml Normal file
View file

@ -0,0 +1,4 @@
[workspace]
members = ["vmkit", "vmkit-proc"]
default-members = ["vmkit"]
resolver = "2"

7
vmkit-proc/Cargo.lock generated Normal file
View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "vmkit-proc"
version = "0.1.0"

13
vmkit-proc/Cargo.toml Normal file
View file

@ -0,0 +1,13 @@
[package]
name = "vmkit-proc"
version = "0.1.0"
edition = "2021"
[dependencies]
proc-macro2 = "1.0.93"
quote = "1.0.38"
syn = { version = "2.0.98", features = ["full"] }
[lib]
proc-macro = true

3
vmkit-proc/README.md Normal file
View file

@ -0,0 +1,3 @@
# vmkit-proc
Various procedural macros which simplify the usage of VMKit.

6
vmkit-proc/src/lib.rs Normal file
View file

@ -0,0 +1,6 @@
#[proc_macro]
pub fn define_options(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
let _input = proc_macro2::TokenStream::from(input);
proc_macro::TokenStream::new()
}

3
vmkit-proc/src/main.rs Normal file
View file

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

1295
vmkit/Cargo.lock generated Normal file

File diff suppressed because it is too large Load diff

33
vmkit/Cargo.toml Normal file
View file

@ -0,0 +1,33 @@
[package]
name = "vmkit"
version = "0.1.0"
edition = "2021"
[dependencies]
atomic = "0.6.0"
bytemuck = "1.21.0"
cfg-if = "1.0.0"
clap = { version = "4.5.28", features = ["derive"] }
easy-bitfield = "0.1.0"
errno = "0.3.10"
libc = "0.2.169"
mmtk = { git = "https://github.com/mmtk/mmtk-core" }
parking_lot = "0.12.3"
rand = "0.9.0"
[features]
default = ["cooperative"]
# VMKit is built for use in cooperative runtime. Such runtime
# would be able to use write barriers and safepoints. Such environment
# must also provide precise object layout (stack can be uncooperative).
cooperative = ["mmtk/vo_bit", "mmtk/is_mmtk_object", "mmtk/vo_bit_access"]
# VMKit is built for use in full-precise runtime. Such runtime
# would be able to use precise write barriers and safepoints, object
# layout is fully precise.
full-precise = []
[target.'cfg(windows)'.dependencies]
winapi = { version = "0.3.9", features = ["everything"] }

134
vmkit/src/lib.rs Normal file
View file

@ -0,0 +1,134 @@
use std::{marker::PhantomData, sync::atomic::AtomicBool};
use mm::{aslr::aslr_vm_layout, traits::SlotExtra, MemoryManager};
use mmtk::{ MMTKBuilder, MMTK};
use threading::ThreadManager;
pub mod mm;
pub mod object_model;
pub mod options;
pub mod sync;
pub mod threading;
pub trait VirtualMachine: Sized + 'static + Send + Sync {
type ThreadContext: threading::ThreadContext<Self>;
type BlockAdapterList: threading::BlockAdapterList<Self>;
type Metadata: object_model::metadata::Metadata<Self>;
type Slot: SlotExtra;
const MAX_ALIGNMENT: usize = 16;
const MIN_ALIGNMENT: usize = 16;
/// Does this VM use conservative tracing? If `true` then VM can
/// query VO-bit (valid-object bit) to check if an object is live
/// during tracing work.
///
/// Note that this is distinct from conservative stack scanning. When
/// collecting roots VO-bits are always available.
///
/// Read more: [ObjectModel::NEED_VO_BITS_DURING_TRACING](mmtk::vm::ObjectModel::NEED_VO_BITS_DURING_TRACING).
///
/// # Note
///
/// - [`InternalPointer`](mm::conservative_roots::InternalPointer) can only be used when this is `true`.
#[cfg(feature = "cooperative")]
const CONSERVATIVE_TRACING: bool = false;
/// Get currently active VM instance.
///
/// # Notes
///
/// At the moment we assume only one active VM per process. This can be changed in the future once MMTk supports
/// instances. In that case this function can return active VM for current thread instead of one global instance.
fn get() -> &'static Self;
fn vmkit(&self) -> &VMKit<Self>;
/// Prepare for another round of root scanning in the same GC.
///
/// For details: [Scanning::prepare_for_roots_re_scanning](mmtk::vm::Scanning::prepare_for_roots_re_scanning)
fn prepare_for_roots_re_scanning();
/// MMTk calls this method at the first time during a collection that thread's stacks have been scanned. This can be used (for example) to clean up obsolete compiled methods that are no longer being executed.
fn notify_initial_thread_scan_complete(partial_scan: bool, tls: mmtk::util::VMWorkerThread);
/// Process weak references.
///
/// This function is called after a transitive closure is completed.
///
/// For details: [Scanning::process_weak_refs](mmtk::vm::Scanning::process_weak_refs)
fn process_weak_refs(
_worker: &mut mmtk::scheduler::GCWorker<MemoryManager<Self>>,
_tracer_context: impl mmtk::vm::ObjectTracerContext<MemoryManager<Self>>,
) -> bool {
false
}
fn forward_weak_refs(
_worker: &mut mmtk::scheduler::GCWorker<MemoryManager<Self>>,
_tracer_context: impl mmtk::vm::ObjectTracerContext<MemoryManager<Self>>,
);
/// Scan one mutator for stack roots.
///
/// For details: [Scanning::scan_roots_in_mutator_thread](mmtk::vm::Scanning::scan_roots_in_mutator_thread)
fn scan_roots_in_mutator_thread(
tls: mmtk::util::VMWorkerThread,
mutator: &'static mut mmtk::Mutator<MemoryManager<Self>>,
factory: impl mmtk::vm::RootsWorkFactory<<MemoryManager<Self> as mmtk::vm::VMBinding>::VMSlot>,
);
/// Scan VM-specific roots.
///
/// For details: [Scanning::scan_vm_specific_roots](mmtk::vm::Scanning::scan_vm_specific_roots)
fn scan_vm_specific_roots(
tls: mmtk::util::VMWorkerThread,
factory: impl mmtk::vm::RootsWorkFactory<<MemoryManager<Self> as mmtk::vm::VMBinding>::VMSlot>,
);
/// A hook for the VM to do work after forwarding objects.
fn post_forwarding(tls: mmtk::util::VMWorkerThread) {
let _ = tls;
}
fn schedule_finalization(tls: mmtk::util::VMWorkerThread) {
let _ = tls;
}
fn vm_live_bytes() -> usize {
0
}
fn out_of_memory(tls: mmtk::util::VMThread, err_kind: mmtk::util::alloc::AllocationError) {
let _ = tls;
let _ = err_kind;
eprintln!("Out of memory: {:?}", err_kind);
std::process::exit(1);
}
}
pub struct VMKit<VM: VirtualMachine> {
thread_manager: ThreadManager<VM>,
pub mmtk: MMTK<MemoryManager<VM>>,
pub(crate) collector_started: AtomicBool,
marker: PhantomData<VM>,
}
impl<VM: VirtualMachine> VMKit<VM> {
pub fn new(mut builder: MMTKBuilder) -> Self {
let vm_layout = aslr_vm_layout(&mut builder.options);
builder.set_vm_layout(vm_layout);
VMKit {
mmtk: builder.build(),
marker: PhantomData,
collector_started: AtomicBool::new(false),
thread_manager: ThreadManager::new(),
}
}
pub(crate) fn are_collector_threads_spawned(&self) -> bool {
self.collector_started.load(atomic::Ordering::Relaxed)
}
pub fn thread_manager(&self) -> &ThreadManager<VM> {
&self.thread_manager
}
}

167
vmkit/src/main.rs Normal file
View file

@ -0,0 +1,167 @@
use easy_bitfield::{FromBitfield, ToBitfield};
use mmtk::{util::options::PlanSelector, vm::slot::SimpleSlot, AllocationSemantics, MMTKBuilder};
use std::sync::OnceLock;
use vmkit::{
mm::{traits::ToSlot, MemoryManager},
object_model::metadata::{GCMetadata, Metadata, Trace},
threading::{GCBlockAdapter, Thread, ThreadContext},
VMKit, VirtualMachine,
};
struct TestContext;
impl ThreadContext<VM> for TestContext {
fn save_thread_state(&self) {}
fn scan_roots(&self, _factory: impl mmtk::vm::RootsWorkFactory<<VM as VirtualMachine>::Slot>) {}
fn scan_conservative_roots(
&self,
croots: &mut vmkit::mm::conservative_roots::ConservativeRoots,
) {
let _ = croots;
}
fn new(_collector_context: bool) -> Self {
Self
}
}
static VM_STORAGE: OnceLock<VM> = OnceLock::new();
impl VirtualMachine for VM {
const MAX_ALIGNMENT: usize = 16;
const MIN_ALIGNMENT: usize = 16;
type ThreadContext = TestContext;
type BlockAdapterList = (GCBlockAdapter, ());
type Metadata = &'static GCMetadata<VM>;
type Slot = SimpleSlot;
fn get() -> &'static Self {
VM_STORAGE.get().unwrap()
}
fn vmkit(&self) -> &VMKit<Self> {
&self.vmkit
}
fn prepare_for_roots_re_scanning() {}
fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: mmtk::util::VMWorkerThread) {}
fn scan_roots_in_mutator_thread(
_tls: mmtk::util::VMWorkerThread,
_mutator: &'static mut mmtk::Mutator<vmkit::mm::MemoryManager<Self>>,
_factory: impl mmtk::vm::RootsWorkFactory<
<vmkit::mm::MemoryManager<Self> as mmtk::vm::VMBinding>::VMSlot,
>,
) {
}
fn scan_vm_specific_roots(
_tls: mmtk::util::VMWorkerThread,
_factory: impl mmtk::vm::RootsWorkFactory<
<vmkit::mm::MemoryManager<Self> as mmtk::vm::VMBinding>::VMSlot,
>,
) {
}
fn forward_weak_refs(
_worker: &mut mmtk::scheduler::GCWorker<vmkit::mm::MemoryManager<Self>>,
_tracer_context: impl mmtk::vm::ObjectTracerContext<vmkit::mm::MemoryManager<Self>>,
) {
todo!()
}
}
struct VM {
vmkit: VMKit<Self>,
}
static METADATA: GCMetadata<VM> = GCMetadata {
instance_size: 48,
compute_size: None,
trace: Trace::TraceObject(|object, _tracer| {
println!("tracing {}", object.as_address());
}),
alignment: 16,
};
struct FooMeta;
impl Metadata<VM> for FooMeta {
const METADATA_BIT_SIZE: usize = 56;
fn gc_metadata(&self) -> &'static GCMetadata<VM> {
&METADATA
}
fn is_object(&self) -> bool {
false
}
fn from_object_reference(_reference: mmtk::util::ObjectReference) -> Self {
unreachable!()
}
fn to_object_reference(&self) -> Option<mmtk::util::ObjectReference> {
unreachable!()
}
}
impl ToSlot<SimpleSlot> for FooMeta {
fn to_slot(&self) -> Option<SimpleSlot> {
None
}
}
impl FromBitfield<u64> for FooMeta {
fn from_bitfield(_bitfield: u64) -> Self {
FooMeta
}
fn from_i64(_value: i64) -> Self {
FooMeta
}
}
impl ToBitfield<u64> for FooMeta {
fn to_bitfield(self) -> u64 {
0
}
fn one() -> Self {
FooMeta
}
fn zero() -> Self {
FooMeta
}
}
extern "C-unwind" fn handler(signum: libc::c_int) {
println!("signal {signum}");
println!("backtrace:\n{}", std::backtrace::Backtrace::force_capture());
std::process::exit(1);
}
fn main() {
unsafe {
libc::signal(libc::SIGSEGV, handler as usize);
}
let mut mmtk = MMTKBuilder::new();
mmtk.options.plan.set(PlanSelector::Immix);
mmtk.options.threads.set(1);
VM_STORAGE.get_or_init(|| VM {
vmkit: VMKit::new(mmtk),
});
Thread::<VM>::main(TestContext, || {
let tls = Thread::<VM>::current();
let my_obj = MemoryManager::allocate(tls, 48, 16, &METADATA, AllocationSemantics::Default);
println!("Allocated object at {}", my_obj.as_address());
MemoryManager::<VM>::request_gc();
println!("object {} at {:p}", my_obj.as_address(), &my_obj);
});
}

402
vmkit/src/mm.rs Normal file
View file

@ -0,0 +1,402 @@
use crate::{
object_model::{
header::{HeapObjectHeader, OBJECT_REF_OFFSET},
object::VMKitObject,
VMKitObjectModel,
},
threading::Thread,
VirtualMachine,
};
use easy_bitfield::{AtomicBitfieldContainer, ToBitfield};
use mmtk::{
util::{
alloc::{AllocatorSelector, BumpAllocator, ImmixAllocator},
metadata::side_metadata::GLOBAL_SIDE_METADATA_BASE_ADDRESS,
VMMutatorThread,
},
vm::{
slot::{Slot, UnimplementedMemorySlice},
VMBinding,
},
AllocationSemantics, BarrierSelector, MutatorContext,
};
use std::marker::PhantomData;
#[derive(Clone, Copy)]
pub struct MemoryManager<VM: VirtualMachine>(PhantomData<VM>);
impl<VM: VirtualMachine> Default for MemoryManager<VM> {
fn default() -> Self {
Self(PhantomData)
}
}
impl<VM: VirtualMachine> VMBinding for MemoryManager<VM> {
type VMMemorySlice = UnimplementedMemorySlice<VM::Slot>;
type VMSlot = VM::Slot;
type VMObjectModel = VMKitObjectModel<VM>;
type VMActivePlan = active_plan::VMKitActivePlan<VM>;
type VMCollection = collection::VMKitCollection<VM>;
type VMScanning = scanning::VMKitScanning<VM>;
type VMReferenceGlue = ref_glue::VMKitReferenceGlue<VM>;
const MAX_ALIGNMENT: usize = VM::MAX_ALIGNMENT;
const MIN_ALIGNMENT: usize = VM::MIN_ALIGNMENT;
}
pub mod active_plan;
pub mod aslr;
pub mod collection;
pub mod conservative_roots;
pub mod ref_glue;
pub mod scanning;
pub mod stack_bounds;
pub mod tlab;
pub mod traits;
impl<VM: VirtualMachine> MemoryManager<VM> {
pub extern "C-unwind" fn request_gc() -> bool {
let tls = Thread::<VM>::current();
mmtk::memory_manager::handle_user_collection_request(
&VM::get().vmkit().mmtk,
VMMutatorThread(tls.to_vm_thread()),
)
}
/// Allocate object with `size`, `alignment`, and `metadata` with specified `semantics`.
///
/// This function is a fast-path for allocation. If you allocate with `Default` semantics,
/// this function will first try to allocate through bump-pointer in TLAB. If other
/// semantic is specified or TLAB is empty (or unavailable) we invoke MMTK API directly.
#[inline]
pub extern "C-unwind" fn allocate(
thread: &Thread<VM>,
size: usize,
alignment: usize,
metadata: VM::Metadata,
mut semantics: AllocationSemantics,
) -> VMKitObject {
if semantics == AllocationSemantics::Default
&& size >= thread.max_non_los_default_alloc_bytes()
{
semantics = AllocationSemantics::Los;
}
// all allocator functions other than this actually invoke `flush_tlab` due to the fact
// that GC can happen inside them.
match semantics {
AllocationSemantics::Los => Self::allocate_los(thread, size, alignment, metadata),
AllocationSemantics::NonMoving => {
Self::allocate_nonmoving(thread, size, alignment, metadata)
}
AllocationSemantics::Immortal => {
Self::allocate_immortal(thread, size, alignment, metadata)
}
_ => unsafe {
let tlab = thread.tlab.get().as_mut().unwrap();
let object_start = tlab.allocate(size, alignment);
if !object_start.is_zero() {
object_start.store(HeapObjectHeader::<VM> {
metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()),
marker: PhantomData,
});
let object = VMKitObject::from_address(object_start + OBJECT_REF_OFFSET);
Self::set_vo_bit(object);
Self::refill_tlab(thread);
return object;
}
Self::allocate_slow(thread, size, alignment, metadata, semantics)
},
}
}
pub extern "C-unwind" fn allocate_los(
thread: &Thread<VM>,
size: usize,
alignment: usize,
metadata: VM::Metadata,
) -> VMKitObject {
debug_assert!(thread.id() == Thread::<VM>::current().id());
unsafe {
Self::flush_tlab(thread);
let object_start = mmtk::memory_manager::alloc(
thread.mutator(),
size,
alignment,
0,
AllocationSemantics::Los,
);
let object = VMKitObject::from_address(object_start + OBJECT_REF_OFFSET);
object_start.store(HeapObjectHeader::<VM> {
metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()),
marker: PhantomData,
});
Self::set_vo_bit(object);
Self::refill_tlab(thread);
object
}
}
pub extern "C-unwind" fn allocate_nonmoving(
thread: &Thread<VM>,
size: usize,
alignment: usize,
metadata: VM::Metadata,
) -> VMKitObject {
debug_assert!(thread.id() == Thread::<VM>::current().id());
unsafe {
Self::flush_tlab(thread);
let object_start = mmtk::memory_manager::alloc(
thread.mutator(),
size,
alignment,
0,
AllocationSemantics::NonMoving,
);
let object = VMKitObject::from_address(object_start + OBJECT_REF_OFFSET);
object_start.store(HeapObjectHeader::<VM> {
metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()),
marker: PhantomData,
});
Self::set_vo_bit(object);
Self::refill_tlab(thread);
object
}
}
pub extern "C-unwind" fn allocate_immortal(
thread: &Thread<VM>,
size: usize,
alignment: usize,
metadata: VM::Metadata,
) -> VMKitObject {
debug_assert!(thread.id() == Thread::<VM>::current().id());
unsafe {
Self::flush_tlab(thread);
let object_start = mmtk::memory_manager::alloc(
thread.mutator(),
size,
alignment,
0,
AllocationSemantics::Immortal,
);
let object = VMKitObject::from_address(object_start + OBJECT_REF_OFFSET);
object_start.store(HeapObjectHeader::<VM> {
metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()),
marker: PhantomData,
});
Self::set_vo_bit(object);
Self::refill_tlab(thread);
object
}
}
/// Allocate object in a slowpath. This will reset TLAB and invoke MMTK directly to allocate. This
/// function potentially triggers GC or allocates more heap memory if necessary.
#[inline(never)]
#[cold]
pub extern "C-unwind" fn allocate_slow(
thread: &Thread<VM>,
size: usize,
alignment: usize,
metadata: VM::Metadata,
semantics: AllocationSemantics,
) -> VMKitObject {
unsafe {
Self::flush_tlab(thread);
let object_start =
mmtk::memory_manager::alloc_slow(thread.mutator(), size, alignment, 0, semantics);
object_start.store(HeapObjectHeader::<VM> {
metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()),
marker: PhantomData,
});
let object = VMKitObject::from_address(object_start + OBJECT_REF_OFFSET);
Self::set_vo_bit(object);
Self::refill_tlab(thread);
object
}
}
#[inline(never)]
pub extern "C-unwind" fn set_vo_bit(object: VMKitObject) {
#[cfg(feature = "cooperative")]
unsafe {
let meta = mmtk::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_ADDR
+ (object.as_address() >> 6);
let byte = meta.load::<u8>();
let mask = 1 << ((object.as_address() >> 3) & 7);
let new_byte = byte | mask;
meta.store::<u8>(new_byte);
}
}
/// Flushes the TLAB by storing the TLAB to underlying allocator.
///
/// Leaves TLAB cursor set to zero. Invoke `refill_tlab` to rebind the TLAB.
///
///
/// # Safety
///
/// Must be invoked in the context of current thread and before `alloc()` calls into MMTK
pub unsafe fn flush_tlab(thread: &Thread<VM>) {
let tlab = thread.tlab.get().as_mut().unwrap();
let (cursor, limit) = tlab.take();
if cursor.is_zero() {
assert!(limit.is_zero());
return;
}
let selector = mmtk::memory_manager::get_allocator_mapping(
&VM::get().vmkit().mmtk,
AllocationSemantics::Default,
);
match selector {
AllocatorSelector::Immix(_) => {
let allocator = thread
.mutator_unchecked()
.allocator_impl_mut::<ImmixAllocator<Self>>(selector);
allocator.bump_pointer.reset(cursor, limit);
}
AllocatorSelector::BumpPointer(_) => {
let allocator = thread
.mutator_unchecked()
.allocator_impl_mut::<BumpAllocator<Self>>(selector);
allocator.bump_pointer.reset(cursor, limit);
}
_ => {
assert!(
cursor.is_zero() && limit.is_zero(),
"currently selected plan has no TLAB"
);
}
}
}
pub unsafe fn refill_tlab(thread: &Thread<VM>) {
let tlab = thread.tlab.get().as_mut().unwrap();
let selector = mmtk::memory_manager::get_allocator_mapping(
&VM::get().vmkit().mmtk,
AllocationSemantics::Default,
);
match selector {
AllocatorSelector::Immix(_) => {
let allocator = thread
.mutator()
.allocator_impl::<ImmixAllocator<Self>>(selector);
let (cursor, limit) = (allocator.bump_pointer.cursor, allocator.bump_pointer.limit);
tlab.rebind(cursor, limit);
}
AllocatorSelector::BumpPointer(_) => {
let allocator = thread
.mutator()
.allocator_impl::<BumpAllocator<Self>>(selector);
let (cursor, limit) = (allocator.bump_pointer.cursor, allocator.bump_pointer.limit);
tlab.rebind(cursor, limit);
}
_ => {}
}
}
/// The write barrier by MMTk. This is a *post* write barrier, which we expect a binding to call
/// *after* it modifies an object. For performance reasons, a VM should implement the write barrier
/// fast-path on their side rather than just calling this function.
///
/// For a correct barrier implementation, a VM binding needs to choose one of the following options:
/// * Use subsuming barrier `object_reference_write`
/// * Use both `object_reference_write_pre` and `object_reference_write_post`, or both, if the binding has difficulty delegating the store to mmtk-core with the subsuming barrier.
/// * Implement fast-path on the VM side, and call the generic api `object_reference_write_slow` as barrier slow-path call.
/// * Implement fast-path on the VM side, and do a specialized slow-path call.
///
/// Arguments:
/// * `thread`: a current thread.
/// * `src`: The modified source object.
/// * `slot`: The location of the field to be modified.
/// * `target`: The target for the write operation. `NULL` if the slot no longer hold an object
/// reference after the write operation. This may happen when writing a `null` reference, a small
/// integers, or a special value such as`true`, `false`, `undefined`, etc., into the slot.
pub fn object_reference_write_post(
thread: &Thread<VM>,
src: VMKitObject,
slot: VM::Slot,
target: VMKitObject,
) {
match thread.barrier() {
BarrierSelector::ObjectBarrier => unsafe {
let addr = src.as_address();
let meta_addr = GLOBAL_SIDE_METADATA_BASE_ADDRESS + (addr >> 6);
let shift = (addr >> 3) & 0b111;
let byte_val = meta_addr.load::<u8>();
if (byte_val >> shift) & 1 == 1 {
thread.mutator().barrier().object_reference_write_slow(
src.as_object_unchecked(),
slot,
if target.is_null() {
None
} else {
Some(target.as_object_unchecked())
},
);
}
},
BarrierSelector::NoBarrier => {}
}
}
/// The write barrier by MMTk. This is a *pre* write barrier, which we expect a binding to call
/// *before* it modifies an object. For performance reasons, a VM should implement the write barrier
/// fast-path on their side rather than just calling this function.
///
/// For a correct barrier implementation, a VM binding needs to choose one of the following options:
/// * Use subsuming barrier `object_reference_write`
/// * Use both `object_reference_write_pre` and `object_reference_write_post`, or both, if the binding has difficulty delegating the store to mmtk-core with the subsuming barrier.
/// * Implement fast-path on the VM side, and call the generic api `object_reference_write_slow` as barrier slow-path call.
/// * Implement fast-path on the VM side, and do a specialized slow-path call.
///
/// Arguments:
/// * `thread`: a current thread.
/// * `src`: The modified source object.
/// * `slot`: The location of the field to be modified.
/// * `target`: The target for the write operation. `NULL` if the slot did not hold an object
/// reference before the write operation. For example, the slot may be holding a `null`
/// reference, a small integer, or special values such as `true`, `false`, `undefined`, etc.
pub fn object_reference_write_pre(
thread: &Thread<VM>,
src: VMKitObject,
slot: VM::Slot,
target: VMKitObject,
) {
let _ = thread;
let _ = src;
let _ = slot;
let _ = target;
}
pub fn object_reference_write(
thread: &Thread<VM>,
src: VMKitObject,
slot: VM::Slot,
target: VMKitObject,
) {
assert!(target.is_not_null());
Self::object_reference_write_pre(thread, src, slot, target);
unsafe {
slot.store(target.as_object_unchecked());
}
Self::object_reference_write_post(thread, src, slot, target);
}
}

View file

@ -0,0 +1,51 @@
use std::marker::PhantomData;
use mmtk::vm::*;
use crate::{mm::MemoryManager, threading::Thread, VirtualMachine};
pub struct VMKitActivePlan<VM: VirtualMachine>(PhantomData<VM>);
impl<VM: VirtualMachine> ActivePlan<MemoryManager<VM>> for VMKitActivePlan<VM> {
fn is_mutator(tls: mmtk::util::VMThread) -> bool {
Thread::<VM>::from_vm_thread(tls).active_mutator_context()
}
fn mutator(tls: mmtk::util::VMMutatorThread) -> &'static mut mmtk::Mutator<MemoryManager<VM>> {
// SAFETY: `mutator()` is invoked by MMTk only when all threads are suspended for GC or
// MMTk is in allocation slowpath in *current* thread.
// At this point no other thread can access this thread's context.
unsafe { Thread::<VM>::from_vm_mutator_thread(tls).mutator_unchecked() }
}
fn mutators<'a>() -> Box<dyn Iterator<Item = &'a mut mmtk::Mutator<MemoryManager<VM>>> + 'a> {
let vm = VM::get();
Box::new(vm.vmkit().thread_manager()
.threads()
.filter(|t| t.active_mutator_context())
.map(|t|
// SAFETY: `mutators()` is invoked by MMTk only when all threads are suspended for GC.
// At this point no other thread can access this thread's context.
unsafe { t.mutator_unchecked() }))
}
fn number_of_mutators() -> usize {
let vm = VM::get();
vm.vmkit().thread_manager()
.threads()
.filter(|t| t.active_mutator_context())
.count()
}
fn vm_trace_object<Q: mmtk::ObjectQueue>(
_queue: &mut Q,
_object: mmtk::util::ObjectReference,
_worker: &mut mmtk::scheduler::GCWorker<MemoryManager<VM>>,
) -> mmtk::util::ObjectReference {
todo!()
}
}

95
vmkit/src/mm/aslr.rs Normal file
View file

@ -0,0 +1,95 @@
//! Address space layout randomization for MMTk.
use mmtk::util::{
conversions::raw_align_down, heap::vm_layout::{VMLayout, BYTES_IN_CHUNK}, options::{GCTriggerSelector, Options}, Address
};
use rand::Rng;
use crate::options::OPTIONS;
fn get_random_mmap_addr() -> Address {
let mut rng = rand::rng();
let uniform = rand::distr::Uniform::new(0, usize::MAX).unwrap();
let mut raw_addr = rng.sample(uniform);
raw_addr = raw_align_down(raw_addr, BYTES_IN_CHUNK);
raw_addr &= 0x3FFFFFFFF000;
unsafe { Address::from_usize(raw_addr) }
}
fn is_address_maybe_unmapped(addr: Address, size: usize) -> bool {
#[cfg(unix)]
{
unsafe {
let res = libc::msync(addr.to_mut_ptr(), size, libc::MS_ASYNC);
if res == -1 && errno::errno().0 == libc::ENOMEM {
return true;
}
}
}
false
}
pub fn aslr_vm_layout(mmtk_options: &mut Options) -> VMLayout {
let mut vm_layout = VMLayout::default();
let options = &*OPTIONS;
if options.compressed_pointers {
vm_layout.heap_start = unsafe { Address::from_usize(0x4000_0000) };
vm_layout.heap_end = vm_layout.heap_start + 32usize * 1024 * 1024
}
let size = if options.compressed_pointers {
if options.tag_compressed_pointers {
32 * 1024 * 1024
} else {
16 * 1024 * 1024
}
} else {
options.max_heap_size
};
if options.aslr {
vm_layout.force_use_contiguous_spaces = false;
loop {
let start = get_random_mmap_addr().align_down(BYTES_IN_CHUNK);
if !is_address_maybe_unmapped(start, size) {
continue;
}
let end = (start + size).align_up(BYTES_IN_CHUNK);
vm_layout.heap_start = start;
vm_layout.heap_end = end;
if !options.compressed_pointers {
vm_layout.log_address_space = if cfg!(target_pointer_width = "32") {
31
} else {
47
};
vm_layout.log_space_extent = size.trailing_zeros() as usize - 1;
}
break;
}
}
if options.compressed_pointers {
vm_layout.log_address_space = 36;
vm_layout.log_space_extent = 31;
}
mmtk_options.gc_trigger.set(GCTriggerSelector::DynamicHeapSize(
options.min_heap_size,
options.max_heap_size,
));
vm_layout
}

View file

@ -0,0 +1,87 @@
use std::marker::PhantomData;
use crate::{
mm::MemoryManager,
threading::{GCBlockAdapter, Thread},
VirtualMachine,
};
use mmtk::{
util::VMWorkerThread,
vm::{Collection, GCThreadContext},
};
pub struct VMKitCollection<VM: VirtualMachine>(PhantomData<VM>);
impl<VM: VirtualMachine> Collection<MemoryManager<VM>> for VMKitCollection<VM> {
fn stop_all_mutators<F>(_tls: mmtk::util::VMWorkerThread, mut mutator_visitor: F)
where
F: FnMut(&'static mut mmtk::Mutator<MemoryManager<VM>>),
{
let vmkit = VM::get().vmkit();
let mutators = vmkit.thread_manager().block_all_mutators_for_gc();
for mutator in mutators {
unsafe {
// reset TLAb if there's any.
MemoryManager::flush_tlab(&mutator);
mutator_visitor(mutator.mutator_unchecked());
}
}
}
fn block_for_gc(tls: mmtk::util::VMMutatorThread) {
let tls = Thread::<VM>::from_vm_mutator_thread(tls);
tls.block_unchecked::<GCBlockAdapter>(true);
}
fn resume_mutators(_tls: mmtk::util::VMWorkerThread) {
let vmkit = VM::get().vmkit();
vmkit.thread_manager().unblock_all_mutators_for_gc();
}
fn create_gc_trigger() -> Box<dyn mmtk::util::heap::GCTriggerPolicy<MemoryManager<VM>>> {
todo!()
}
fn is_collection_enabled() -> bool {
true
}
fn out_of_memory(_tls: mmtk::util::VMThread, _err_kind: mmtk::util::alloc::AllocationError) {
todo!()
}
fn post_forwarding(_tls: mmtk::util::VMWorkerThread) {
VM::post_forwarding(_tls);
}
fn schedule_finalization(_tls: mmtk::util::VMWorkerThread) {
}
fn spawn_gc_thread(
_tls: mmtk::util::VMThread,
ctx: mmtk::vm::GCThreadContext<MemoryManager<VM>>,
) {
let thread = Thread::<VM>::for_collector();
match ctx {
GCThreadContext::Worker(worker_ctx) => {
thread.start(move || {
let vm = VM::get();
let tls = Thread::<VM>::current().to_vm_thread();
mmtk::memory_manager::start_worker(
&vm.vmkit().mmtk,
VMWorkerThread(tls),
worker_ctx,
);
});
}
}
}
fn vm_live_bytes() -> usize {
VM::vm_live_bytes()
}
}

View file

@ -0,0 +1,203 @@
use std::{
hash::Hash,
marker::PhantomData,
ops::{Deref, DerefMut},
ptr::NonNull,
};
use mmtk::{
util::{Address, ObjectReference},
vm::{slot::Slot, RootsWorkFactory},
};
use crate::{object_model::object::VMKitObject, options::OPTIONS, VirtualMachine};
pub struct ConservativeRoots {
pub roots: Vec<ObjectReference>,
pub internal_pointer_limit: usize,
}
impl ConservativeRoots {
/// Add a pointer to conservative root set.
///
/// If pointer is not in the heap, this function does nothing.
pub fn add_pointer(&mut self, pointer: Address) {
let starting_address = mmtk::memory_manager::starting_heap_address();
let ending_address = mmtk::memory_manager::last_heap_address();
if pointer < starting_address || pointer > ending_address {
return;
}
let Some(start) = mmtk::memory_manager::find_object_from_internal_pointer(
pointer,
self.internal_pointer_limit,
) else {
return;
};
self.roots.push(start);
}
/// Add all pointers in the span to the roots.
///
/// # SAFETY
///
/// `start` and `end` must be valid addresses.
pub unsafe fn add_span(&mut self, mut start: Address, mut end: Address) {
if start > end {
std::mem::swap(&mut start, &mut end);
}
let mut current = start;
while current < end {
let addr = current.load::<Address>();
self.add_pointer(addr);
current = current.add(size_of::<Address>());
}
}
pub fn new() -> Self {
Self {
roots: Vec::new(),
internal_pointer_limit: OPTIONS.interior_pointer_max_bytes,
}
}
pub fn add_to_factory<SL: Slot>(&mut self, factory: &mut impl RootsWorkFactory<SL>) {
factory.create_process_pinning_roots_work(std::mem::take(&mut self.roots));
}
}
/// Pointer to some part of an heap object.
/// This type can only be used when `cooperative` feature is enabled and [`VM::CONSERVATIVE_TRACING`](VirtualMachine::CONSERVATIVE_TRACING) is `true`.
pub struct InternalPointer<T, VM: VirtualMachine> {
address: Address,
_marker: std::marker::PhantomData<(NonNull<T>, &'static VM)>,
}
unsafe impl<T: Send, VM: VirtualMachine> Send for InternalPointer<T, VM> {}
unsafe impl<T: Sync, VM: VirtualMachine> Sync for InternalPointer<T, VM> {}
impl<T, VM: VirtualMachine> InternalPointer<T, VM> {
pub fn new(address: Address) -> Self {
if !cfg!(feature = "cooperative") {
unreachable!("Internal pointers are not supported in precise mode");
}
debug_assert!(
mmtk::memory_manager::find_object_from_internal_pointer(
address,
OPTIONS.interior_pointer_max_bytes
)
.is_some(),
"Internal pointer is not in the heap"
);
assert!(VM::CONSERVATIVE_TRACING, "Internal pointers are not supported without VM::CONSERVATIVE_TRACING set to true");
Self {
address,
_marker: std::marker::PhantomData,
}
}
pub fn as_address(&self) -> Address {
self.address
}
/// Get original object from the pointer.
///
/// # Panics
///
/// Panics if the pointer is not in the heap.
pub fn object(&self) -> VMKitObject {
mmtk::memory_manager::find_object_from_internal_pointer(self.address, OPTIONS.interior_pointer_max_bytes).unwrap().into()
}
/// Return offset from the object start.
pub fn offset(&self) -> usize {
self.address.as_usize() - self.object().as_address().as_usize()
}
}
impl<T, VM: VirtualMachine> Deref for InternalPointer<T, VM> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { &*self.address.as_ref() }
}
}
impl<T, VM: VirtualMachine> DerefMut for InternalPointer<T, VM> {
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { &mut *self.address.as_mut_ref() }
}
}
/// Identical to [`InternalPointer`] but does not rely on VO-bits being enabled.
///
/// Stores original object and offset instead. This makes
/// this struct "fat" as its a 16-byte pointer on 64-bit systems compared to 8-bytes
/// of [`InternalPointer`]. This type can be used in precise mode.
pub struct FatInternalPointer<T, VM: VirtualMachine> {
offset: usize,
object: VMKitObject,
marker: PhantomData<(NonNull<T>, &'static VM)>,
}
unsafe impl<T: Send, VM: VirtualMachine> Send for FatInternalPointer<T, VM> {}
unsafe impl<T: Sync, VM: VirtualMachine> Sync for FatInternalPointer<T, VM> {}
impl<T, VM: VirtualMachine> Deref for FatInternalPointer<T, VM> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { self.object.as_address().add(self.offset).as_ref() }
}
}
impl<T, VM: VirtualMachine> DerefMut for FatInternalPointer<T, VM> {
fn deref_mut(&mut self) -> &mut Self::Target {
unsafe { self.object.as_address().add(self.offset).as_mut_ref() }
}
}
impl<T, VM: VirtualMachine> Clone for FatInternalPointer<T, VM> {
fn clone(&self) -> Self {
Self {
offset: self.offset,
object: self.object,
marker: PhantomData,
}
}
}
impl<T, VM: VirtualMachine> PartialEq for FatInternalPointer<T, VM> {
fn eq(&self, other: &Self) -> bool {
self.offset == other.offset && self.object == other.object
}
}
impl<T, VM: VirtualMachine> Eq for FatInternalPointer<T, VM> {}
impl<T, VM: VirtualMachine> Hash for FatInternalPointer<T, VM> {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.offset.hash(state);
self.object.hashcode::<VM>().hash(state);
}
}
impl<T, VM: VirtualMachine> FatInternalPointer<T, VM> {
pub fn new(object: VMKitObject, offset: usize) -> Self {
Self {
offset,
object,
marker: PhantomData,
}
}
pub fn offset(&self) -> usize {
self.offset
}
pub fn object(&self) -> VMKitObject {
self.object
}
}

49
vmkit/src/mm/ref_glue.rs Normal file
View file

@ -0,0 +1,49 @@
#![allow(dead_code, unused_variables)]
use std::marker::PhantomData;
use mmtk::vm::{Finalizable, ReferenceGlue};
use crate::{object_model::object::VMKitObject, VirtualMachine};
use super::MemoryManager;
pub struct VMKitReferenceGlue<VM: VirtualMachine>(PhantomData<VM>);
impl<VM: VirtualMachine> ReferenceGlue<MemoryManager<VM>> for VMKitReferenceGlue<VM> {
type FinalizableType = VMKitObject;
fn clear_referent(new_reference: mmtk::util::ObjectReference) {
}
fn enqueue_references(references: &[mmtk::util::ObjectReference], tls: mmtk::util::VMWorkerThread) {
}
fn get_referent(object: mmtk::util::ObjectReference) -> Option<mmtk::util::ObjectReference> {
todo!()
}
fn set_referent(reff: mmtk::util::ObjectReference, referent: mmtk::util::ObjectReference) {
todo!()
}
}
impl Finalizable for VMKitObject {
fn get_reference(&self) -> mmtk::util::ObjectReference {
todo!()
}
fn keep_alive<E: mmtk::scheduler::ProcessEdgesWork>(&mut self, trace: &mut E) {
}
fn set_reference(&mut self, object: mmtk::util::ObjectReference) {
}
}

154
vmkit/src/mm/scanning.rs Normal file
View file

@ -0,0 +1,154 @@
use std::marker::PhantomData;
use crate::{
mm::MemoryManager,
object_model::{
metadata::{Metadata, Trace},
object::VMKitObject,
},
options::OPTIONS,
threading::{Thread, ThreadContext},
VirtualMachine,
};
use mmtk::{
vm::{slot::Slot, ObjectTracer, Scanning, SlotVisitor},
MutatorContext,
};
use super::{conservative_roots::ConservativeRoots, traits::ToSlot};
pub struct VMKitScanning<VM: VirtualMachine>(PhantomData<VM>);
impl<VM: VirtualMachine> Scanning<MemoryManager<VM>> for VMKitScanning<VM> {
fn forward_weak_refs(
_worker: &mut mmtk::scheduler::GCWorker<MemoryManager<VM>>,
_tracer_context: impl mmtk::vm::ObjectTracerContext<MemoryManager<VM>>,
) {
VM::forward_weak_refs(_worker, _tracer_context);
}
fn notify_initial_thread_scan_complete(partial_scan: bool, tls: mmtk::util::VMWorkerThread) {
VM::notify_initial_thread_scan_complete(partial_scan, tls);
}
fn prepare_for_roots_re_scanning() {
VM::prepare_for_roots_re_scanning();
}
fn process_weak_refs(
_worker: &mut mmtk::scheduler::GCWorker<MemoryManager<VM>>,
_tracer_context: impl mmtk::vm::ObjectTracerContext<MemoryManager<VM>>,
) -> bool {
VM::process_weak_refs(_worker, _tracer_context)
}
fn scan_object<
SV: mmtk::vm::SlotVisitor<<MemoryManager<VM> as mmtk::vm::VMBinding>::VMSlot>,
>(
_tls: mmtk::util::VMWorkerThread,
object: mmtk::util::ObjectReference,
slot_visitor: &mut SV,
) {
let object = VMKitObject::from(object);
let metadata = object.header::<VM>().metadata();
if metadata.is_object() {
slot_visitor.visit_slot(metadata.to_slot().expect("Object is not a slot"));
}
let gc_metadata = metadata.gc_metadata();
let trace = &gc_metadata.trace;
match trace {
Trace::ScanSlots(fun) => {
fun(object, slot_visitor);
}
Trace::None => (),
Trace::TraceObject(_) => {
unreachable!("TraceObject is not supported for scanning");
}
}
}
fn scan_object_and_trace_edges<OT: mmtk::vm::ObjectTracer>(
_tls: mmtk::util::VMWorkerThread,
object: mmtk::util::ObjectReference,
object_tracer: &mut OT,
) {
let object = VMKitObject::from(object);
let metadata = object.header::<VM>().metadata();
let gc_metadata = metadata.gc_metadata();
let trace = &gc_metadata.trace;
match trace {
Trace::ScanSlots(fun) => {
// wrap object tracer in a trait that implements SlotVisitor
// but actually traces the object directly.
let mut visitor = TraceSlotVisitor::<VM, OT> {
ot: object_tracer,
marker: PhantomData,
};
fun(object, &mut visitor);
}
Trace::None => (),
Trace::TraceObject(fun) => {
fun(object, object_tracer);
}
}
}
fn scan_roots_in_mutator_thread(
_tls: mmtk::util::VMWorkerThread,
mutator: &'static mut mmtk::Mutator<MemoryManager<VM>>,
mut factory: impl mmtk::vm::RootsWorkFactory<VM::Slot>,
) {
let tls = Thread::<VM>::from_vm_mutator_thread(mutator.get_tls());
tls.context.scan_roots(factory.clone());
if OPTIONS.conservative_stacks {
let mut croots = ConservativeRoots::new();
let bounds = *tls.stack_bounds();
unsafe { croots.add_span(bounds.origin(), bounds.end()) };
tls.context.scan_conservative_roots(&mut croots);
croots.add_to_factory(&mut factory);
}
}
fn scan_vm_specific_roots(
tls: mmtk::util::VMWorkerThread,
factory: impl mmtk::vm::RootsWorkFactory<VM::Slot>,
) {
VM::scan_vm_specific_roots(tls, factory);
}
fn support_slot_enqueuing(
_tls: mmtk::util::VMWorkerThread,
object: mmtk::util::ObjectReference,
) -> bool {
let object = VMKitObject::from(object);
let metadata = object.header::<VM>().metadata();
matches!(metadata.gc_metadata().trace, Trace::ScanSlots(_))
&& (!metadata.is_object() || metadata.to_slot().is_some())
}
fn supports_return_barrier() -> bool {
false
}
}
struct TraceSlotVisitor<'a, VM: VirtualMachine, OT: ObjectTracer> {
ot: &'a mut OT,
marker: PhantomData<VM>,
}
impl<'a, VM: VirtualMachine, OT: ObjectTracer> SlotVisitor<VM::Slot>
for TraceSlotVisitor<'a, VM, OT>
{
fn visit_slot(&mut self, slot: VM::Slot) {
let value = slot.load();
match value {
Some(object) => {
let object = self.ot.trace_object(object);
slot.store(object);
}
None => (),
}
}
}

View file

@ -0,0 +1,272 @@
use std::ptr::null_mut;
use libc::pthread_attr_destroy;
use mmtk::util::Address;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct StackBounds {
origin: Address,
bound: Address,
}
impl StackBounds {
// Default reserved zone size
pub const DEFAULT_RESERVED_ZONE: usize = 64 * 1024;
pub const fn empty_bounds() -> Self {
Self {
origin: Address::ZERO,
bound: Address::ZERO,
}
}
pub fn current_thread_stack_bounds() -> Self {
let result = Self::current_thread_stack_bounds_internal();
result.check_consistency();
result
}
pub fn origin(&self) -> Address {
debug_assert!(!self.origin.is_zero());
self.origin
}
pub fn end(&self) -> Address {
debug_assert!(!self.bound.is_zero());
self.bound
}
pub fn size(&self) -> usize {
self.origin - self.bound
}
pub fn is_empty(&self) -> bool {
self.origin.is_zero()
}
pub fn contains(&self, p: Address) -> bool {
if self.is_empty() {
return false;
}
self.origin >= p && p > self.bound
}
pub fn recursion_limit(&self, min_reserved_zone: usize) -> Address {
self.check_consistency();
self.bound + min_reserved_zone
}
pub fn recursion_limit_with_params(
&self,
start_of_user_stack: Address,
max_user_stack: usize,
reserved_zone_size: usize,
) -> Address {
self.check_consistency();
let reserved_zone_size = reserved_zone_size.min(max_user_stack);
let mut max_user_stack_with_reserved_zone = max_user_stack - reserved_zone_size;
let end_of_stack_with_reserved_zone = self.bound.add(reserved_zone_size);
if start_of_user_stack < end_of_stack_with_reserved_zone {
return end_of_stack_with_reserved_zone;
}
let available_user_stack =
start_of_user_stack.get_offset(end_of_stack_with_reserved_zone) as usize;
if max_user_stack_with_reserved_zone > available_user_stack {
max_user_stack_with_reserved_zone = available_user_stack;
}
start_of_user_stack - max_user_stack_with_reserved_zone
}
pub fn with_soft_origin(&self, origin: Address) -> Self {
assert!(self.contains(origin));
Self {
origin,
bound: self.bound,
}
}
fn check_consistency(&self) {
#[cfg(debug_assertions)]
{
let current_position = current_stack_pointer();
assert_ne!(self.origin, self.bound);
assert!(current_position < self.origin && current_position > self.bound);
}
}
pub fn is_growing_downwards(&self) -> bool {
assert!(!self.origin.is_zero() && !self.bound.is_zero());
self.bound <= self.origin
}
}
#[cfg(unix)]
impl StackBounds {
pub unsafe fn new_thread_stack_bounds(handle: libc::pthread_t) -> Self {
#[cfg(target_vendor = "apple")]
unsafe {
let origin = libc::pthread_get_stackaddr_np(handle);
let size = libc::pthread_get_stacksize_np(handle);
let bound = origin.byte_sub(size);
Self {
origin: Address::from_ptr(origin),
bound: Address::from_ptr(bound),
}
}
#[cfg(target_os = "openbsd")]
{
let mut stack: std::mem::MaybeUninit<libc::stack_t> = std::mem::MaybeUninit::zeroed();
unsafe {
libc::pthread_stackseg_np(handle, stack.as_mut_ptr() as _);
}
let stack = unsafe { stack.assume_init() };
let origin = Address::from_ptr(stack.ss_sp);
let bound = origin - stack.ss_size as usize;
Self { origin, bound }
}
#[cfg(all(unix, not(target_vendor = "apple"), not(target_os = "openbsd")))]
{
let mut bound = null_mut();
let mut stack_size = 0;
let mut sattr: std::mem::MaybeUninit<libc::pthread_attr_t> =
std::mem::MaybeUninit::zeroed();
unsafe {
libc::pthread_attr_init(sattr.as_mut_ptr() as _);
#[cfg(target_os = "netbsd")]
{
libc::pthread_attr_get_np(handle, sattr.as_mut_ptr() as _);
}
#[cfg(not(target_os = "netbsd"))]
{
libc::pthread_getattr_np(handle, sattr.as_mut_ptr() as _);
}
libc::pthread_attr_getstack(sattr.assume_init_mut(), &mut bound, &mut stack_size);
pthread_attr_destroy(sattr.assume_init_mut());
let bound = Address::from_ptr(bound);
let origin = bound + stack_size;
Self { origin, bound }
}
}
}
fn current_thread_stack_bounds_internal() -> Self {
let ret = unsafe { Self::new_thread_stack_bounds(libc::pthread_self()) };
/*#[cfg(target_os = "linux")]
unsafe {
// on glibc, pthread_attr_getstack will generally return the limit size (minus a guard page)
// for the main thread; this is however not necessarily always true on every libc - for example
// on musl, it will return the currently reserved size - since the stack bounds are expected to
// be constant (and they are for every thread except main, which is allowed to grow), check
// resource limits and use that as the boundary instead (and prevent stack overflows).
if libc::getpid() == libc::syscall(libc::SYS_gettid) as libc::pid_t {
let origin = ret.origin();
let mut limit: std::mem::MaybeUninit<libc::rlimit> =
std::mem::MaybeUninit::zeroed();
libc::getrlimit(libc::RLIMIT_STACK, limit.as_mut_ptr() as _);
let limit = limit.assume_init();
let mut size = limit.rlim_cur;
if size == libc::RLIM_INFINITY {
size = 8 * 1024 * 1024;
}
size -= libc::sysconf(libc::_SC_PAGE_SIZE) as u64;
let bound = origin - size as usize;
return Self { origin, bound };
}
}*/
ret
}
}
#[inline(never)]
pub fn current_stack_pointer() -> Address {
#[cfg(target_arch = "x86_64")]
{
let current_sp: usize;
unsafe {
std::arch::asm!(
"mov rax, rsp",
out("rax") current_sp,
);
Address::from_usize(current_sp)
}
}
#[cfg(not(target_arch = "x86_64"))]
{
let mut current_sp = Address::ZERO;
current_sp = Address::from_ptr(&current_sp);
current_sp
}
}
#[cfg(windows)]
impl StackBounds {
fn current_thread_stack_bounds_internal() -> Self {
use std::mem::*;
use winapi::um::winnt::*;
use winapi::um::memoryapi::*;
unsafe {
let mut stack_origin: MaybeUninit<MEMORY_BASIC_INFORMATION> = MaybeUninit::uninit();
VirtualQuery(
stack_origin.as_mut_ptr(),
stack_origin.as_mut_ptr(),
size_of::<MEMORY_BASIC_INFORMATION>(),
);
let stack_origin = stack_origin.assume_init();
let origin =
Address::from_ptr(stack_origin.BaseAddress) + stack_origin.RegionSize as usize;
// The stack on Windows consists out of three parts (uncommitted memory, a guard page and present
// committed memory). The 3 regions have different BaseAddresses but all have the same AllocationBase
// since they are all from the same VirtualAlloc. The 3 regions are laid out in memory (from high to
// low) as follows:
//
// High |-------------------| -----
// | committedMemory | ^
// |-------------------| |
// | guardPage | reserved memory for the stack
// |-------------------| |
// | uncommittedMemory | v
// Low |-------------------| ----- <--- stackOrigin.AllocationBase
//
// See http://msdn.microsoft.com/en-us/library/ms686774%28VS.85%29.aspx for more information.
let mut uncommited_memory: MaybeUninit<MEMORY_BASIC_INFORMATION> = MaybeUninit::uninit();
VirtualQuery(
stack_origin.AllocationBase as _,
uncommited_memory.as_mut_ptr(),
size_of::<MEMORY_BASIC_INFORMATION>(),
);
let uncommited_memory = uncommited_memory.assume_init();
assert_eq!(uncommited_memory.State, MEM_RESERVE);
let mut guard_page: MaybeUninit<MEMORY_BASIC_INFORMATION> = MaybeUninit::uninit();
VirtualQuery(
(uncommited_memory.BaseAddress as usize + uncommited_memory.RegionSize as usize) as _,
guard_page.as_mut_ptr(),
size_of::<MEMORY_BASIC_INFORMATION>(),
);
let guard_page = guard_page.assume_init();
assert!(guard_page.Protect & PAGE_GUARD != 0);
let end_of_stack = Address::from_ptr(stack_origin.AllocationBase);
let bound = end_of_stack + guard_page.RegionSize as usize;
Self { origin, bound }
}
}
}

45
vmkit/src/mm/tlab.rs Normal file
View file

@ -0,0 +1,45 @@
use mmtk::util::{conversions::raw_align_up, Address};
/// Thread-local allocation buffer.
pub struct TLAB {
pub cursor: Address,
pub limit: Address,
}
impl TLAB {
pub fn new() -> Self {
Self {
cursor: Address::ZERO,
limit: Address::ZERO,
}
}
pub fn allocate(&mut self, size: usize, alignment: usize) -> Address {
let aligned_size = raw_align_up(size, alignment);
let result = self.cursor.align_up(alignment);
if result + aligned_size > self.limit {
return Address::ZERO;
} else {
self.cursor = result.add(aligned_size);
return result;
}
}
pub fn rebind(&mut self, cursor: Address, limit: Address) {
self.cursor = cursor;
self.limit = limit;
}
pub fn reset(&mut self) {
self.cursor = Address::ZERO;
self.limit = Address::ZERO;
}
pub fn take(&mut self) -> (Address, Address) {
let cursor = self.cursor;
let limit = self.limit;
self.reset();
(cursor, limit)
}
}

301
vmkit/src/mm/traits.rs Normal file
View file

@ -0,0 +1,301 @@
//! Collection of traits for the memory manager.
//!
//! We provide all the traits to simplify implementation of a VM. You can simply
//! implement `Trace` or `Scan` to get a trace-able object for example.
use mmtk::{
util::Address,
vm::{
slot::{SimpleSlot, Slot},
ObjectModel, ObjectTracer, SlotVisitor,
},
};
use crate::{
object_model::{object::{VMKitNarrow, VMKitObject}, VMKitObjectModel},
options::OPTIONS,
VirtualMachine,
};
use super::conservative_roots::{FatInternalPointer, InternalPointer};
pub trait ToSlot<SL: Slot> {
fn to_slot(&self) -> Option<SL>;
}
pub trait Trace {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer);
}
pub trait Scan<SL: Slot> {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>);
}
impl Trace for VMKitObject {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
if self.is_null() {
return;
}
let new_object = VMKitObject::from(tracer.trace_object((*self).try_into().unwrap()));
if new_object != *self {
*self = new_object;
}
}
}
impl<T: Trace, VM: VirtualMachine> Trace for InternalPointer<T, VM> {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
#[cfg(feature = "cooperative")]
{
assert!(
VMKitObjectModel::<VM>::NEED_VO_BITS_DURING_TRACING,
"VO-bits are not enabled during tracing, can't use internal pointers"
);
let start = mmtk::memory_manager::find_object_from_internal_pointer(
self.as_address(),
OPTIONS.interior_pointer_max_bytes,
);
if let Some(start) = start {
let offset = self.as_address() - start.to_raw_address();
let new_object = VMKitObject::from(tracer.trace_object(start));
*self = InternalPointer::new(new_object.as_address() + offset);
}
}
#[cfg(not(feature = "cooperative"))]
{
unreachable!("Internal pointers are not supported in precise mode");
}
}
}
impl<SL: Slot + SlotExtra> Scan<SL> for VMKitObject {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>) {
if let Some(slot) = self.to_slot() {
visitor.visit_slot(slot);
}
}
}
impl<SL: Slot + SlotExtra> ToSlot<SL> for VMKitObject {
fn to_slot(&self) -> Option<SL> {
Some(SL::from_vmkit_object(self))
}
}
/// Extra methods for types implementing `Slot` trait from MMTK.
pub trait SlotExtra: Slot {
/// Construct a slot from a `VMKitObject`. Must be always implemented
/// as internally we use `VMKitObject` to represent all objects.
fn from_vmkit_object(object: &VMKitObject) -> Self;
fn from_address(address: Address) -> Self;
/// Construct a slot from an `InternalPointer`. VMs are not required to implement
/// this as InternalPointer can also be traced.
fn from_internal_pointer<T, VM: VirtualMachine>(pointer: &InternalPointer<T, VM>) -> Self {
let _ = pointer;
unimplemented!()
}
/// Construct a slot from a `FatInternalPointer`. VMs are not required to implement
/// this as `FatInternalPointer` can also be traced.
fn from_fat_internal_pointer<T, VM: VirtualMachine>(
pointer: &FatInternalPointer<T, VM>,
) -> Self {
let _ = pointer;
unimplemented!()
}
fn from_narrow(narrow: &VMKitNarrow) -> Self {
let _ = narrow;
unimplemented!()
}
}
impl SlotExtra for SimpleSlot {
fn from_vmkit_object(object: &VMKitObject) -> Self {
Self::from_address(Address::from_ptr(object))
}
fn from_address(address: Address) -> Self {
SimpleSlot::from_address(address)
}
fn from_internal_pointer<T, VM: VirtualMachine>(pointer: &InternalPointer<T, VM>) -> Self {
let _ = pointer;
unimplemented!("SimpleSlot does not support internal pointers")
}
}
impl SlotExtra for Address {
fn from_vmkit_object(object: &VMKitObject) -> Self {
Address::from_ptr(object)
}
fn from_address(address: Address) -> Self {
address
}
fn from_internal_pointer<T, VM: VirtualMachine>(pointer: &InternalPointer<T, VM>) -> Self {
let _ = pointer;
unimplemented!("Address does not support internal pointers")
}
}
/// Trait to check if type can be enqueued as a slot of an object.
///
/// Slot is an address of a field of an object. When field
/// can't be enqueued, we simply trace it using `ObjectTracer`.
pub trait SupportsEnqueuing {
const VALUE: bool;
}
impl<T, VM: VirtualMachine> SupportsEnqueuing for InternalPointer<T, VM> {
const VALUE: bool = false;
}
impl SupportsEnqueuing for VMKitObject {
const VALUE: bool = true;
}
macro_rules! impl_prim {
($($t:ty)*) => {
$(
impl SupportsEnqueuing for $t {
const VALUE: bool = true;
}
impl<SL: Slot> Scan<SL> for $t {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>) {
let _ = visitor;
}
}
impl<SL: Slot> ToSlot<SL> for $t {
fn to_slot(&self) -> Option<SL> {
None
}
}
impl Trace for $t {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
let _ = tracer;
}
}
)*
};
}
impl_prim! {
u8 u16 u32 u64 u128 usize
i8 i16 i32 i64 i128 isize
f32 f64
bool char
String
std::fs::File
}
impl<T: SupportsEnqueuing> SupportsEnqueuing for Vec<T> {
const VALUE: bool = T::VALUE; // we don't enque vec itself but its elements.
}
impl<T: SupportsEnqueuing> SupportsEnqueuing for Option<T> {
const VALUE: bool = T::VALUE;
}
impl<T: SupportsEnqueuing, U: SupportsEnqueuing> SupportsEnqueuing for Result<T, U> {
const VALUE: bool = T::VALUE && U::VALUE;
}
impl<T: Trace> Trace for Option<T> {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
if let Some(value) = self {
value.trace_object(tracer);
}
}
}
impl<T: Trace> Trace for Vec<T> {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
for value in self {
value.trace_object(tracer);
}
}
}
impl<T: Trace, const N: usize> Trace for [T; N] {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
for value in self {
value.trace_object(tracer);
}
}
}
impl<T: Trace> Trace for Box<T> {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
(**self).trace_object(tracer);
}
}
impl<SL: Slot, T: Scan<SL>> Scan<SL> for Vec<T> {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>) {
for value in self {
value.scan_object(visitor);
}
}
}
impl<SL: Slot, T: Scan<SL>, const N: usize> Scan<SL> for [T; N] {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>) {
for value in self {
value.scan_object(visitor);
}
}
}
impl<T, VM: VirtualMachine> SupportsEnqueuing for FatInternalPointer<T, VM> {
const VALUE: bool = true;
}
impl<T, VM: VirtualMachine, SL: Slot + SlotExtra> Scan<SL> for FatInternalPointer<T, VM> {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>) {
visitor.visit_slot(self.object().to_slot().expect("never fails"));
}
}
impl<T, VM: VirtualMachine, SL: Slot + SlotExtra> ToSlot<SL> for FatInternalPointer<T, VM> {
fn to_slot(&self) -> Option<SL> {
Some(self.object().to_slot().expect("never fails"))
}
}
impl<T, VM: VirtualMachine> Trace for FatInternalPointer<T, VM> {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
self.object().trace_object(tracer);
}
}
impl Trace for VMKitNarrow {
fn trace_object(&mut self, tracer: &mut impl ObjectTracer) {
let mut object = self.to_object();
object.trace_object(tracer);
*self = VMKitNarrow::encode(object);
}
}
impl<SL: SlotExtra> Scan<SL> for VMKitNarrow {
fn scan_object(&self, visitor: &mut impl SlotVisitor<SL>) {
let slot = SL::from_narrow(self);
visitor.visit_slot(slot);
}
}
impl<SL: SlotExtra> ToSlot<SL> for VMKitNarrow {
fn to_slot(&self) -> Option<SL> {
Some(SL::from_narrow(self))
}
}

194
vmkit/src/object_model.rs Normal file
View file

@ -0,0 +1,194 @@
use std::marker::PhantomData;
use crate::{mm::MemoryManager, VirtualMachine};
use easy_bitfield::BitFieldTrait;
use header::{HashState, HashStateField, HASHCODE_OFFSET, OBJECT_HEADER_OFFSET, OBJECT_REF_OFFSET};
use mmtk::{
util::{alloc::fill_alignment_gap, constants::LOG_BYTES_IN_ADDRESS, ObjectReference},
vm::*,
};
use object::{MoveTarget, VMKitObject};
pub mod header;
pub mod metadata;
pub mod object;
pub mod compression;
pub struct VMKitObjectModel<VM: VirtualMachine>(PhantomData<VM>);
pub const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first();
pub const FORWARDING_POINTER_METADATA_SPEC: VMLocalForwardingPointerSpec =
VMLocalForwardingPointerSpec::in_header(0);
pub const FORWARDING_BITS_METADATA_SPEC: VMLocalForwardingBitsSpec =
VMLocalForwardingBitsSpec::in_header(HashStateField::NEXT_BIT as _);
pub const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_first();
pub const LOS_METADATA_SPEC: VMLocalLOSMarkNurserySpec =
VMLocalLOSMarkNurserySpec::in_header(HashStateField::NEXT_BIT as _);
impl<VM: VirtualMachine> ObjectModel<MemoryManager<VM>> for VMKitObjectModel<VM> {
const GLOBAL_LOG_BIT_SPEC: mmtk::vm::VMGlobalLogBitSpec = LOGGING_SIDE_METADATA_SPEC;
const LOCAL_FORWARDING_POINTER_SPEC: mmtk::vm::VMLocalForwardingPointerSpec =
FORWARDING_POINTER_METADATA_SPEC;
const LOCAL_FORWARDING_BITS_SPEC: mmtk::vm::VMLocalForwardingBitsSpec =
FORWARDING_BITS_METADATA_SPEC;
const LOCAL_MARK_BIT_SPEC: mmtk::vm::VMLocalMarkBitSpec = MARKING_METADATA_SPEC;
const LOCAL_LOS_MARK_NURSERY_SPEC: mmtk::vm::VMLocalLOSMarkNurserySpec = LOS_METADATA_SPEC;
const OBJECT_REF_OFFSET_LOWER_BOUND: isize = OBJECT_REF_OFFSET;
const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false;
const VM_WORST_CASE_COPY_EXPANSION: f64 = 1.3;
#[cfg(feature = "cooperative")]
const NEED_VO_BITS_DURING_TRACING: bool = VM::CONSERVATIVE_TRACING;
fn copy(
from: mmtk::util::ObjectReference,
semantics: mmtk::util::copy::CopySemantics,
copy_context: &mut mmtk::util::copy::GCWorkerCopyContext<MemoryManager<VM>>,
) -> mmtk::util::ObjectReference {
let vmkit_from = VMKitObject::from(from);
let bytes = vmkit_from.bytes_required_when_copied::<VM>();
let align = vmkit_from.alignment::<VM>();
let offset = vmkit_from.get_offset_for_alignment::<VM>();
let addr = copy_context.alloc_copy(from, bytes, align, offset, semantics);
let vmkit_to_obj = Self::move_object(vmkit_from, MoveTarget::ToAddress(addr), bytes);
let to_obj = ObjectReference::from_raw_address(vmkit_to_obj.as_address()).unwrap();
copy_context.post_copy(to_obj, bytes, semantics);
to_obj
}
fn copy_to(
from: mmtk::util::ObjectReference,
to: mmtk::util::ObjectReference,
region: mmtk::util::Address,
) -> mmtk::util::Address {
let vmkit_from = VMKitObject::from(from);
let copy = from != to;
let bytes = if copy {
let vmkit_to = VMKitObject::from(to);
let bytes = vmkit_to.bytes_required_when_copied::<VM>();
Self::move_object(vmkit_from, MoveTarget::ToObject(vmkit_to), bytes);
bytes
} else {
vmkit_from.bytes_used::<VM>()
};
let start = Self::ref_to_object_start(to);
fill_alignment_gap::<MemoryManager<VM>>(region, start);
start + bytes
}
fn get_reference_when_copied_to(
from: mmtk::util::ObjectReference,
to: mmtk::util::Address,
) -> mmtk::util::ObjectReference {
let vmkit_from = VMKitObject::from(from);
let res_addr = to + OBJECT_REF_OFFSET + vmkit_from.hashcode_overhead::<VM, true>();
debug_assert!(!res_addr.is_zero());
// SAFETY: we just checked that the address is not zero
unsafe {
ObjectReference::from_raw_address_unchecked(res_addr)
}
}
fn get_type_descriptor(_reference: mmtk::util::ObjectReference) -> &'static [i8] {
unreachable!()
}
fn get_align_offset_when_copied(object: mmtk::util::ObjectReference) -> usize {
VMKitObject::from_objref_nullable(Some(object)).get_offset_for_alignment::<VM>()
}
fn get_align_when_copied(object: mmtk::util::ObjectReference) -> usize {
VMKitObject::from(object).alignment::<VM>()
}
fn get_current_size(object: mmtk::util::ObjectReference) -> usize {
VMKitObject::from_objref_nullable(Some(object)).get_current_size::<VM>()
}
fn get_size_when_copied(object: mmtk::util::ObjectReference) -> usize {
VMKitObject::from_objref_nullable(Some(object)).get_size_when_copied::<VM>()
}
fn ref_to_object_start(reference: mmtk::util::ObjectReference) -> mmtk::util::Address {
VMKitObject::from_objref_nullable(Some(reference)).object_start::<VM>()
}
fn ref_to_header(reference: mmtk::util::ObjectReference) -> mmtk::util::Address {
reference.to_raw_address().offset(OBJECT_HEADER_OFFSET)
}
fn dump_object(object: mmtk::util::ObjectReference) {
let _ = object;
}
}
impl<VM: VirtualMachine> VMKitObjectModel<VM> {
fn move_object(from_obj: VMKitObject, mut to: MoveTarget, num_bytes: usize) -> VMKitObject {
let mut copy_bytes = num_bytes;
let mut obj_ref_offset = OBJECT_REF_OFFSET;
let hash_state = from_obj.header::<VM>().hash_state();
// Adjust copy bytes and object reference offset based on hash state
match hash_state {
HashState::Hashed => {
copy_bytes -= size_of::<usize>(); // Exclude hash code from copy
if let MoveTarget::ToAddress(ref mut addr) = to {
*addr += size_of::<usize>(); // Adjust address for hash code
}
}
HashState::HashedAndMoved => {
obj_ref_offset += size_of::<usize>() as isize; // Adjust for larger header
}
_ => {}
}
// Determine target address and object based on MoveTarget
let (to_address, to_obj) = match to {
MoveTarget::ToAddress(addr) => {
let obj = VMKitObject::from_address(addr + obj_ref_offset);
(addr, obj)
}
MoveTarget::ToObject(object) => {
let addr = object.as_address() + (-obj_ref_offset);
(addr, object)
}
};
let from_address = from_obj.as_address() + (-obj_ref_offset);
// Perform the memory copy
unsafe {
std::ptr::copy(
from_address.to_ptr::<u8>(),
to_address.to_mut_ptr::<u8>(),
copy_bytes,
);
}
// Update hash state if necessary
if hash_state == HashState::Hashed {
unsafe {
let hash_code = from_obj.as_address().as_usize() >> LOG_BYTES_IN_ADDRESS;
to_obj
.as_address()
.offset(HASHCODE_OFFSET)
.store(hash_code as u64);
to_obj
.header::<VM>()
.set_hash_state(HashState::HashedAndMoved);
}
} else {
to_obj.header::<VM>().set_hash_state(HashState::Hashed);
}
to_obj
}
}

View file

@ -0,0 +1,131 @@
//! Pointer compression support.
use std::cell::UnsafeCell;
use mmtk::util::Address;
use crate::{options::OPTIONS, VirtualMachine};
use super::object::{VMKitNarrow, VMKitObject};
pub const UNSCALED_OP_HEAP_MAX: u64 = u32::MAX as u64 + 1;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CompressionMode {
/// Use 32-bits oops without encoding when
/// NarrowOopHeapBaseMin + heap_size < 4Gb
UnscaledNarrowOp = 0,
/// Use zero based compressed oops with encoding when
/// NarrowOopHeapBaseMin + heap_size < 32Gb
ZeroBasedNarrowOp = 1,
/// Use compressed oops with disjoint heap base if
/// base is 32G-aligned and base > 0. This allows certain
/// optimizations in encoding/decoding.
/// Disjoint: Bits used in base are disjoint from bits used
/// for oops ==> oop = (cOop << 3) | base. One can disjoint
/// the bits of an oop into base and compressed oop.b
DisjointBaseNarrowOp = 2,
/// Use compressed oops with heap base + encoding.
HeapBasedNarrowOp = 3,
}
pub struct CompressedOps {
pub base: Address,
pub shift: u32,
pub range: (Address, Address),
}
struct CompressedOpsStorage(UnsafeCell<CompressedOps>);
unsafe impl Send for CompressedOpsStorage {}
unsafe impl Sync for CompressedOpsStorage {}
static COMPRESSED_OPS: CompressedOpsStorage =
CompressedOpsStorage(UnsafeCell::new(CompressedOps {
base: Address::ZERO,
shift: 0,
range: (Address::ZERO, Address::ZERO),
}));
impl CompressedOps {
pub(crate) fn init<VM: VirtualMachine>() {
let start = mmtk::memory_manager::starting_heap_address();
let end = mmtk::memory_manager::last_heap_address();
if OPTIONS.tag_compressed_pointers {
let shift = 2;
let base = start.sub(4096);
unsafe {
COMPRESSED_OPS.0.get().write(CompressedOps {
base,
shift,
range: (start, end),
});
}
return;
}
let shift;
// Subtract a page because something can get allocated at heap base.
// This also makes implicit null checking work, because the
// memory+1 page below heap_base needs to cause a signal.
// See needs_explicit_null_check.
// Only set the heap base for compressed oops because it indicates
// compressed oops for pstack code.
if end > unsafe { Address::from_usize(UNSCALED_OP_HEAP_MAX as usize) } {
// Didn't reserve heap below 4Gb. Must shift.
shift = VM::MIN_ALIGNMENT.trailing_zeros();
} else {
shift = 0;
}
let max = (u32::MAX as u64 + 1) << VM::MIN_ALIGNMENT.trailing_zeros();
let base = if end <= unsafe { Address::from_usize(max as usize) } {
// heap below 32Gb, can use base == 0
Address::ZERO
} else {
start.sub(4096)
};
unsafe {
COMPRESSED_OPS.0.get().write(CompressedOps {
base,
shift,
range: (start, end),
});
}
}
pub fn base() -> Address {
unsafe { (*COMPRESSED_OPS.0.get()).base }
}
pub fn shift() -> u32 {
unsafe { (*COMPRESSED_OPS.0.get()).shift }
}
pub fn decode_raw(v: VMKitNarrow) -> VMKitObject {
debug_assert!(!v.is_null());
let base = Self::base();
VMKitObject::from_address(base + ((v.raw() as usize) << Self::shift()))
}
pub fn decode(v: VMKitNarrow) -> VMKitObject {
if v.is_null() {
VMKitObject::NULL
} else {
Self::decode_raw(v)
}
}
pub fn encode(v: VMKitObject) -> VMKitNarrow {
if v.is_null() {
unsafe { VMKitNarrow::from_raw(0) }
} else {
let pd = v.as_address().as_usize() - Self::base().as_usize();
unsafe { VMKitNarrow::from_raw((pd >> Self::shift()) as u32) }
}
}
pub const fn is_null(v: VMKitNarrow) -> bool {
v.raw() == 0
}
}

View file

@ -0,0 +1,91 @@
use std::marker::PhantomData;
use crate::VirtualMachine;
use easy_bitfield::*;
/// Offset from allocation pointer to the actual object start.
pub const OBJECT_REF_OFFSET: isize = 8;
/// Object header behind object.
pub const OBJECT_HEADER_OFFSET: isize = -OBJECT_REF_OFFSET;
pub const HASHCODE_OFFSET: isize = -(OBJECT_REF_OFFSET + size_of::<usize>() as isize);
pub type MetadataField = BitField<u64, usize, 0, 58, false>;
pub type HashStateField = BitField<u64, HashState, { MetadataField::NEXT_BIT }, 2, false>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HashState {
Unhashed,
Hashed,
HashedAndMoved,
}
impl FromBitfield<u64> for HashState {
fn from_bitfield(value: u64) -> Self {
match value {
0 => Self::Unhashed,
1 => Self::Hashed,
2 => Self::HashedAndMoved,
_ => unreachable!(),
}
}
fn from_i64(value: i64) -> Self {
match value {
0 => Self::Unhashed,
1 => Self::Hashed,
2 => Self::HashedAndMoved,
_ => unreachable!(),
}
}
}
impl ToBitfield<u64> for HashState {
fn to_bitfield(self) -> u64 {
match self {
Self::Unhashed => 0,
Self::Hashed => 1,
Self::HashedAndMoved => 2,
}
}
fn one() -> Self {
Self::Hashed
}
fn zero() -> Self {
Self::Unhashed
}
}
pub struct HeapObjectHeader<VM: VirtualMachine> {
pub metadata: AtomicBitfieldContainer<u64>,
pub marker: PhantomData<VM>,
}
impl<VM: VirtualMachine> HeapObjectHeader<VM> {
pub fn new(metadata: VM::Metadata) -> Self {
Self {
metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()),
marker: PhantomData,
}
}
pub fn hash_state(&self) -> HashState {
self.metadata.read::<HashStateField>()
}
pub fn set_hash_state(&self, state: HashState) {
self.metadata.update_synchronized::<HashStateField>(state);
}
pub fn metadata(&self) -> VM::Metadata {
VM::Metadata::from_bitfield(self.metadata.read::<MetadataField>() as _)
}
pub fn set_metadata(&self, metadata: VM::Metadata) {
self.metadata.update_synchronized::<MetadataField>(metadata.to_bitfield() as _);
}
}

View file

@ -0,0 +1,149 @@
pub use easy_bitfield::{FromBitfield, ToBitfield};
use mmtk::{
util::ObjectReference,
vm::{ObjectTracer, SlotVisitor},
};
use crate::{mm::traits::ToSlot, VirtualMachine};
use super::object::VMKitObject;
#[repr(C)]
pub struct GCMetadata<VM: VirtualMachine> {
pub trace: Trace<VM>,
pub instance_size: usize,
pub compute_size: Option<fn(VMKitObject) -> usize>,
pub alignment: usize,
}
#[derive(Debug)]
pub enum Trace<VM: VirtualMachine> {
ScanSlots(fn(VMKitObject, &mut dyn SlotVisitor<VM::Slot>)),
TraceObject(fn(VMKitObject, &mut dyn ObjectTracer)),
None,
}
/// Metadata for objects in VM which uses VMKit.
///
/// Main purpose of this trait is to provide VMKit a way to get GC metadata from the object.
/// It is also used to convert object references to metadata and vice versa if you store
/// metadata as an object field.
pub trait Metadata<VM: VirtualMachine>: ToBitfield<u64> + FromBitfield<u64> + ToSlot<VM::Slot> {
/// Size of the metadata in bits. Must be `<= 62`.
const METADATA_BIT_SIZE: usize;
fn gc_metadata(&self) -> &'static GCMetadata<VM>;
fn is_object(&self) -> bool;
fn to_object_reference(&self) -> Option<ObjectReference>;
fn from_object_reference(reference: ObjectReference) -> Self;
}
/// Maximum object size that can be handled by uncooperative GC (64GiB default).
///
/// This is due to the fact that we store the object size in the metadata
/// and it needs to fit into 58 bits we give to runtime. By limiting the size
/// we get some free space for runtime to use as well.
pub const MAX_UNCOOPERATIVE_OBJECT_SIZE: usize = 1usize << 36;
#[macro_export]
macro_rules! make_uncooperative_metadata {
($max_obj_size: expr, $name: ident) => {
pub struct $name {
pub wsize: usize,
}
const _: () = {
assert!($max_obj_size <= $crate::object_model::metadata::MAX_UNCOOPERATIVE_OBJECT_SIZE);
};
impl<VM: VirtualMachine> Metadata<VM> for $name {
const METADATA_BIT_SIZE: usize = 36;
fn gc_metadata(&self) -> &'static GCMetadata<VM> {
&UNCOOPERATIVE_GC_META
}
fn is_object(&self) -> bool {
false
}
fn to_object_reference(&self) -> Option<ObjectReference> {
None
}
fn from_object_reference(_: ObjectReference) -> Self {
unreachable!()
}
}
impl<VM: $crate::VirtualMachine> $crate::object_model::metadata::ToBitfield<u64> for $name {
fn to_bitfield(self) -> u64 {
self.wsize as u64
}
}
impl<VM: $crate::VirtualMachine> $crate::object_model::metadata::FromBitfield<u64>
for $name
{
fn from_bitfield(value: u64) -> Self {
Self {
wsize: value as usize,
}
}
}
};
}
impl<VM: VirtualMachine> ToBitfield<u64> for &'static GCMetadata<VM> {
fn to_bitfield(self) -> u64 {
let res = self as *const GCMetadata<VM> as usize as u64;
res
}
fn one() -> Self {
unreachable!()
}
fn zero() -> Self {
unreachable!()
}
}
impl<VM: VirtualMachine> FromBitfield<u64> for &'static GCMetadata<VM> {
fn from_bitfield(value: u64) -> Self {
unsafe { &*(value as usize as *const GCMetadata<VM>) }
}
fn from_i64(_value: i64) -> Self {
unreachable!()
}
}
impl<VM: VirtualMachine> Metadata<VM> for &'static GCMetadata<VM> {
const METADATA_BIT_SIZE: usize = 58;
fn gc_metadata(&self) -> &'static GCMetadata<VM> {
*self
}
fn is_object(&self) -> bool {
false
}
fn to_object_reference(&self) -> Option<ObjectReference> {
None
}
fn from_object_reference(_: ObjectReference) -> Self {
unreachable!()
}
}
impl<VM: VirtualMachine> ToSlot<VM::Slot> for &'static GCMetadata<VM> {
fn to_slot(&self) -> Option<VM::Slot> {
None
}
}

View file

@ -0,0 +1,575 @@
use crate::mm::traits::SlotExtra;
use crate::threading::Thread;
use crate::{mm::MemoryManager, VirtualMachine};
use atomic::Atomic;
use mmtk::util::{
constants::LOG_BYTES_IN_ADDRESS, conversions::raw_align_up, Address, ObjectReference,
};
use super::{
compression::CompressedOps,
header::{
HashState, HeapObjectHeader, HASHCODE_OFFSET, OBJECT_HEADER_OFFSET, OBJECT_REF_OFFSET,
},
metadata::Metadata,
};
#[repr(transparent)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct VMKitObject(Address);
impl From<ObjectReference> for VMKitObject {
fn from(value: ObjectReference) -> Self {
Self::from_address(value.to_raw_address())
}
}
impl TryInto<ObjectReference> for VMKitObject {
type Error = ();
fn try_into(self) -> Result<ObjectReference, Self::Error> {
ObjectReference::from_raw_address(self.0).ok_or(())
}
}
impl Into<Option<ObjectReference>> for VMKitObject {
fn into(self) -> Option<ObjectReference> {
self.try_into().ok()
}
}
impl VMKitObject {
/// The null `VMKitObject`.
pub const NULL: Self = Self(Address::ZERO);
/// Creates a new `VMKitObject` from a given address.
///
/// # Arguments
///
/// * `address` - The address of the object.
///
/// # Returns
///
/// * `VMKitObject` - A new `VMKitObject` instance.
#[inline(always)]
pub fn from_address(address: Address) -> Self {
Self(address)
}
/// Returns the address of the `VMKitObject`.
///
/// # Returns
///
/// * `Address` - The address of the object.
#[inline(always)]
pub fn as_address(self) -> Address {
self.0
}
pub unsafe fn as_object_unchecked(self) -> ObjectReference {
unsafe { ObjectReference::from_raw_address_unchecked(self.as_address()) }
}
/// Creates a new `VMKitObject` from an optional object reference.
///
/// # Arguments
///
/// * `objref` - An optional object reference.
///
/// # Returns
///
/// * `VMKitObject` - A new `VMKitObject` instance.
#[inline(always)]
pub fn from_objref_nullable(objref: Option<ObjectReference>) -> Self {
match objref {
Some(objref) => Self::from_address(objref.to_raw_address()),
None => Self::NULL,
}
}
/// Checks if the `VMKitObject` is null.
///
/// # Returns
///
/// * `bool` - `true` if the object is null, `false` otherwise.
#[inline(always)]
pub fn is_null(self) -> bool {
self == Self::NULL
}
/// Checks if the `VMKitObject` is not null.
///
/// # Returns
///
/// * `bool` - `true` if the object is not null, `false` otherwise.
#[inline(always)]
pub fn is_not_null(self) -> bool {
self != Self::NULL
}
/// Returns a reference to the `HeapObjectHeader` of the `VMKitObject`.
///
/// # Returns
///
/// * `&HeapObjectHeader<VM>` - A reference to the header.
#[inline(always)]
pub fn header<'a, VM: VirtualMachine>(self) -> &'a HeapObjectHeader<VM> {
assert!(!self.is_null());
unsafe { self.0.offset(OBJECT_HEADER_OFFSET).as_ref() }
}
/// Returns the alignment of the `VMKitObject`.
///
/// # Returns
///
/// * `usize` - The alignment of the object.
pub fn alignment<VM: VirtualMachine>(&self) -> usize {
self.header::<VM>().metadata().gc_metadata().alignment
}
/// Returns the number of bytes used by the `VMKitObject`.
///
/// # Returns
///
/// * `usize` - The number of bytes used.
#[inline(always)]
pub fn bytes_used<VM: VirtualMachine>(self) -> usize {
let metadata = self.header::<VM>().metadata().gc_metadata();
let overhead = self.hashcode_overhead::<VM, false>();
if metadata.instance_size != 0 {
raw_align_up(metadata.instance_size + size_of::<HeapObjectHeader<VM>>(), align_of::<usize>()) + overhead
} else {
let Some(compute_size) = metadata.compute_size else {
panic!("compute_size is not set for object at {}", self.0);
};
raw_align_up(compute_size(self) + size_of::<HeapObjectHeader<VM>>(), align_of::<usize>()) + overhead
}
}
/// Returns the number of bytes required when the `VMKitObject` is copied.
///
/// # Returns
///
/// * `usize` - The number of bytes required.
#[inline(always)]
pub fn bytes_required_when_copied<VM: VirtualMachine>(&self) -> usize {
let metadata = self.header::<VM>().metadata().gc_metadata();
let overhead = self.hashcode_overhead::<VM, true>();
raw_align_up(metadata.instance_size, align_of::<usize>()) + overhead
}
/// Returns the overhead for the hashcode of the `VMKitObject`.
///
/// # Arguments
///
/// * `WHEN_COPIED` - A constant indicating whether the object is being copied.
///
/// # Returns
///
/// * `usize` - The hashcode overhead.
#[inline(always)]
pub fn hashcode_overhead<VM: VirtualMachine, const WHEN_COPIED: bool>(&self) -> usize {
let hash_state = self.header::<VM>().hash_state();
let has_hashcode = if WHEN_COPIED {
hash_state != HashState::Unhashed
} else {
hash_state == HashState::HashedAndMoved
};
if has_hashcode {
size_of::<usize>()
} else {
0
}
}
/// Returns the real starting address of the `VMKitObject`.
///
/// # Returns
///
/// * `Address` - The starting address of the object.
#[inline(always)]
pub fn object_start<VM: VirtualMachine>(&self) -> Address {
let res = self
.0
.offset(-(OBJECT_REF_OFFSET as isize + self.hashcode_overhead::<VM, false>() as isize));
res
}
/// Returns the offset for alignment of the `VMKitObject`.
///
/// # Returns
///
/// * `usize` - The offset for alignment.
#[inline(always)]
pub fn get_offset_for_alignment<VM: VirtualMachine>(&self) -> usize {
size_of::<HeapObjectHeader<VM>>() + self.hashcode_overhead::<VM, true>()
}
/// Returns the current size of the `VMKitObject`.
///
/// # Returns
///
/// * `usize` - The current size.
#[inline(always)]
pub fn get_current_size<VM: VirtualMachine>(&self) -> usize {
self.bytes_used::<VM>()
}
/// Returns the size of the `VMKitObject` when it is copied.
///
/// # Returns
///
/// * `usize` - The size when copied.
#[inline(always)]
pub fn get_size_when_copied<VM: VirtualMachine>(&self) -> usize {
self.bytes_required_when_copied::<VM>()
}
pub fn hashcode<VM: VirtualMachine>(&self) -> usize {
let header = self.header::<VM>();
match header.hash_state() {
HashState::HashedAndMoved => {
return unsafe { self.as_address().offset(HASHCODE_OFFSET).load() }
}
_ => (),
}
let hashcode = self.as_address().as_usize() >> LOG_BYTES_IN_ADDRESS;
header.set_hash_state(HashState::Hashed);
hashcode
}
pub fn get_field_primitive<T, VM: VirtualMachine, const VOLATILE: bool>(
&self,
offset: usize,
) -> T
where
T: Copy + bytemuck::NoUninit + bytemuck::Pod,
{
unsafe {
debug_assert!(
offset < self.bytes_used::<VM>(),
"attempt to access field out of bounds"
);
let ordering = if !VOLATILE {
atomic::Ordering::Relaxed
} else {
atomic::Ordering::SeqCst
};
self.as_address()
.add(offset)
.as_ref::<Atomic<T>>()
.load(ordering)
}
}
pub fn set_field_primitive<T, VM: VirtualMachine, const VOLATILE: bool>(
&self,
offset: usize,
value: T,
) where
T: Copy + bytemuck::NoUninit,
{
debug_assert!(
offset < self.bytes_used::<VM>(),
"attempt to access field out of bounds"
);
unsafe {
let ordering = if !VOLATILE {
atomic::Ordering::Relaxed
} else {
atomic::Ordering::SeqCst
};
self.as_address()
.add(offset)
.as_ref::<Atomic<T>>()
.store(value, ordering);
}
}
pub fn get_field_bool<VM: VirtualMachine>(&self, offset: usize) -> bool {
self.get_field_primitive::<u8, VM, false>(offset) != 0
}
pub fn set_field_bool<VM: VirtualMachine>(&self, offset: usize, value: bool) {
self.set_field_primitive::<u8, VM, false>(offset, if value { 1 } else { 0 });
}
pub fn get_field_u8<VM: VirtualMachine>(&self, offset: usize) -> u8 {
self.get_field_primitive::<u8, VM, false>(offset)
}
pub fn set_field_u8<VM: VirtualMachine>(&self, offset: usize, value: u8) {
self.set_field_primitive::<u8, VM, false>(offset, value);
}
pub fn get_field_u16<VM: VirtualMachine>(&self, offset: usize) -> u16 {
self.get_field_primitive::<u16, VM, false>(offset)
}
pub fn set_field_u16<VM: VirtualMachine>(&self, offset: usize, value: u16) {
self.set_field_primitive::<u16, VM, false>(offset, value);
}
pub fn get_field_u32<VM: VirtualMachine>(&self, offset: usize) -> u32 {
self.get_field_primitive::<u32, VM, false>(offset)
}
pub fn set_field_u32<VM: VirtualMachine>(&self, offset: usize, value: u32) {
self.set_field_primitive::<u32, VM, false>(offset, value);
}
pub fn get_field_u64<VM: VirtualMachine>(&self, offset: usize) -> u64 {
self.get_field_primitive::<u64, VM, false>(offset)
}
pub fn set_field_u64<VM: VirtualMachine>(&self, offset: usize, value: u64) {
self.set_field_primitive::<u64, VM, false>(offset, value);
}
pub fn get_field_i8<VM: VirtualMachine>(&self, offset: usize) -> i8 {
self.get_field_primitive::<i8, VM, false>(offset)
}
pub fn set_field_i8<VM: VirtualMachine>(&self, offset: usize, value: i8) {
self.set_field_primitive::<i8, VM, false>(offset, value);
}
pub fn get_field_i16<VM: VirtualMachine>(&self, offset: usize) -> i16 {
self.get_field_primitive::<i16, VM, false>(offset)
}
pub fn set_field_i16<VM: VirtualMachine>(&self, offset: usize, value: i16) {
self.set_field_primitive::<i16, VM, false>(offset, value);
}
pub fn get_field_i32<VM: VirtualMachine>(&self, offset: usize) -> i32 {
self.get_field_primitive::<i32, VM, false>(offset)
}
pub fn set_field_i32<VM: VirtualMachine>(&self, offset: usize, value: i32) {
self.set_field_primitive::<i32, VM, false>(offset, value);
}
pub fn get_field_i64<VM: VirtualMachine>(&self, offset: usize) -> i64 {
self.get_field_primitive::<i64, VM, false>(offset)
}
pub fn set_field_i64<VM: VirtualMachine>(&self, offset: usize, value: i64) {
self.set_field_primitive::<i64, VM, false>(offset, value);
}
pub fn get_field_f32<VM: VirtualMachine>(&self, offset: usize) -> f32 {
self.get_field_primitive::<f32, VM, false>(offset)
}
pub fn set_field_f32<VM: VirtualMachine>(&self, offset: usize, value: f32) {
self.set_field_primitive::<f32, VM, false>(offset, value);
}
pub fn get_field_f64<VM: VirtualMachine>(&self, offset: usize) -> f64 {
self.get_field_primitive::<f64, VM, false>(offset)
}
pub fn set_field_f64<VM: VirtualMachine>(&self, offset: usize, value: f64) {
self.set_field_primitive::<f64, VM, false>(offset, value);
}
pub fn get_field_isize<VM: VirtualMachine>(&self, offset: usize) -> isize {
self.get_field_primitive::<isize, VM, false>(offset)
}
pub fn set_field_isize<VM: VirtualMachine>(&self, offset: usize, value: isize) {
self.set_field_primitive::<isize, VM, false>(offset, value);
}
pub fn get_field_usize<VM: VirtualMachine>(&self, offset: usize) -> usize {
self.get_field_primitive::<usize, VM, false>(offset)
}
pub fn set_field_usize<VM: VirtualMachine>(&self, offset: usize, value: usize) {
self.set_field_primitive::<usize, VM, false>(offset, value);
}
pub unsafe fn set_field_object_no_write_barrier<VM: VirtualMachine, const VOLATILE: bool>(
&self,
offset: usize,
value: VMKitObject,
) {
self.set_field_primitive::<usize, VM, VOLATILE>(offset, value.as_address().as_usize());
}
pub fn slot_at<VM: VirtualMachine>(&self, offset: usize) -> VM::Slot {
VM::Slot::from_address(self.as_address() + offset)
}
pub fn set_field_object<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
value: VMKitObject,
) {
let tls = Thread::<VM>::current();
MemoryManager::object_reference_write_pre(tls, self, self.slot_at::<VM>(offset), value);
unsafe {
self.set_field_object_no_write_barrier::<VM, VOLATILE>(offset, value);
}
MemoryManager::object_reference_write_post(tls, self, self.slot_at::<VM>(offset), value);
}
/// Same as [`set_field_object`](Self::set_field_object) but sets
/// tagged value instead of object address. Accepts object address as a last
/// parameter to perform write barrier.
pub fn set_field_object_tagged<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
value_to_set: usize,
object: VMKitObject,
) {
let tls = Thread::<VM>::current();
MemoryManager::object_reference_write_pre(tls, self, self.slot_at::<VM>(offset), object);
self.set_field_primitive::<usize, VM, VOLATILE>(offset, value_to_set);
MemoryManager::object_reference_write_post(tls, self, self.slot_at::<VM>(offset), object);
}
pub fn get_field_object<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
) -> VMKitObject {
unsafe {
let addr = Address::from_usize(self.get_field_primitive::<usize, VM, VOLATILE>(offset));
VMKitObject::from_address(addr)
}
}
pub fn get_field_narrow<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
) -> VMKitNarrow {
unsafe { VMKitNarrow::from_raw(self.get_field_primitive::<u32, VM, VOLATILE>(offset)) }
}
pub fn set_field_narrow<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
value: VMKitNarrow,
) {
let tls = Thread::<VM>::current();
MemoryManager::object_reference_write_pre(
tls,
self,
self.slot_at::<VM>(offset),
value.to_object(),
);
self.set_field_primitive::<u32, VM, VOLATILE>(offset, value.raw());
MemoryManager::object_reference_write_post(
tls,
self,
self.slot_at::<VM>(offset),
value.to_object(),
);
}
pub unsafe fn set_field_narrow_no_write_barrier<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
value: VMKitNarrow,
) {
self.set_field_primitive::<u32, VM, VOLATILE>(offset, value.raw());
}
pub fn set_field_narrow_tagged<VM: VirtualMachine, const VOLATILE: bool>(
self,
offset: usize,
value_to_set: u32,
object: VMKitNarrow,
) {
let tls = Thread::<VM>::current();
MemoryManager::object_reference_write_pre(
tls,
self,
self.slot_at::<VM>(offset),
object.to_object(),
);
self.set_field_primitive::<u32, VM, VOLATILE>(offset, value_to_set);
MemoryManager::object_reference_write_post(
tls,
self,
self.slot_at::<VM>(offset),
object.to_object(),
);
}
}
/// Used as a parameter of `move_object` to specify where to move an object to.
pub enum MoveTarget {
/// Move an object to the address returned from `alloc_copy`.
ToAddress(Address),
/// Move an object to an `VMKitObject` pointing to an object previously computed from
/// `get_reference_when_copied_to`.
ToObject(VMKitObject),
}
/// Narrow pointer to an object. This is used when pointer compression
/// is enabled.
#[repr(transparent)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct VMKitNarrow(u32);
impl VMKitNarrow {
pub const NULL: Self = Self(0);
/// Return the raw value of the narrow pointer.
pub const fn raw(self) -> u32 {
self.0
}
pub fn from_object(object: VMKitObject) -> Self {
Self::encode(object)
}
pub fn encode(object: VMKitObject) -> Self {
CompressedOps::encode(object)
}
pub fn decode(self) -> VMKitObject {
CompressedOps::decode(self)
}
/// Create a new `VMKitNarrow` from a raw value.
///
/// # Safety
///
/// This function is unsafe because it assumes that the raw value is a valid narrow pointer.
pub unsafe fn from_raw(raw: u32) -> Self {
Self(raw)
}
pub const fn is_null(self) -> bool {
self.raw() == 0
}
pub fn to_address(self) -> Address {
CompressedOps::decode(self).as_address()
}
pub fn to_object(self) -> VMKitObject {
CompressedOps::decode(self)
}
pub fn header<'a, VM: VirtualMachine>(&'a self) -> &'a HeapObjectHeader<VM> {
self.to_object().header::<VM>()
}
pub fn hashcode<VM: VirtualMachine>(&self) -> usize {
self.to_object().hashcode::<VM>()
}
pub fn object_start<VM: VirtualMachine>(&self) -> Address {
self.to_object().object_start::<VM>()
}
}

33
vmkit/src/options.rs Normal file
View file

@ -0,0 +1,33 @@
use std::sync::LazyLock;
use clap::Parser;
#[derive(Parser)]
pub struct Options {
#[cfg(feature = "cooperative")]
#[clap(long, default_value_t = true)]
pub conservative_stacks: bool,
/// Maximum amount of bytes to scan for interior pointers.
#[clap(long, default_value_t = 1024)]
pub interior_pointer_max_bytes: usize,
#[clap(long, default_value_t = true)]
pub aslr: bool,
#[clap(long, default_value_t = true)]
pub compressed_pointers: bool,
/// If true, the VM will guarantee two bits for the tag of compressed pointers.
///
/// Requires MIN_ALIGNMENT to be 16.
#[clap(long, default_value_t = true)]
pub tag_compressed_pointers: bool,
#[clap(long, default_value_t = 256 * 1024 * 1024)]
pub max_heap_size: usize,
#[clap(long, default_value_t = 64 * 1024 * 1024)]
pub min_heap_size: usize,
}
pub static OPTIONS: LazyLock<Options> = LazyLock::new(Options::parse);

195
vmkit/src/sync.rs Normal file
View file

@ -0,0 +1,195 @@
use std::{
mem::ManuallyDrop,
num::NonZeroU64,
ops::Deref,
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
time::Duration,
};
use parking_lot::{Condvar, Mutex, MutexGuard, WaitTimeoutResult};
use crate::{
threading::{parked_scope, Thread, ThreadContext},
VirtualMachine,
};
fn get_thread_id() -> NonZeroU64 {
thread_local! {
static KEY: u64 = 0;
}
KEY.with(|x| {
NonZeroU64::new(x as *const _ as u64).expect("thread-local variable address is null")
})
}
/// Implementation of a heavy lock and condition variable implemented using
/// the primitives available from the `parking_lot`. Currently we use
/// a `Mutex` and `Condvar`.
/// <p>
/// It is perfectly safe to use this throughout the VM for locking. It is
/// meant to provide roughly the same functionality as ReentrantMutex combined with Condvar,
/// except:
/// <ul>
/// <li>This struct provides a faster slow path than ReentrantMutex.</li>
/// <li>This struct provides a slower fast path than ReentrantMutex.</li>
/// <li>This struct will work in the inner guts of the VM runtime because
/// it gives you the ability to lock and unlock, as well as wait and
/// notify, without using any other VM runtime functionality.</li>
/// <li>This struct allows you to optionally block without letting the thread
/// system know that you are blocked. The benefit is that you can
/// perform synchronization without depending on VM thread subsystem functionality.
/// However, most of the time, you should use the methods that inform
/// the thread system that you are blocking. Methods that have the
/// `with_handshake` suffix will inform the thread system if you are blocked,
/// while methods that do not have the suffix will either not block
/// (as is the case with `unlock()` and `broadcast()`)
/// or will block without letting anyone know (like `lock_no_handshake()`
/// and `wait_no_handshake()`). Not letting the threading
/// system know that you are blocked may cause things like GC to stall
/// until you unblock.</li>
/// <li>This struct does not provide mutable access to the protected data as it is unsound,
/// instead use `RefCell` to mutate the protected data.</li>
/// </ul>
pub struct Monitor<T> {
mutex: Mutex<T>,
cvar: Condvar,
rec_count: AtomicUsize,
holder: AtomicU64,
}
impl<T> Monitor<T> {
pub fn new(value: T) -> Self {
Self {
mutex: Mutex::new(value),
cvar: Condvar::new(),
rec_count: AtomicUsize::new(0),
holder: AtomicU64::new(0),
}
}
pub fn lock_no_handshake(&self) -> MonitorGuard<T> {
let my_slot = get_thread_id().get();
let guard = if self.holder.load(Ordering::Relaxed) != my_slot {
let guard = self.mutex.lock();
self.holder.store(my_slot, Ordering::Release);
MonitorGuard {
monitor: self,
guard: ManuallyDrop::new(guard),
}
} else {
MonitorGuard {
monitor: self,
guard: unsafe { ManuallyDrop::new(self.mutex.make_guard_unchecked()) },
}
};
self.rec_count.fetch_add(1, Ordering::Relaxed);
guard
}
pub fn notify(&self) {
self.cvar.notify_one();
}
pub fn notify_all(&self) {
self.cvar.notify_all();
}
pub unsafe fn relock_with_handshake<VM: VirtualMachine>(
&self,
rec_count: usize,
) -> MonitorGuard<'_, T> {
let thread = Thread::<VM>::current();
thread.context.save_thread_state();
let guard = loop {
Thread::<VM>::enter_native();
let lock = self.mutex.lock();
if Thread::<VM>::attempt_leave_native_no_block() {
break lock;
} else {
drop(lock);
Thread::<VM>::leave_native();
}
};
self.holder.store(get_thread_id().get(), Ordering::Relaxed);
self.rec_count.store(rec_count, Ordering::Relaxed);
MonitorGuard {
monitor: self,
guard: ManuallyDrop::new(guard),
}
}
}
pub struct MonitorGuard<'a, T> {
monitor: &'a Monitor<T>,
guard: ManuallyDrop<MutexGuard<'a, T>>,
}
impl<T> Deref for MonitorGuard<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.guard
}
}
impl<'a, T> MonitorGuard<'a, T> {
pub fn wait_no_handshake(&mut self) {
let rec_count = self.monitor.rec_count.swap(0, Ordering::Relaxed);
let holder = self.monitor.holder.swap(0, Ordering::Relaxed);
self.monitor.cvar.wait(&mut self.guard);
self.monitor.rec_count.store(rec_count, Ordering::Relaxed);
self.monitor.holder.store(holder, Ordering::Relaxed);
}
pub fn wait_for_no_handshake(&mut self, timeout: Duration) -> WaitTimeoutResult {
let rec_count = self.monitor.rec_count.swap(0, Ordering::Relaxed);
let holder = self.monitor.holder.swap(0, Ordering::Relaxed);
let result = self.monitor.cvar.wait_for(&mut self.guard, timeout);
self.monitor.rec_count.store(rec_count, Ordering::Relaxed);
self.monitor.holder.store(holder, Ordering::Relaxed);
result
}
pub fn notify(&self) {
self.monitor.cvar.notify_one();
}
pub fn notify_all(&self) {
self.monitor.cvar.notify_all();
}
pub fn monitor(&self) -> &Monitor<T> {
self.monitor
}
pub unsafe fn unlock_completely(&mut self) -> usize {
let result = self.monitor.rec_count.load(Ordering::Relaxed);
self.monitor.rec_count.store(0, Ordering::Relaxed);
self.monitor.holder.store(0, Ordering::Relaxed);
unsafe {
ManuallyDrop::drop(&mut self.guard);
}
result
}
pub fn wait_with_handshake<VM: VirtualMachine>(mut self) -> Self {
let t = Thread::<VM>::current();
t.context.save_thread_state();
let rec_count = parked_scope::<usize, VM>(|| {
self.wait_no_handshake();
let rec_count = unsafe { self.unlock_completely() };
rec_count
});
unsafe { self.monitor.relock_with_handshake::<VM>(rec_count) }
}
}
impl<'a, T> Drop for MonitorGuard<'a, T> {
fn drop(&mut self) {
if self.monitor.rec_count.fetch_sub(1, Ordering::Relaxed) == 1 {
self.monitor.holder.store(0, Ordering::Relaxed);
unsafe { ManuallyDrop::drop(&mut self.guard) };
}
}
}

1547
vmkit/src/threading.rs Normal file

File diff suppressed because it is too large Load diff