From e20870c604826f1625e83d7ff76e3befbbb1885b Mon Sep 17 00:00:00 2001 From: playX18 Date: Thu, 13 Feb 2025 17:24:08 +0700 Subject: [PATCH] support aligning allocations --- vmkit/examples/binarytrees.rs | 2 +- vmkit/src/lib.rs | 12 ++++-- vmkit/src/mm.rs | 51 +++++++++++++++------- vmkit/src/mm/align.rs | 79 +++++++++++++++++++++++++++++++++++ vmkit/src/mm/tlab.rs | 19 ++++++--- vmkit/src/threading.rs | 1 - 6 files changed, 139 insertions(+), 25 deletions(-) create mode 100644 vmkit/src/mm/align.rs diff --git a/vmkit/examples/binarytrees.rs b/vmkit/examples/binarytrees.rs index 62dc149..8045384 100644 --- a/vmkit/examples/binarytrees.rs +++ b/vmkit/examples/binarytrees.rs @@ -113,7 +113,7 @@ impl NodeRef { let node = MemoryManager::::allocate( thread, size_of::(), - 16, + 32, &METADATA, AllocationSemantics::Default, ); diff --git a/vmkit/src/lib.rs b/vmkit/src/lib.rs index 579840e..f98d5c7 100644 --- a/vmkit/src/lib.rs +++ b/vmkit/src/lib.rs @@ -1,4 +1,7 @@ -use std::{marker::PhantomData, sync::atomic::{AtomicBool, AtomicUsize}}; +use std::{ + marker::PhantomData, + sync::atomic::{AtomicBool, AtomicUsize}, +}; use mm::{aslr::aslr_vm_layout, traits::SlotExtra, MemoryManager}; use mmtk::{MMTKBuilder, MMTK}; @@ -8,12 +11,12 @@ pub mod machine_context; pub mod mm; pub mod object_model; pub mod options; +pub mod platform; pub mod semaphore; pub mod sync; pub mod threading; -pub mod platform; -#[cfg(feature="uncooperative")] +#[cfg(feature = "uncooperative")] pub mod bdwgc_shim; pub trait VirtualMachine: Sized + 'static + Send + Sync { @@ -22,7 +25,8 @@ pub trait VirtualMachine: Sized + 'static + Send + Sync { type Metadata: object_model::metadata::Metadata; type Slot: SlotExtra; - const MAX_ALIGNMENT: usize = 16; + const ALIGNMENT_VALUE: u32 = 0xdead_beef; + const MAX_ALIGNMENT: usize = 32; const MIN_ALIGNMENT: usize = 8; /// Does this VM use conservative tracing? If `true` then VM can diff --git a/vmkit/src/mm.rs b/vmkit/src/mm.rs index d3e3668..c5887b6 100644 --- a/vmkit/src/mm.rs +++ b/vmkit/src/mm.rs @@ -10,7 +10,10 @@ use crate::{ use easy_bitfield::{AtomicBitfieldContainer, ToBitfield}; use mmtk::{ util::{ - alloc::{AllocatorSelector, BumpAllocator, ImmixAllocator}, conversions::raw_align_up, metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, VMMutatorThread + alloc::{AllocatorSelector, BumpAllocator, ImmixAllocator}, + conversions::raw_align_up, + metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, + VMMutatorThread, }, vm::{ slot::{Slot, UnimplementedMemorySlice}, @@ -44,6 +47,7 @@ impl VMBinding for MemoryManager { } pub mod active_plan; +pub mod align; pub mod aslr; pub mod collection; pub mod conservative_roots; @@ -137,7 +141,9 @@ impl MemoryManager { AllocationSemantics::Default => match thread.alloc_fastpath() { AllocFastPath::TLAB => unsafe { let tlab = thread.tlab.get().as_mut().unwrap(); - let object_start = tlab.allocate(size, alignment); + let object_start = + tlab.allocate::(size, alignment, OBJECT_REF_OFFSET as usize); + if !object_start.is_zero() { object_start.store(HeapObjectHeader:: { metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()), @@ -167,14 +173,13 @@ impl MemoryManager { alignment: usize, metadata: VM::Metadata, ) -> VMKitObject { - unsafe { Self::flush_tlab(thread); let object_start = mmtk::memory_manager::alloc( thread.mutator(), size, alignment, - 0, + OBJECT_REF_OFFSET as usize, AllocationSemantics::Los, ); @@ -186,7 +191,12 @@ impl MemoryManager { //Self::set_vo_bit(object); Self::refill_tlab(thread); - mmtk::memory_manager::post_alloc(thread.mutator(), object.as_object_unchecked(), size, AllocationSemantics::Los); + mmtk::memory_manager::post_alloc( + thread.mutator(), + object.as_object_unchecked(), + size, + AllocationSemantics::Los, + ); object } } @@ -206,7 +216,7 @@ impl MemoryManager { thread.mutator(), size, alignment, - 0, + OBJECT_REF_OFFSET as usize, AllocationSemantics::NonMoving, ); @@ -236,7 +246,7 @@ impl MemoryManager { thread.mutator(), size, alignment, - 0, + OBJECT_REF_OFFSET as usize, AllocationSemantics::Immortal, ); @@ -264,8 +274,13 @@ impl MemoryManager { ) -> VMKitObject { unsafe { Self::flush_tlab(thread); - let object_start = - mmtk::memory_manager::alloc_slow(thread.mutator(), size, alignment, 0, semantics); + let object_start = mmtk::memory_manager::alloc_slow( + thread.mutator(), + size, + alignment, + OBJECT_REF_OFFSET as usize, + semantics, + ); object_start.store(HeapObjectHeader:: { metadata: AtomicBitfieldContainer::new(metadata.to_bitfield()), @@ -393,7 +408,6 @@ impl MemoryManager { let shift = (addr >> 3) & 0b111; let byte_val = meta_addr.load::(); if (byte_val >> shift) & 1 == 1 { - thread.mutator().barrier().object_reference_write_slow( src.as_object_unchecked(), slot, @@ -452,17 +466,26 @@ impl MemoryManager { Self::object_reference_write_post(thread, src, slot, target); } - pub fn disable_gc() { - VM::get().vmkit().gc_disabled_depth.fetch_add(1, atomic::Ordering::SeqCst); + VM::get() + .vmkit() + .gc_disabled_depth + .fetch_add(1, atomic::Ordering::SeqCst); } pub fn enable_gc() { - VM::get().vmkit().gc_disabled_depth.fetch_sub(1, atomic::Ordering::SeqCst); + VM::get() + .vmkit() + .gc_disabled_depth + .fetch_sub(1, atomic::Ordering::SeqCst); } pub fn is_gc_enabled() -> bool { - VM::get().vmkit().gc_disabled_depth.load(atomic::Ordering::SeqCst) == 0 + VM::get() + .vmkit() + .gc_disabled_depth + .load(atomic::Ordering::SeqCst) + == 0 } } diff --git a/vmkit/src/mm/align.rs b/vmkit/src/mm/align.rs new file mode 100644 index 0000000..af3063a --- /dev/null +++ b/vmkit/src/mm/align.rs @@ -0,0 +1,79 @@ +use mmtk::util::Address; + +use crate::VirtualMachine; + +pub const fn round_down(x: isize, align: isize) -> isize { + x & -align +} + +pub const fn round_up(x: isize, align: isize, offset: isize) -> isize { + round_down(x + align - 1 + offset, align) - offset +} + +pub const fn is_aligned(x: isize, alignment: isize, offset: isize) -> bool { + (x & (alignment - 1)) == offset +} + +#[inline(always)] +pub fn align_allocation_inner( + region: Address, + alignment: usize, + offset: usize, + _known_alignment: usize, + fill: bool, +) -> Address { + // No alignment ever required. + /*if alignment <= known_alignment || VM::MAX_ALIGNMENT <= VM::MIN_ALIGNMENT { + return region; + }*/ + + // May require alignment + let mask = (alignment - 1) as isize; + let neg_off: isize = -(offset as isize); + let delta = neg_off.wrapping_sub(region.as_usize() as isize) & mask; + + if fill && VM::ALIGNMENT_VALUE != 0 { + fill_alignment_gap::(region, region + delta as usize); + } + + let x = region + delta; + + x +} + +/// Fill the specified region with the alignment value. +pub fn fill_alignment_gap(immut_start: Address, end: Address) { + let mut start = immut_start; + + if VM::MAX_ALIGNMENT - VM::MIN_ALIGNMENT == size_of::() { + // At most a single hole + if end - start != 0 { + unsafe { + start.store(VM::ALIGNMENT_VALUE); + } + } + } else { + while start < end { + unsafe { + start.store(VM::ALIGNMENT_VALUE); + } + start += size_of::(); + } + } +} + +pub fn align_allocation_no_fill( + region: Address, + alignment: usize, + offset: usize, +) -> Address { + align_allocation_inner::(region, alignment, offset, VM::MIN_ALIGNMENT, false) +} + +pub fn align_allocation( + region: Address, + alignment: usize, + offset: usize, +) -> Address { + align_allocation_inner::(region, alignment, offset, VM::MIN_ALIGNMENT, true) +} diff --git a/vmkit/src/mm/tlab.rs b/vmkit/src/mm/tlab.rs index 33885d8..32e2536 100644 --- a/vmkit/src/mm/tlab.rs +++ b/vmkit/src/mm/tlab.rs @@ -1,5 +1,9 @@ use mmtk::util::{conversions::raw_align_up, Address}; +use crate::VirtualMachine; + +use super::align::align_allocation_no_fill; + /// Thread-local allocation buffer. pub struct TLAB { pub cursor: Address, @@ -14,13 +18,18 @@ impl TLAB { } } - pub fn allocate(&mut self, size: usize, alignment: usize) -> Address { - let aligned_size = raw_align_up(size, alignment); - let result = self.cursor.align_up(alignment); - if result + aligned_size > self.limit { + pub fn allocate( + &mut self, + size: usize, + alignment: usize, + offset: usize, + ) -> Address { + let size = raw_align_up(size, alignment); + let result = align_allocation_no_fill::(self.cursor, alignment, offset); + if result + size > self.limit { return Address::ZERO; } else { - self.cursor = result.add(aligned_size); + self.cursor = result.add(size); return result; } } diff --git a/vmkit/src/threading.rs b/vmkit/src/threading.rs index 3ba717b..7ed82bb 100644 --- a/vmkit/src/threading.rs +++ b/vmkit/src/threading.rs @@ -24,7 +24,6 @@ use crate::{ tlab::TLAB, AllocFastPath, MemoryManager, }, - object_model::compression::CompressedOps, semaphore::Semaphore, sync::{Monitor, MonitorGuard}, VirtualMachine,