Skip to content

Commit 833a3fc

Browse files
committed
feat(virtq): replace host-to-guest calls with virtq
Signed-off-by: Tomasz Andrzejak <andreiltd@gmail.com>
1 parent 95cfb6c commit 833a3fc

16 files changed

Lines changed: 706 additions & 179 deletions

File tree

src/hyperlight_common/src/layout.rs

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,24 +36,28 @@ pub use arch::{SNAPSHOT_PT_GVA_MAX, SNAPSHOT_PT_GVA_MIN};
3636
pub const SCRATCH_TOP_SIZE_OFFSET: u64 = 0x08;
3737
pub const SCRATCH_TOP_ALLOCATOR_OFFSET: u64 = 0x10;
3838
pub const SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET: u64 = 0x18;
39-
pub const SCRATCH_TOP_G2H_RING_GVA_OFFSET: u64 = 0x20;
40-
pub const SCRATCH_TOP_H2G_RING_GVA_OFFSET: u64 = 0x28;
41-
pub const SCRATCH_TOP_G2H_QUEUE_DEPTH_OFFSET: u64 = 0x30;
42-
pub const SCRATCH_TOP_H2G_QUEUE_DEPTH_OFFSET: u64 = 0x32;
43-
pub const SCRATCH_TOP_VIRTQ_POOL_PAGES_OFFSET: u64 = 0x34;
44-
pub const SCRATCH_TOP_VIRTQ_GENERATION_OFFSET: u64 = 0x36;
45-
pub const SCRATCH_TOP_EXN_STACK_OFFSET: u64 = 0x40;
39+
pub const SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET: u64 = 0x20;
40+
pub const SCRATCH_TOP_G2H_RING_GVA_OFFSET: u64 = 0x28;
41+
pub const SCRATCH_TOP_H2G_RING_GVA_OFFSET: u64 = 0x30;
42+
pub const SCRATCH_TOP_G2H_QUEUE_DEPTH_OFFSET: u64 = 0x38;
43+
pub const SCRATCH_TOP_H2G_QUEUE_DEPTH_OFFSET: u64 = 0x3A;
44+
pub const SCRATCH_TOP_G2H_POOL_PAGES_OFFSET: u64 = 0x3C;
45+
pub const SCRATCH_TOP_H2G_POOL_PAGES_OFFSET: u64 = 0x3E;
46+
pub const SCRATCH_TOP_H2G_POOL_GVA_OFFSET: u64 = 0x48;
47+
pub const SCRATCH_TOP_EXN_STACK_OFFSET: u64 = 0x50;
4648

4749
const _: () = {
48-
assert!(SCRATCH_TOP_SIZE_OFFSET + 8 <= SCRATCH_TOP_ALLOCATOR_OFFSET);
49-
assert!(SCRATCH_TOP_ALLOCATOR_OFFSET + 8 <= SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET);
50-
assert!(SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET + 8 <= SCRATCH_TOP_G2H_RING_GVA_OFFSET);
51-
assert!(SCRATCH_TOP_G2H_RING_GVA_OFFSET + 8 <= SCRATCH_TOP_H2G_RING_GVA_OFFSET);
52-
assert!(SCRATCH_TOP_H2G_RING_GVA_OFFSET + 8 <= SCRATCH_TOP_G2H_QUEUE_DEPTH_OFFSET);
53-
assert!(SCRATCH_TOP_G2H_QUEUE_DEPTH_OFFSET + 2 <= SCRATCH_TOP_H2G_QUEUE_DEPTH_OFFSET);
54-
assert!(SCRATCH_TOP_H2G_QUEUE_DEPTH_OFFSET + 2 <= SCRATCH_TOP_VIRTQ_POOL_PAGES_OFFSET);
55-
assert!(SCRATCH_TOP_VIRTQ_POOL_PAGES_OFFSET + 2 <= SCRATCH_TOP_VIRTQ_GENERATION_OFFSET);
56-
assert!(SCRATCH_TOP_VIRTQ_GENERATION_OFFSET + 2 <= SCRATCH_TOP_EXN_STACK_OFFSET);
50+
assert!(SCRATCH_TOP_ALLOCATOR_OFFSET >= SCRATCH_TOP_SIZE_OFFSET + 8);
51+
assert!(SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET >= SCRATCH_TOP_ALLOCATOR_OFFSET + 8);
52+
assert!(SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET >= SCRATCH_TOP_SNAPSHOT_PT_GPA_BASE_OFFSET + 8);
53+
assert!(SCRATCH_TOP_G2H_RING_GVA_OFFSET >= SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET + 8);
54+
assert!(SCRATCH_TOP_H2G_RING_GVA_OFFSET >= SCRATCH_TOP_G2H_RING_GVA_OFFSET + 8);
55+
assert!(SCRATCH_TOP_G2H_QUEUE_DEPTH_OFFSET >= SCRATCH_TOP_H2G_RING_GVA_OFFSET + 8);
56+
assert!(SCRATCH_TOP_H2G_QUEUE_DEPTH_OFFSET >= SCRATCH_TOP_G2H_QUEUE_DEPTH_OFFSET + 2);
57+
assert!(SCRATCH_TOP_G2H_POOL_PAGES_OFFSET >= SCRATCH_TOP_H2G_QUEUE_DEPTH_OFFSET + 2);
58+
assert!(SCRATCH_TOP_H2G_POOL_PAGES_OFFSET >= SCRATCH_TOP_G2H_POOL_PAGES_OFFSET + 2);
59+
assert!(SCRATCH_TOP_H2G_POOL_GVA_OFFSET >= SCRATCH_TOP_H2G_POOL_PAGES_OFFSET + 8);
60+
assert!(SCRATCH_TOP_EXN_STACK_OFFSET >= SCRATCH_TOP_H2G_POOL_GVA_OFFSET + 8);
5761
assert!(SCRATCH_TOP_EXN_STACK_OFFSET % 0x10 == 0);
5862
};
5963

src/hyperlight_common/src/virtq/mod.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ mod event;
157157
pub mod msg;
158158
mod pool;
159159
mod producer;
160+
pub mod recycle_pool;
160161
mod ring;
161162

162163
use core::num::NonZeroU16;
@@ -170,7 +171,7 @@ pub use producer::*;
170171
pub use ring::*;
171172
use thiserror::Error;
172173

173-
/// A trait for notifying about new requests in the virtqueue.
174+
/// A trait for notifying the consumer about virtqueue events.
174175
pub trait Notifier {
175176
fn notify(&self, stats: QueueStats);
176177
}
@@ -439,15 +440,14 @@ pub(crate) mod test_utils {
439440
}
440441
}
441442

443+
type TestProducer = VirtqProducer<Arc<TestMem>, TestNotifier, TestPool>;
444+
type TestConsumer = VirtqConsumer<Arc<TestMem>, TestNotifier>;
445+
442446
/// Create test infrastructure: a producer, consumer, and notifier backed
443447
/// by the supplied [`OwnedRing`].
444448
pub(crate) fn make_test_producer(
445449
ring: &OwnedRing,
446-
) -> (
447-
VirtqProducer<Arc<TestMem>, TestNotifier, TestPool>,
448-
VirtqConsumer<Arc<TestMem>, TestNotifier>,
449-
TestNotifier,
450-
) {
450+
) -> (TestProducer, TestConsumer, TestNotifier) {
451451
let layout = ring.layout();
452452
let mem = ring.mem();
453453

src/hyperlight_common/src/virtq/pool.rs

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ limitations under the License.
7979
//! owning slab (`Slab::resize`) but will never move allocations between
8080
//! slabs.
8181
82-
#[cfg(all(test, loom))]
8382
use alloc::sync::Arc;
8483
use core::cmp::Ordering;
8584

@@ -124,6 +123,9 @@ pub trait BufferProvider {
124123

125124
/// Resize by trying in-place grow; otherwise reserve a new block and free old.
126125
fn resize(&self, old_alloc: Allocation, new_len: usize) -> Result<Allocation, AllocError>;
126+
127+
/// Reset the pool to initial state.
128+
fn reset(&self) {}
127129
}
128130

129131
impl<T: BufferProvider> BufferProvider for alloc::rc::Rc<T> {
@@ -136,9 +138,12 @@ impl<T: BufferProvider> BufferProvider for alloc::rc::Rc<T> {
136138
fn resize(&self, old_alloc: Allocation, new_len: usize) -> Result<Allocation, AllocError> {
137139
(**self).resize(old_alloc, new_len)
138140
}
141+
fn reset(&self) {
142+
(**self).reset()
143+
}
139144
}
140145

141-
impl<T: BufferProvider> BufferProvider for alloc::sync::Arc<T> {
146+
impl<T: BufferProvider> BufferProvider for Arc<T> {
142147
fn alloc(&self, len: usize) -> Result<Allocation, AllocError> {
143148
(**self).alloc(len)
144149
}
@@ -148,6 +153,9 @@ impl<T: BufferProvider> BufferProvider for alloc::sync::Arc<T> {
148153
fn resize(&self, old_alloc: Allocation, new_len: usize) -> Result<Allocation, AllocError> {
149154
(**self).resize(old_alloc, new_len)
150155
}
156+
fn reset(&self) {
157+
(**self).reset()
158+
}
151159
}
152160

153161
/// The owner of a mapped buffer, ensuring its lifetime.
@@ -540,26 +548,20 @@ struct Inner<const L: usize, const U: usize> {
540548
}
541549

542550
/// Two tier buffer pool with small and large slabs.
543-
#[derive(Debug)]
551+
#[derive(Debug, Clone)]
544552
pub struct BufferPool<const L: usize = 256, const U: usize = 4096> {
545-
inner: AtomicRefCell<Inner<L, U>>,
553+
// TODO: Use Rc instead, relax Sync + Send bounds
554+
inner: Arc<AtomicRefCell<Inner<L, U>>>,
546555
}
547556

548557
impl<const L: usize, const U: usize> BufferPool<L, U> {
549558
/// Create a new buffer pool over a fixed region.
550559
pub fn new(base_addr: u64, region_len: usize) -> Result<Self, AllocError> {
551560
let inner = Inner::<L, U>::new(base_addr, region_len)?;
552561
Ok(Self {
553-
inner: inner.into(),
562+
inner: Arc::new(inner.into()),
554563
})
555564
}
556-
557-
/// Reset the pool to initial state
558-
pub fn reset(&self) {
559-
let mut inner = self.inner.borrow_mut();
560-
inner.lower.reset();
561-
inner.upper.reset();
562-
}
563565
}
564566

565567
#[cfg(all(test, loom))]
@@ -672,6 +674,12 @@ impl<const L: usize, const U: usize> BufferProvider for BufferPool<L, U> {
672674
fn resize(&self, old_alloc: Allocation, new_len: usize) -> Result<Allocation, AllocError> {
673675
self.inner.borrow_mut().resize(old_alloc, new_len)
674676
}
677+
678+
fn reset(&self) {
679+
let mut inner = self.inner.borrow_mut();
680+
inner.lower.reset();
681+
inner.upper.reset();
682+
}
675683
}
676684

677685
#[cfg(all(test, loom))]

src/hyperlight_common/src/virtq/producer.rs

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,13 @@ where
282282
*slot = Some(inflight);
283283

284284
let should_notify = self.inner.should_notify_since(cursor_before)?;
285+
286+
// TODO(virtq): for now simulate current outb behavior of only
287+
// notifying on bidirectional (request/response) entries.
288+
// Eventually this should be decoupled from the buffer layout
289+
// and driven entirely by event suppression rules.
290+
let should_notify = should_notify && matches!(inflight, Inflight::ReadWrite { .. });
291+
285292
if should_notify {
286293
self.notifier.notify(QueueStats {
287294
num_free: self.inner.num_free(),
@@ -292,6 +299,17 @@ where
292299
Ok(Token(id))
293300
}
294301

302+
/// Signal backpressure to the consumer.
303+
///
304+
/// Bypasses event suppression. Call this when submit fails with a backpressure error and the consumer needs to drain.
305+
#[inline]
306+
pub fn notify_backpressure(&self) {
307+
self.notifier.notify(QueueStats {
308+
num_free: self.inner.num_free(),
309+
num_inflight: self.inner.num_inflight(),
310+
});
311+
}
312+
295313
/// Get the current used cursor position.
296314
///
297315
/// Useful for setting up descriptor-based event suppression.
@@ -330,10 +348,20 @@ where
330348
Ok(())
331349
}
332350

333-
/// Reset ring and inflight state to initial values.
334-
/// Does not reset the buffer pool; call pool.reset() separately if needed.
351+
/// Reset ring, inflight, and pool state to initial values.
352+
///
353+
/// # Safety
354+
///
355+
/// All [`RecvCompletion`]s (and their backing [`Bytes`]) from
356+
/// previous `poll()` calls must have been dropped before calling
357+
/// this. Outstanding completions hold pool allocations via
358+
/// `BufferOwner`; resetting the pool while they exist would cause
359+
/// double-free on drop.
360+
///
361+
/// TODO(virtq): properly restore state after snapshot instead of just resetting everything
335362
pub fn reset(&mut self) {
336363
self.inner.reset();
364+
self.pool.reset();
337365
self.inflight.fill(None);
338366
}
339367
}
@@ -343,14 +371,14 @@ where
343371
/// If dropped without building, no resources are leaked (allocations are
344372
/// deferred to [`build`](Self::build)).
345373
#[must_use = "call .build() to create a SendEntry"]
346-
pub struct ChainBuilder<M: MemOps + Clone, P: BufferProvider + Clone> {
374+
pub struct ChainBuilder<M: MemOps, P: BufferProvider + Clone> {
347375
mem: M,
348376
pool: P,
349377
entry_cap: Option<usize>,
350378
cqe_cap: Option<usize>,
351379
}
352380

353-
impl<M: MemOps + Clone, P: BufferProvider + Clone> ChainBuilder<M, P> {
381+
impl<M: MemOps, P: BufferProvider + Clone> ChainBuilder<M, P> {
354382
fn new(mem: M, pool: P) -> Self {
355383
Self {
356384
mem,
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
Copyright 2026 The Hyperlight Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
//! A simple fixed-size buffer recycler for H2G prefill entries.
18+
//!
19+
//! Unlike [`super::BufferPool`] which uses a bitmap allocator, this
20+
//! holds a fixed set of same-sized buffer addresses in a free list.
21+
//! Alloc and dealloc are O(1). Intended for H2G writable buffers
22+
//! that are pre-allocated once and recycled after each use.
23+
24+
use alloc::sync::Arc;
25+
26+
use atomic_refcell::AtomicRefCell;
27+
use smallvec::SmallVec;
28+
29+
use super::{AllocError, Allocation, BufferProvider};
30+
31+
/// A recycling buffer provider with fixed-size slots.
32+
#[derive(Clone)]
33+
pub struct RecyclePool {
34+
inner: Arc<AtomicRefCell<RecyclePoolInner>>,
35+
}
36+
37+
struct RecyclePoolInner {
38+
base_addr: u64,
39+
slot_size: usize,
40+
count: usize,
41+
free: SmallVec<[u64; 64]>,
42+
}
43+
44+
impl RecyclePool {
45+
/// Create a new recycling pool by carving `base..base+region_len` into slots of `slot_size` bytes.
46+
pub fn new(base_addr: u64, region_len: usize, slot_size: usize) -> Result<Self, AllocError> {
47+
if slot_size == 0 {
48+
return Err(AllocError::InvalidArg);
49+
}
50+
51+
let count = region_len / slot_size;
52+
if count == 0 {
53+
return Err(AllocError::EmptyRegion);
54+
}
55+
56+
let mut free = SmallVec::with_capacity(count);
57+
for i in 0..count {
58+
free.push(base_addr + (i * slot_size) as u64);
59+
}
60+
61+
let inner = AtomicRefCell::new(RecyclePoolInner {
62+
base_addr,
63+
slot_size,
64+
count,
65+
free,
66+
});
67+
68+
Ok(Self {
69+
inner: inner.into(),
70+
})
71+
}
72+
73+
/// Number of free slots.
74+
pub fn num_free(&self) -> usize {
75+
self.inner.borrow().free.len()
76+
}
77+
}
78+
79+
impl BufferProvider for RecyclePool {
80+
fn alloc(&self, len: usize) -> Result<Allocation, AllocError> {
81+
let mut inner = self.inner.borrow_mut();
82+
if len > inner.slot_size {
83+
return Err(AllocError::OutOfMemory);
84+
}
85+
86+
let addr = inner.free.pop().ok_or(AllocError::OutOfMemory)?;
87+
88+
Ok(Allocation {
89+
addr,
90+
len: inner.slot_size,
91+
})
92+
}
93+
94+
fn dealloc(&self, alloc: Allocation) -> Result<(), AllocError> {
95+
let mut inner = self.inner.borrow_mut();
96+
inner.free.push(alloc.addr);
97+
Ok(())
98+
}
99+
100+
fn resize(&self, old: Allocation, new_len: usize) -> Result<Allocation, AllocError> {
101+
let inner = self.inner.borrow();
102+
if new_len > inner.slot_size {
103+
return Err(AllocError::OutOfMemory);
104+
}
105+
Ok(old)
106+
}
107+
108+
fn reset(&self) {
109+
let mut inner = self.inner.borrow_mut();
110+
let base = inner.base_addr;
111+
let slot = inner.slot_size;
112+
let count = inner.count;
113+
114+
inner.free.clear();
115+
116+
for i in 0..count {
117+
inner.free.push(base + (i * slot) as u64);
118+
}
119+
}
120+
}

0 commit comments

Comments
 (0)