Allow the guest to manage its own stack

This moves management of the guest stack into the guest, allowing the
guest to flexibly allocate pages from the scratch region to the stack.
This also makes the stack dynamically growable and essentially
unbounded, as is common on other architectures.

Signed-off-by: Lucy Menon <168595099+syntactically@users.noreply.github.com>

Author: syntactically

fuzz/fuzz_targets/guest_trace.rs

@@ -70,7 +70,7 @@ fuzz_target!(
     init: {
         let mut cfg = SandboxConfiguration::default();
         // In local tests, 256 KiB seemed sufficient for deep recursion
-        cfg.set_stack_size(256 * 1024);
+        cfg.set_scratch_size(256 * 1024);
         let path = simple_guest_for_fuzzing_as_string().expect("Guest Binary Missing");
         let u_sbox = UninitializedSandbox::new(
             GuestBinary::FilePath(path),

src/hyperlight_common/src/arch/amd64/layout.rs

@@ -30,3 +30,13 @@ pub const SNAPSHOT_PT_GVA_MAX: usize = 0xffff_80ff_ffff_ffff;
 /// bits, so we could consider bumping this in the future if we were
 /// ever memory-constrained.
 pub const MAX_GPA: usize = 0x0000_000f_ffff_ffff;
+
+/// On amd64, this is:
+/// - Two pages for the TSS and IDT
+/// - (up to) 4 pages for the PTEs for mapping that (including CoW'ing the root PT)
+/// - A page for the smallest possible non-exception stack
+/// - (up to) 3 pages for mapping that
+/// - Two pages for the exception stack and metadata
+pub fn min_scratch_size() -> usize {
+    12 * crate::vmem::PAGE_SIZE
+}

src/hyperlight_common/src/arch/i686/layout.rs

@@ -21,3 +21,7 @@ pub const MAX_GVA: usize = 0xffff_efff;
 pub const SNAPSHOT_PT_GVA_MIN: usize = 0xef00_0000;
 pub const SNAPSHOT_PT_GVA_MAX: usize = 0xefff_efff;
 pub const MAX_GPA: usize = 0xffff_ffff;
+
+pub fn min_scratch_size() -> usize {
+    1 * crate::vmem::PAGE_SIZE
+}

src/hyperlight_common/src/flatbuffer_wrappers/guest_error.rs

@@ -35,7 +35,6 @@ pub enum ErrorCode {
     GispatchFunctionPointerNotSet = 6,
     OutbError = 7,
     UnknownError = 8,
-    StackOverflow = 9,
     GsCheckFailed = 10,
     TooManyGuestFunctions = 11,
     FailureInDlmalloc = 12,
@@ -59,7 +58,6 @@ impl From<ErrorCode> for FbErrorCode {
             ErrorCode::GispatchFunctionPointerNotSet => Self::GispatchFunctionPointerNotSet,
             ErrorCode::OutbError => Self::OutbError,
             ErrorCode::UnknownError => Self::UnknownError,
-            ErrorCode::StackOverflow => Self::StackOverflow,
             ErrorCode::GsCheckFailed => Self::GsCheckFailed,
             ErrorCode::TooManyGuestFunctions => Self::TooManyGuestFunctions,
             ErrorCode::FailureInDlmalloc => Self::FailureInDlmalloc,
@@ -86,7 +84,6 @@ impl From<FbErrorCode> for ErrorCode {
             }
             FbErrorCode::GispatchFunctionPointerNotSet => Self::GispatchFunctionPointerNotSet,
             FbErrorCode::OutbError => Self::OutbError,
-            FbErrorCode::StackOverflow => Self::StackOverflow,
             FbErrorCode::GsCheckFailed => Self::GsCheckFailed,
             FbErrorCode::TooManyGuestFunctions => Self::TooManyGuestFunctions,
             FbErrorCode::FailureInDlmalloc => Self::FailureInDlmalloc,
@@ -113,7 +110,6 @@ impl From<u64> for ErrorCode {
             6 => Self::GispatchFunctionPointerNotSet,
             7 => Self::OutbError,
             8 => Self::UnknownError,
-            9 => Self::StackOverflow,
             10 => Self::GsCheckFailed,
             11 => Self::TooManyGuestFunctions,
             12 => Self::FailureInDlmalloc,
@@ -138,7 +134,6 @@ impl From<ErrorCode> for u64 {
             ErrorCode::GispatchFunctionPointerNotSet => 6,
             ErrorCode::OutbError => 7,
             ErrorCode::UnknownError => 8,
-            ErrorCode::StackOverflow => 9,
             ErrorCode::GsCheckFailed => 10,
             ErrorCode::TooManyGuestFunctions => 11,
             ErrorCode::FailureInDlmalloc => 12,
@@ -164,7 +159,6 @@ impl From<ErrorCode> for String {
             ErrorCode::GispatchFunctionPointerNotSet => "GispatchFunctionPointerNotSet".to_string(),
             ErrorCode::OutbError => "OutbError".to_string(),
             ErrorCode::UnknownError => "UnknownError".to_string(),
-            ErrorCode::StackOverflow => "StackOverflow".to_string(),
             ErrorCode::GsCheckFailed => "GsCheckFailed".to_string(),
             ErrorCode::TooManyGuestFunctions => "TooManyGuestFunctions".to_string(),
             ErrorCode::FailureInDlmalloc => "FailureInDlmalloc".to_string(),

src/hyperlight_common/src/flatbuffers/hyperlight/generated/error_code_generated.rs

@@ -25,7 +25,7 @@ pub const ENUM_MAX_ERROR_CODE: u64 = 17;
     note = "Use associated constants instead. This will no longer be generated in 2021."
 )]
 #[allow(non_camel_case_types)]
-pub const ENUM_VALUES_ERROR_CODE: [ErrorCode; 17] = [
+pub const ENUM_VALUES_ERROR_CODE: [ErrorCode; 16] = [
     ErrorCode::NoError,
     ErrorCode::UnsupportedParameterType,
     ErrorCode::GuestFunctionNameNotProvided,
@@ -34,7 +34,6 @@ pub const ENUM_VALUES_ERROR_CODE: [ErrorCode; 17] = [
     ErrorCode::GispatchFunctionPointerNotSet,
     ErrorCode::OutbError,
     ErrorCode::UnknownError,
-    ErrorCode::StackOverflow,
     ErrorCode::GsCheckFailed,
     ErrorCode::TooManyGuestFunctions,
     ErrorCode::FailureInDlmalloc,
@@ -58,7 +57,6 @@ impl ErrorCode {
     pub const GispatchFunctionPointerNotSet: Self = Self(6);
     pub const OutbError: Self = Self(7);
     pub const UnknownError: Self = Self(8);
-    pub const StackOverflow: Self = Self(9);
     pub const GsCheckFailed: Self = Self(10);
     pub const TooManyGuestFunctions: Self = Self(11);
     pub const FailureInDlmalloc: Self = Self(12);
@@ -79,7 +77,6 @@ impl ErrorCode {
         Self::GispatchFunctionPointerNotSet,
         Self::OutbError,
         Self::UnknownError,
-        Self::StackOverflow,
         Self::GsCheckFailed,
         Self::TooManyGuestFunctions,
         Self::FailureInDlmalloc,
@@ -102,7 +99,6 @@ impl ErrorCode {
             Self::GispatchFunctionPointerNotSet => Some("GispatchFunctionPointerNotSet"),
             Self::OutbError => Some("OutbError"),
             Self::UnknownError => Some("UnknownError"),
-            Self::StackOverflow => Some("StackOverflow"),
             Self::GsCheckFailed => Some("GsCheckFailed"),
             Self::TooManyGuestFunctions => Some("TooManyGuestFunctions"),
             Self::FailureInDlmalloc => Some("FailureInDlmalloc"),

src/hyperlight_common/src/layout.rs

@@ -32,3 +32,6 @@ pub fn scratch_base_gpa(size: usize) -> u64 {
 pub fn scratch_base_gva(size: usize) -> u64 {
     (MAX_GVA - size + 1) as u64
 }
+
+/// Compute the minimum scratch region size needed for a sandbox.
+pub use arch::min_scratch_size;

src/hyperlight_common/src/mem.rs

@@ -28,16 +28,6 @@ pub struct GuestMemoryRegion {
     pub ptr: u64,
 }
 
-/// A memory region in the guest address space that is used for the stack
-#[derive(Debug, Clone, Copy)]
-#[repr(C)]
-pub struct GuestStack {
-    /// The top of the user stack
-    pub min_user_stack_address: u64,
-    /// The user stack pointer
-    pub user_stack_address: u64,
-}
-
 #[derive(Debug, Clone, Copy)]
 #[repr(C)]
 pub struct HyperlightPEB {
@@ -46,5 +36,4 @@ pub struct HyperlightPEB {
     pub output_stack: GuestMemoryRegion,
     pub init_data: GuestMemoryRegion,
     pub guest_heap: GuestMemoryRegion,
-    pub guest_stack: GuestStack,
 }

src/hyperlight_guest/src/arch/amd64/layout.rs

@@ -18,7 +18,13 @@ limitations under the License.
 // src/hyperlight_common/src/arch/amd64/layout.rs and
 // src/hyperlight_guest_bin/src/arch/amd64/layout.rs
 
-pub const MAIN_STACK_TOP_GVA: u64 = 0xffff_feff_ffff_f000;
+/// Note that the x86-64 ELF psABI requires that the stack be 16-byte
+/// aligned before a call instruction; we use the aligned version
+/// here, even though this requires adjusting the pointer by 8 bytes
+/// when entering the guest without a call instruction to push a
+/// return address.
+pub const MAIN_STACK_TOP_GVA: u64 = 0xffff_ff00_0000_0000;
+pub const MAIN_STACK_LIMIT_GVA: u64 = 0xffff_fe00_0000_0000;
 
 pub fn scratch_size() -> u64 {
     let addr = crate::layout::scratch_size_gva();

src/hyperlight_guest/src/arch/i686/layout.rs

@@ -18,6 +18,7 @@ limitations under the License.
 // allow compiling the guest for real mode boot scenarios.
 
 pub const MAIN_STACK_TOP_GVA: usize = 0xdfff_efff;
+pub const MAIN_STACK_LIMIT_GVA: usize = 0xdf00_0000;
 
 pub fn scratch_size() -> u64 {
     hyperlight_common::vmem::PAGE_SIZE as u64

src/hyperlight_guest/src/layout.rs

@@ -18,7 +18,7 @@ limitations under the License.
 #[cfg_attr(target_arch = "x86", path = "arch/i686/layout.rs")]
 mod arch;
 
-pub use arch::MAIN_STACK_TOP_GVA;
+pub use arch::{MAIN_STACK_LIMIT_GVA, MAIN_STACK_TOP_GVA};
 pub fn scratch_size_gva() -> *mut u64 {
     use hyperlight_common::layout::{MAX_GVA, SCRATCH_TOP_SIZE_OFFSET};
     (MAX_GVA as u64 - SCRATCH_TOP_SIZE_OFFSET + 1) as *mut u64