Skip to content

Allow the guest to manage its own stack #1205

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
syntactically merged 7 commits into from
Feb 4, 2026
Merged

Allow the guest to manage its own stack #1205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9095115
amd64: move processor control structures to snapshot region
syntactically Jan 28, 2026
2b783dd
amd64: add exception stack at top of scratch memory
syntactically Jan 28, 2026
6104cbc
Remove the remainder of host-managed stack cookies
syntactically Jan 28, 2026
5907be8
Add test for creating sandbox from snapshot from sandbox
syntactically Jan 29, 2026
620b5e8
Allow the guest to manage its own stack
syntactically Feb 2, 2026
91af3e2
[just] Ensure that fuzzers compile on `like-ci`
syntactically Feb 3, 2026
78b07d7
[nix] update, refactor, add dependencies
syntactically Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions Justfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,10 +164,17 @@ run-examples-like-ci config=default-target hypervisor="kvm":
@# Run Rust examples - linux
{{ if os() == "linux" { "just run-rust-examples-linux " + config + " " } else { "" } }}

benchmarks-like-ci config=default-target hypervisor="$vm":
benchmarks-like-ci config=default-target hypervisor="kvm":
@# Run benchmarks
{{ if config == "release" { "just bench-ci main" } else { "" } }}

fuzz-like-ci target config=default-target hypervisor="kvm":
@# Run Fuzzing
# Use a much shorter time limit (1 vs 300 seconds), because the
# local version of this step is mostly intended just for making
# sure that the fuzz harnesses compile
{{ if config == "release" { "just fuzz-timed " + target + " 1" } else { "" } }}

like-ci config=default-target hypervisor="kvm":
@# .github/workflows/dep_code_checks.yml
just code-checks-like-ci {{config}} {{hypervisor}}
Expand All @@ -184,7 +191,12 @@ like-ci config=default-target hypervisor="kvm":
@# .github/workflows/dep_benchmarks.yml
just benchmarks-like-ci {{config}} {{hypervisor}}

@# can't run fuzzing locally
@# .github/workflows/dep_fuzzing.yml
just fuzz-like-ci fuzz_host_print {{config}} {{hypervisor}}
just fuzz-like-ci fuzz_guest_call {{config}} {{hypervisor}}
just fuzz-like-ci fuzz_host_call {{config}} {{hypervisor}}
just fuzz-like-ci fuzz_guest_estimate_trace_event {{config}} {{hypervisor}}
just fuzz-like-ci fuzz_guest_trace {{config}} {{hypervisor}}

@# spelling
typos
Expand Down
12 changes: 6 additions & 6 deletions flake.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

122 changes: 68 additions & 54 deletions flake.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,65 +2,77 @@
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
inputs.nixpkgs-mozilla.url = "github:mozilla/nixpkgs-mozilla/master";
outputs = { self, nixpkgs, nixpkgs-mozilla, ... } @ inputs:
{
rec {
overlays.fix-rust = self: super: {
# Work around the nixpkgs-mozilla equivalent of
# https://github.com/NixOS/nixpkgs/issues/278508 and an
# incompatibility between nixpkgs-mozilla and makeRustPlatform
rustChannelOf = args: let
orig = super.rustChannelOf args;
patchRustPkg = pkg: (pkg.overrideAttrs (oA: {
buildCommand = (builtins.replaceStrings
[ "rustc,rustdoc" ]
[ "rustc,rustdoc,clippy-driver,cargo-clippy,miri,cargo-miri" ]
oA.buildCommand) + (let
wrapperPath = self.path + "/pkgs/build-support/bintools-wrapper/ld-wrapper.sh";
baseOut = self.clangStdenv.cc.bintools.out;
getStdenvAttrs = drv: (drv.overrideAttrs (oA: {
passthru.origAttrs = oA;
})).origAttrs;
baseEnv = (getStdenvAttrs self.clangStdenv.cc.bintools).env;
baseSubstitutedWrapper = self.replaceVars wrapperPath
{
inherit (baseEnv)
shell coreutils_bin suffixSalt mktemp rm;
use_response_file_by_default = "0";
prog = null;
out = null;
};
in ''
# work around a bug in the overlay
${oA.postInstall}

# copy over helper scripts that the wrapper needs
(cd "${baseOut}"; find . -type f \( -name '*.sh' -or -name '*.bash' \) -print0) | while read -d $'\0' script; do
mkdir -p "$out/$(dirname "$script")"
substitute "${baseOut}/$script" "$out/$script" --replace-quiet "${baseOut}" "$out"
done

# TODO: Work out how to make this work with cross builds
ldlld="$out/lib/rustlib/${self.clangStdenv.targetPlatform.config}/bin/gcc-ld/ld.lld";
if [ -e "$ldlld" ]; then
export prog="$(readlink -f "$ldlld")"
rm "$ldlld"
substitute ${baseSubstitutedWrapper} "$ldlld" --subst-var "out" --subst-var "prog"
chmod +x "$ldlld"
fi
'');
})) // {
targetPlatforms = [ "x86_64-linux" ];
badTargetPlatforms = [ ];
};
overrideRustPkg = pkg: self.lib.makeOverridable (origArgs:
patchRustPkg (pkg.override origArgs)
) {};
in builtins.mapAttrs (_: overrideRustPkg) orig;
};
gcroots =
let gcrootForShell = pkg: pkg // derivation (pkg.drvAttrs // {
origArgs = pkg.drvAttrs.args;
# assume the builder is bash for now (it always is for
# stdenv, which is the only thing that we will encounter
# in this flake).
args = [ "-c" "declare > $out" ];
});
in {
shells.default = gcrootForShell devShells.x86_64-linux.default;
};
devShells.x86_64-linux.default =
let pkgs = import nixpkgs {
system = "x86_64-linux";
overlays = [ (import (nixpkgs-mozilla + "/rust-overlay.nix")) ];
overlays = [ (import (nixpkgs-mozilla + "/rust-overlay.nix")) overlays.fix-rust ];
};
in with pkgs; let
# Work around the nixpkgs-mozilla equivalent of
# https://github.com/NixOS/nixpkgs/issues/278508 and an
# incompatibility between nixpkgs-mozilla and makeRustPlatform
rustChannelOf = args: let
orig = pkgs.rustChannelOf args;
patchRustPkg = pkg: (pkg.overrideAttrs (oA: {
buildCommand = (builtins.replaceStrings
[ "rustc,rustdoc" ]
[ "rustc,rustdoc,clippy-driver,cargo-clippy,miri,cargo-miri" ]
oA.buildCommand) + (let
wrapperPath = pkgs.path + "/pkgs/build-support/bintools-wrapper/ld-wrapper.sh";
baseOut = pkgs.clangStdenv.cc.bintools.out;
getStdenvAttrs = drv: (drv.overrideAttrs (oA: {
passthru.origAttrs = oA;
})).origAttrs;
baseEnv = (getStdenvAttrs pkgs.clangStdenv.cc.bintools).env;
baseSubstitutedWrapper = pkgs.replaceVars wrapperPath
{
inherit (baseEnv)
shell coreutils_bin suffixSalt mktemp rm;
use_response_file_by_default = "0";
prog = null;
out = null;
};
in ''
# work around a bug in the overlay
${oA.postInstall}

# copy over helper scripts that the wrapper needs
(cd "${baseOut}"; find . -type f \( -name '*.sh' -or -name '*.bash' \) -print0) | while read -d $'\0' script; do
mkdir -p "$out/$(dirname "$script")"
substitute "${baseOut}/$script" "$out/$script" --replace-quiet "${baseOut}" "$out"
done

# TODO: Work out how to make this work with cross builds
ldlld="$out/lib/rustlib/${pkgs.clangStdenv.targetPlatform.config}/bin/gcc-ld/ld.lld";
if [ -e "$ldlld" ]; then
export prog="$(readlink -f "$ldlld")"
rm "$ldlld"
substitute ${baseSubstitutedWrapper} "$ldlld" --subst-var "out" --subst-var "prog"
chmod +x "$ldlld"
fi
'');
})) // {
targetPlatforms = [ "x86_64-linux" ];
badTargetPlatforms = [ ];
};
overrideRustPkg = pkg: lib.makeOverridable (origArgs:
patchRustPkg (pkg.override origArgs)
) {};
in builtins.mapAttrs (_: overrideRustPkg) orig;

customisedRustChannelOf = args:
lib.flip builtins.mapAttrs (rustChannelOf args) (_: pkg: pkg.override {
targets = [
Expand Down Expand Up @@ -246,6 +258,8 @@
zlib
cargo-hyperlight
typos
flatbuffers
cargo-fuzz
];
buildInputs = [
pango
Expand Down
2 changes: 1 addition & 1 deletion fuzz/fuzz_targets/guest_trace.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ fuzz_target!(
init: {
let mut cfg = SandboxConfiguration::default();
// In local tests, 256 KiB seemed sufficient for deep recursion
cfg.set_stack_size(256 * 1024);
cfg.set_scratch_size(256 * 1024);
let path = simple_guest_for_fuzzing_as_string().expect("Guest Binary Missing");
let u_sbox = UninitializedSandbox::new(
GuestBinary::FilePath(path),
Expand Down
14 changes: 14 additions & 0 deletions src/hyperlight_common/src/arch/amd64/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// The addresses in this file should be coordinated with
// src/hyperlight_guest/src/arch/amd64/layout.rs and
// src/hyperlight_guest_bin/src/arch/amd64/layout.rs

/// We have this the top of the page below the top of memory in order
/// to make working with start/end ptrs in a few places more
/// convenient (not needing to worry about overflow)
Expand All @@ -26,3 +30,13 @@ pub const SNAPSHOT_PT_GVA_MAX: usize = 0xffff_80ff_ffff_ffff;
/// bits, so we could consider bumping this in the future if we were
/// ever memory-constrained.
pub const MAX_GPA: usize = 0x0000_000f_ffff_ffff;

/// On amd64, this is:
/// - Two pages for the TSS and IDT
/// - (up to) 4 pages for the PTEs for mapping that (including CoW'ing the root PT)
/// - A page for the smallest possible non-exception stack
/// - (up to) 3 pages for mapping that
/// - Two pages for the exception stack and metadata
pub fn min_scratch_size() -> usize {
12 * crate::vmem::PAGE_SIZE
}
4 changes: 4 additions & 0 deletions src/hyperlight_common/src/arch/i686/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,7 @@ pub const MAX_GVA: usize = 0xffff_efff;
pub const SNAPSHOT_PT_GVA_MIN: usize = 0xef00_0000;
pub const SNAPSHOT_PT_GVA_MAX: usize = 0xefff_efff;
pub const MAX_GPA: usize = 0xffff_ffff;

pub fn min_scratch_size() -> usize {
1 * crate::vmem::PAGE_SIZE
}
6 changes: 0 additions & 6 deletions src/hyperlight_common/src/flatbuffer_wrappers/guest_error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ pub enum ErrorCode {
GispatchFunctionPointerNotSet = 6,
OutbError = 7,
UnknownError = 8,
StackOverflow = 9,
GsCheckFailed = 10,
TooManyGuestFunctions = 11,
FailureInDlmalloc = 12,
Expand All @@ -59,7 +58,6 @@ impl From<ErrorCode> for FbErrorCode {
ErrorCode::GispatchFunctionPointerNotSet => Self::GispatchFunctionPointerNotSet,
ErrorCode::OutbError => Self::OutbError,
ErrorCode::UnknownError => Self::UnknownError,
ErrorCode::StackOverflow => Self::StackOverflow,
ErrorCode::GsCheckFailed => Self::GsCheckFailed,
ErrorCode::TooManyGuestFunctions => Self::TooManyGuestFunctions,
ErrorCode::FailureInDlmalloc => Self::FailureInDlmalloc,
Expand All @@ -86,7 +84,6 @@ impl From<FbErrorCode> for ErrorCode {
}
FbErrorCode::GispatchFunctionPointerNotSet => Self::GispatchFunctionPointerNotSet,
FbErrorCode::OutbError => Self::OutbError,
FbErrorCode::StackOverflow => Self::StackOverflow,
FbErrorCode::GsCheckFailed => Self::GsCheckFailed,
FbErrorCode::TooManyGuestFunctions => Self::TooManyGuestFunctions,
FbErrorCode::FailureInDlmalloc => Self::FailureInDlmalloc,
Expand All @@ -113,7 +110,6 @@ impl From<u64> for ErrorCode {
6 => Self::GispatchFunctionPointerNotSet,
7 => Self::OutbError,
8 => Self::UnknownError,
9 => Self::StackOverflow,
10 => Self::GsCheckFailed,
11 => Self::TooManyGuestFunctions,
12 => Self::FailureInDlmalloc,
Expand All @@ -138,7 +134,6 @@ impl From<ErrorCode> for u64 {
ErrorCode::GispatchFunctionPointerNotSet => 6,
ErrorCode::OutbError => 7,
ErrorCode::UnknownError => 8,
ErrorCode::StackOverflow => 9,
ErrorCode::GsCheckFailed => 10,
ErrorCode::TooManyGuestFunctions => 11,
ErrorCode::FailureInDlmalloc => 12,
Expand All @@ -164,7 +159,6 @@ impl From<ErrorCode> for String {
ErrorCode::GispatchFunctionPointerNotSet => "GispatchFunctionPointerNotSet".to_string(),
ErrorCode::OutbError => "OutbError".to_string(),
ErrorCode::UnknownError => "UnknownError".to_string(),
ErrorCode::StackOverflow => "StackOverflow".to_string(),
ErrorCode::GsCheckFailed => "GsCheckFailed".to_string(),
ErrorCode::TooManyGuestFunctions => "TooManyGuestFunctions".to_string(),
ErrorCode::FailureInDlmalloc => "FailureInDlmalloc".to_string(),
Expand Down
6 changes: 1 addition & 5 deletions src/hyperlight_common/src/flatbuffers/hyperlight/generated/error_code_generated.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ pub const ENUM_MAX_ERROR_CODE: u64 = 17;
note = "Use associated constants instead. This will no longer be generated in 2021."
)]
#[allow(non_camel_case_types)]
pub const ENUM_VALUES_ERROR_CODE: [ErrorCode; 17] = [
pub const ENUM_VALUES_ERROR_CODE: [ErrorCode; 16] = [
ErrorCode::NoError,
ErrorCode::UnsupportedParameterType,
ErrorCode::GuestFunctionNameNotProvided,
Expand All @@ -34,7 +34,6 @@ pub const ENUM_VALUES_ERROR_CODE: [ErrorCode; 17] = [
ErrorCode::GispatchFunctionPointerNotSet,
ErrorCode::OutbError,
ErrorCode::UnknownError,
ErrorCode::StackOverflow,
ErrorCode::GsCheckFailed,
ErrorCode::TooManyGuestFunctions,
ErrorCode::FailureInDlmalloc,
Expand All @@ -58,7 +57,6 @@ impl ErrorCode {
pub const GispatchFunctionPointerNotSet: Self = Self(6);
pub const OutbError: Self = Self(7);
pub const UnknownError: Self = Self(8);
pub const StackOverflow: Self = Self(9);
pub const GsCheckFailed: Self = Self(10);
pub const TooManyGuestFunctions: Self = Self(11);
pub const FailureInDlmalloc: Self = Self(12);
Expand All @@ -79,7 +77,6 @@ impl ErrorCode {
Self::GispatchFunctionPointerNotSet,
Self::OutbError,
Self::UnknownError,
Self::StackOverflow,
Self::GsCheckFailed,
Self::TooManyGuestFunctions,
Self::FailureInDlmalloc,
Expand All @@ -102,7 +99,6 @@ impl ErrorCode {
Self::GispatchFunctionPointerNotSet => Some("GispatchFunctionPointerNotSet"),
Self::OutbError => Some("OutbError"),
Self::UnknownError => Some("UnknownError"),
Self::StackOverflow => Some("StackOverflow"),
Self::GsCheckFailed => Some("GsCheckFailed"),
Self::TooManyGuestFunctions => Some("TooManyGuestFunctions"),
Self::FailureInDlmalloc => Some("FailureInDlmalloc"),
Expand Down
3 changes: 3 additions & 0 deletions src/hyperlight_common/src/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,6 @@ pub fn scratch_base_gpa(size: usize) -> u64 {
pub fn scratch_base_gva(size: usize) -> u64 {
(MAX_GVA - size + 1) as u64
}

/// Compute the minimum scratch region size needed for a sandbox.
pub use arch::min_scratch_size;
12 changes: 0 additions & 12 deletions src/hyperlight_common/src/mem.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,12 @@ pub struct GuestMemoryRegion {
pub ptr: u64,
}

/// A memory region in the guest address space that is used for the stack
#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct GuestStack {
/// The top of the user stack
pub min_user_stack_address: u64,
/// The user stack pointer
pub user_stack_address: u64,
}

#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct HyperlightPEB {
pub security_cookie_seed: u64,
pub guest_function_dispatch_ptr: u64,
pub input_stack: GuestMemoryRegion,
pub output_stack: GuestMemoryRegion,
pub init_data: GuestMemoryRegion,
pub guest_heap: GuestMemoryRegion,
pub guest_stack: GuestStack,
}
12 changes: 11 additions & 1 deletion src/hyperlight_guest/src/arch/amd64/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,17 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

pub const MAIN_STACK_TOP_GVA: usize = 0xffff_feff_ffff_f000;
// The addresses in this file should be coordinated with
// src/hyperlight_common/src/arch/amd64/layout.rs and
// src/hyperlight_guest_bin/src/arch/amd64/layout.rs

/// Note that the x86-64 ELF psABI requires that the stack be 16-byte
/// aligned before a call instruction; we use the aligned version
/// here, even though this requires adjusting the pointer by 8 bytes
/// when entering the guest without a call instruction to push a
/// return address.
pub const MAIN_STACK_TOP_GVA: u64 = 0xffff_ff00_0000_0000;
pub const MAIN_STACK_LIMIT_GVA: u64 = 0xffff_fe00_0000_0000;

pub fn scratch_size() -> u64 {
let addr = crate::layout::scratch_size_gva();
Expand Down
Loading
Loading