Kernel loading into higher half memory

.harper-dictionary.txt

@@ -18,10 +18,12 @@ RV64
 SBI
 SPL
 StarFive
+Sv39
 VisionFive
 devicetree
 devicetrees
 representable
+stackless
 u32
 usize
 v0

kernel/linker.ld.in

@@ -1,48 +1,153 @@
 ENTRY(_start)
 
-BASE_ADDRESS = @kernelAddress@;
+PAGE_SIZE = 4K;
+GIGAPAGE_SIZE = 0x40000000; /* 1 GiB */
+
+KERNEL_HIGH_BASE = 0xffffffff80000000;
+
+KERNEL_OFFSET = @kernelOffset@;
+
+KERNEL_PHYSICAL_BASE = @kernelAddress@;
+KERNEL_VIRTUAL_BASE = KERNEL_HIGH_BASE + KERNEL_OFFSET;
+
+KERNEL_REGION_SIZE = @kernelRegionSize@;
 
 ASSERT(
-  (BASE_ADDRESS % 4K) == 0,
-  "kernel load address must be 4 KiB aligned"
+  (KERNEL_OFFSET % PAGE_SIZE) == 0,
+  "kernel offset must be page aligned"
 )
 
 ASSERT(
-  _kernel_end > _kernel_start,
-  "kernel image must be non-empty"
+  KERNEL_OFFSET < GIGAPAGE_SIZE,
+  "kernel offset must stay within the bootstrap gigapage"
+)
+
+ASSERT(
+  (KERNEL_HIGH_BASE % GIGAPAGE_SIZE) == 0,
+  "kernel high-half base must be 1 GiB aligned"
+)
+
+ASSERT(
+  (KERNEL_PHYSICAL_BASE % GIGAPAGE_SIZE) == KERNEL_OFFSET,
+  "kernel physical address must match its offset within the bootstrap gigapage"
 )
 
-MEMORY
-{
-  KERNEL (rwx) : ORIGIN = @kernelAddress@, LENGTH = @kernelRegionSize@
-}
 
 SECTIONS
 {
-  . = BASE_ADDRESS;
+  /*
+   * Physical bootstrap
+   */
+
+  . = KERNEL_PHYSICAL_BASE;
+
+  _kernel_physical_start = .;
+
+  /*
+   * Early bootstrap code and constants.
+   */
+  .boot : {
+    KEEP(*(.boot.text))
+    KEEP(*(.boot.text.*))
+
+    KEEP(*(.boot.rodata))
+    KEEP(*(.boot.rodata.*))
+  }
 
-  _kernel_start = .;
+  /*
+   * Reserve one 4 KiB page for the temporary Sv39 root page table.
+   */
+  . = ALIGN(PAGE_SIZE);
+
+  .boot.page_table (NOLOAD) : {
+    __boot_page_table = .;
+
+    . += PAGE_SIZE;
+
+    __boot_page_table_end = .;
+  }
+
+  . = ALIGN(PAGE_SIZE);
+
+  _boot_physical_end = .;
+
+  /*
+   * Physical space consumed before the normal higher-half kernel begins.
+   */
+  _boot_size = _boot_physical_end - KERNEL_PHYSICAL_BASE;
+
+
+  /*
+   * Higher-half kernel
+   */
+  _kernel_virtual_start = KERNEL_VIRTUAL_BASE;
+
+  . = KERNEL_VIRTUAL_BASE + _boot_size;
+
+  . = ALIGN(PAGE_SIZE);
+
+
+  /*
+   * Executable kernel code.
+   *
+   * VMA:
+   *   high-half virtual address
+   *
+   * LMA:
+   *   corresponding physical address inside the loaded kernel image
+   */
+  .text : AT(
+    KERNEL_PHYSICAL_BASE
+      + (ADDR(.text) - KERNEL_VIRTUAL_BASE)
+  ) {
+    _text_start = .;
 
-  .text : {
     KEEP(*(.text.init))
     *(.text .text.*)
-  } > KERNEL
 
-  .rodata : {
+    _text_end = .;
+  }
+
+  . = ALIGN(PAGE_SIZE);
+
+  .rodata : AT(
+    KERNEL_PHYSICAL_BASE
+      + (ADDR(.rodata) - KERNEL_VIRTUAL_BASE)
+  ) {
+    _rodata_start = .;
+
     *(.srodata .srodata.*)
     *(.rodata .rodata.*)
-  } > KERNEL
 
-  .data : {
+    _rodata_end = .;
+  }
+
+  . = ALIGN(PAGE_SIZE);
+
+
+  .data : AT(
+    KERNEL_PHYSICAL_BASE
+      + (ADDR(.data) - KERNEL_VIRTUAL_BASE)
+  ) {
+    _data_start = .;
+
     . = ALIGN(8);
 
     __global_pointer$ = . + 0x800;
 
     *(.sdata .sdata.*)
     *(.data .data.*)
-  } > KERNEL
 
-  .bss (NOLOAD) : {
+    _data_end = .;
+  }
+
+  . = ALIGN(PAGE_SIZE);
+
+
+  .bss (NOLOAD) : AT(
+    KERNEL_PHYSICAL_BASE
+      + (ADDR(.bss) - KERNEL_VIRTUAL_BASE)
+  ) {
     . = ALIGN(8);
 
     _bss_start = .;
@@ -52,16 +157,67 @@ SECTIONS
     *(COMMON)
 
     . = ALIGN(8);
+
     _bss_end = .;
-  } > KERNEL
+  }
+
+  . = ALIGN(PAGE_SIZE);
 
-  .stack (NOLOAD) : {
+  .stack (NOLOAD) : AT(
+    KERNEL_PHYSICAL_BASE
+      + (ADDR(.stack) - KERNEL_VIRTUAL_BASE)
+  ) {
     . = ALIGN(16);
 
     _stack_start = .;
+
     . += 16K;
+
     _stack_end = .;
-  } > KERNEL
+  }
+
+  . = ALIGN(PAGE_SIZE);
+
+  _kernel_virtual_end = .;
 
-  _kernel_end = .;
+
+  /*
+   * Translate the final virtual offset back into its physical equivalent.
+   */
+  _kernel_physical_end =
+    KERNEL_PHYSICAL_BASE
+      + (_kernel_virtual_end - KERNEL_VIRTUAL_BASE);
 }
+
+ASSERT(
+  _start == KERNEL_PHYSICAL_BASE,
+  "bootstrap entry must begin at the physical kernel load address"
+)
+
+ASSERT(
+  (__boot_page_table_end - __boot_page_table) == PAGE_SIZE,
+  "bootstrap page table must occupy exactly one page"
+)
+
+ASSERT(
+  _kernel_physical_end > _kernel_physical_start,
+  "kernel image must be non-empty"
+)
+
+ASSERT(
+  _kernel_physical_end
+    <= (KERNEL_PHYSICAL_BASE + KERNEL_REGION_SIZE),
+  "kernel runtime footprint exceeds reserved physical region"
+)
+
+ASSERT(
+  _kernel_physical_end
+    <= ((KERNEL_PHYSICAL_BASE - KERNEL_OFFSET) + GIGAPAGE_SIZE),
+  "kernel runtime footprint crosses the bootstrap physical gigapage"
+)
+
+ASSERT(
+  _kernel_virtual_end
+    <= (KERNEL_HIGH_BASE + GIGAPAGE_SIZE),
+  "kernel runtime footprint crosses the bootstrap virtual gigapage"
+)

kernel/src/arch/riscv64.rs

@@ -2,72 +2,12 @@
 //!
 //! This module provides the architecture-specific boundary between the
 //! kernel and an RV64 execution environment.
-//!
-//! # Kernel entry
-//!
-//! The `_start` entry point establishes the minimum execution environment
-//! required before entering Rust:
-//!
-//! 1. Initialize the global pointer (`gp`).
-//! 2. Initialize the stack pointer (`sp`).
-//! 3. Clear the `.bss` section.
-//! 4. Transfer control to [`crate::main`].
-//!
-//! The firmware-provided `a0` and `a1` registers are deliberately preserved
-//! so they are passed to `main` as the hart ID and device-tree address,
-//! respectively.
-//!
-//! The linker script provides the symbols used during initialization,
-//! including `__global_pointer$`, `_stack_end`, `_bss_start`, and `_bss_end`.
-//!
-//! `_bss_start` and `_bss_end` are 8-byte aligned because startup clears
-//! `.bss` using 8-byte stores.
 
+mod boot;
 mod trap;
 pub(crate) use trap::init as init_trap;
 
-use core::arch::{asm, global_asm};
-
-// Kernel entry point.
-//
-// Establish the minimum execution environment required by Rust before
-// transferring control to `main`.
-//
-// `a0` and `a1` are deliberately preserved so they remain the first two
-// arguments passed to `main`.
-global_asm!(
-  r#"
-  .section .text.init
-  .global _start
-
-_start:
-  .option push
-  .option norelax
-
-  /* Initialize gp */
-  la gp, __global_pointer$
-
-  .option pop
-
-  /* Initialize stack */
-  la sp, _stack_end
-
-  /* Clear .bss */
-  la t0, _bss_start
-  la t1, _bss_end
-
-.bss_loop:
-  bgeu t0, t1, .bss_done
-
-  sd zero, 0(t0)
-  addi t0, t0, 8
-
-  j .bss_loop
-
-.bss_done:
-  tail main
-  "#
-);
+use core::arch::asm;
 
 /// Parks the current hart indefinitely.
 ///