From 5d72805d2fc354ff07910f20d8c46b9e10b7a84e Mon Sep 17 00:00:00 2001
From: Tulio A M Mendes <tadryanom@hotmail.com>
Date: Mon, 25 May 2026 16:57:13 -0300
Subject: [PATCH] kernel: implement NX (No-Execute) support via IA32_EFER.NXE

Fix A01 (W^X/NX) which was deferred due to IA32_EFER.NXE MSR instability.
Root cause: NX bit was being set in PTEs without NXE enabled, causing
undefined behavior and kernel panic.

Changes:
- boot.S: Check CPUID.0x80000001:EDX bit 20 for NX support before enabling
- boot.S: Enable IA32_EFER.NXE (MSR 0xC0000080, bit 11) if NX supported
- vmm.c: Add g_nxe_enabled flag and check_nxe_enabled() function
- vmm.c: Conditionalize X86_PTE_NX usage based on g_nxe_enabled
- vmm.c: Print NX status in vmm_init()
- Makefile: Add -cpu qemu32,+nx to expose NX support in QEMU
- smoke_test.exp: Add -cpu qemu32,+nx for testing

Behavior:
- With NX support: NXE enabled, VMM uses NX bit for non-executable pages
- Without NX support: NXE not enabled, VMM ignores VMM_FLAG_NX
- W^X now works correctly for ELF loading, mmap/mprotect, etc.

Test: 119/119 PASS (SMP=4)
---
 Makefile             |  1 +
 src/arch/x86/boot.S  | 17 +++++++++++++++++
 src/arch/x86/vmm.c   | 19 ++++++++++++++++++-
 tests/smoke_test.exp |  3 ++-
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 77d0a1f9..96215b60 100644
--- a/Makefile
+++ b/Makefile
@@ -287,6 +287,7 @@ run: iso
 		-drive file=disk.img,if=ide,format=raw \
 		-nic user,model=e1000 \
 		-serial file:serial.log -monitor none -no-reboot -no-shutdown \
+		-cpu qemu32,+nx \
 		$(QEMU_DFLAGS)
 
 run-arm: adros-arm.bin
diff --git a/src/arch/x86/boot.S b/src/arch/x86/boot.S
index e17e1a36..09fff385 100644
--- a/src/arch/x86/boot.S
+++ b/src/arch/x86/boot.S
@@ -207,6 +207,23 @@ _start:
     or $0x20, %ecx              /* CR4.PAE = bit 5 */
     mov %ecx, %cr4
 
+    /* --- Check for NX support via CPUID.0x80000001:EDX bit 20 --- */
+    mov $0x80000000, %eax       /* Check max extended leaf */
+    cpuid
+    cmp $0x80000001, %eax       /* Need leaf 0x80000001 */
+    jb no_nxe                   /* Skip if not available */
+    mov $0x80000001, %eax       /* Get extended feature flags */
+    cpuid
+    bt $20, %edx                /* Test bit 20 (NX) */
+    jnc no_nxe                  /* Skip if NX not supported */
+
+    /* --- Enable NXE in IA32_EFER (MSR 0xC0000080, bit 11) --- */
+    mov $0xC0000080, %ecx       /* IA32_EFER MSR */
+    rdmsr
+    or $0x800, %eax             /* Set bit 11 (NXE) */
+    wrmsr
+
+no_nxe:
     /* --- Load CR3 with PDPT physical address --- */
     mov $boot_pdpt, %ecx
     sub $KERNEL_VIRT_BASE, %ecx
diff --git a/src/arch/x86/vmm.c b/src/arch/x86/vmm.c
index 0934022b..68f4e494 100644
--- a/src/arch/x86/vmm.c
+++ b/src/arch/x86/vmm.c
@@ -67,11 +67,19 @@ extern uint64_t boot_pd3[512];
 
 static uintptr_t g_kernel_as = 0;
 static spinlock_t vmm_lock = {0};
+static int g_nxe_enabled = 0;  /* IA32_EFER.NXE enabled in boot.S */
 
 static inline void invlpg(uintptr_t vaddr) {
     __asm__ volatile("invlpg (%0)" : : "r" (vaddr) : "memory");
 }
 
+/* Check if IA32_EFER.NXE is enabled (MSR 0xC0000080, bit 11) */
+static int check_nxe_enabled(void) {
+    uint32_t eax, edx;
+    __asm__ volatile("rdmsr" : "=a"(eax), "=d"(edx) : "c"(0xC0000080));
+    return (eax & 0x800) != 0;  /* Bit 11 = NXE */
+}
+
 /* --- PAE address decomposition --- */
 
 static inline uint32_t pae_pdpt_index(uint64_t va) {
@@ -106,7 +114,8 @@ static uint64_t vmm_flags_to_x86(uint32_t flags) {
     if (flags & VMM_FLAG_PWT)     x86_flags |= X86_PTE_PWT;
     if (flags & VMM_FLAG_PCD)     x86_flags |= X86_PTE_PCD;
     if (flags & VMM_FLAG_COW)     x86_flags |= X86_PTE_COW;
-    if (flags & VMM_FLAG_NX)      x86_flags |= X86_PTE_NX;
+    /* Only set NX bit if IA32_EFER.NXE is enabled */
+    if ((flags & VMM_FLAG_NX) && g_nxe_enabled) x86_flags |= X86_PTE_NX;
     return x86_flags;
 }
 
@@ -577,6 +586,14 @@ void vmm_init(void) {
 
     g_kernel_as = hal_cpu_get_address_space();
 
+    /* Check if IA32_EFER.NXE was enabled in boot.S */
+    g_nxe_enabled = check_nxe_enabled();
+    if (g_nxe_enabled) {
+        kprintf("[VMM] NX (No-Execute) enabled.\n");
+    } else {
+        kprintf("[VMM] NX not available or not enabled.\n");
+    }
+
     /* Test mapping */
     vmm_map_page(0xB8000, 0xC00B8000, VMM_FLAG_PRESENT | VMM_FLAG_RW);
     kprintf("[VMM] Mapped VGA to 0xC00B8000.\n");
diff --git a/tests/smoke_test.exp b/tests/smoke_test.exp
index b0a71ce2..220cb8a3 100755
--- a/tests/smoke_test.exp
+++ b/tests/smoke_test.exp
@@ -35,7 +35,8 @@ set qemu_pid [exec qemu-system-i386 \
     -drive file=$disk,if=ide,format=raw \
     -nic user,model=e1000 \
     -serial file:$serial_log -monitor none \
-    -no-reboot -no-shutdown &]
+    -no-reboot -no-shutdown \
+    -cpu qemu32,+nx &]
 
 # Wait for QEMU to start writing
 after 1000
-- 
2.43.0