arm64: Implement efi_rt_arch_call

This is a function that calls into the EFI Runtime Services, but can
handle a fault. To support this add a handler in assembly that can
restore the kernel state on an exception and return a failure to the
caller.

Sponsored by:	Arm Ltd
Differential Revision:	https://reviews.freebsd.org/D46817

sys/arm64/arm64/efirt_machdep.c

@@ -50,6 +50,7 @@
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
+#include <vm/vm_extern.h>
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
@@ -239,6 +240,7 @@ efi_arch_enter(void)
 {
 
 	CRITICAL_ASSERT(curthread);
+	curthread->td_md.md_efirt_dis_pf = vm_fault_disable_pagefaults();
 
 	/*
 	 * Temporarily switch to EFI's page table.  However, we leave curpmap
@@ -269,11 +271,6 @@ efi_arch_leave(void)
 	set_ttbr0(pmap_to_ttbr0(PCPU_GET(curpmap)));
 	if (PCPU_GET(bcast_tlbi_workaround) != 0)
 		invalidate_local_icache();
+	vm_fault_enable_pagefaults(curthread->td_md.md_efirt_dis_pf);
 }
 
-int
-efi_rt_arch_call(struct efirt_callinfo *ec)
-{
-
-	panic("not implemented");
-}

sys/arm64/arm64/efirt_support.S

@@ -0,0 +1,101 @@
+/*-
+ * Copyright (c) 2024 Arm Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/elf_common.h>
+#include <sys/errno.h>
+
+#include <machine/asm.h>
+
+#include "assym.inc"
+
+/*
+ * int efi_rt_arch_call(struct efirt_callinfo *);
+ */
+ENTRY(efi_rt_arch_call)
+	sub	sp, sp, #(14 * 8)
+	stp	x19, x20, [sp, #(2  * 8)]
+	stp	x21, x22, [sp, #(4  * 8)]
+	stp	x23, x24, [sp, #(6  * 8)]
+	stp	x25, x26, [sp, #(8  * 8)]
+	stp	x27, x28, [sp, #(10 * 8)]
+	stp	x29, x30, [sp, #(12 * 8)]
+	add	x29, sp, #(12 * 8)
+
+	/* Save the stack pointer so we can find it later */
+	ldr	x23, [x18, #PC_CURTHREAD]
+	mov	x24, sp
+	str	x24, [x23, #TD_MD_EFIRT_TMP]
+
+	mov	x22, x0
+
+	/* Load the function to branch to */
+	ldr	x9, [x22, #(EC_FPTR)]
+
+	/* Load the arguments */
+	ldr	x4, [x22, #(EC_ARG1 + (4 * 8))]
+	ldr	x3, [x22, #(EC_ARG1 + (3 * 8))]
+	ldr	x2, [x22, #(EC_ARG1 + (2 * 8))]
+	ldr	x1, [x22, #(EC_ARG1 + (1 * 8))]
+	ldr	x0, [x22, #(EC_ARG1 + (0 * 8))]
+
+	/* Set the fault handler */
+	adr	x10, efi_rt_fault
+	SET_FAULT_HANDLER(x10, x11)
+
+	blr	x9
+
+	/* Clear the fault handler */
+	SET_FAULT_HANDLER(xzr, x11)
+
+	/* Store the result */
+	str	x0, [x22, #(EC_EFI_STATUS)]
+	mov	x0, #0
+
+.Lefi_rt_arch_call_exit:
+	ldp	x19, x20, [sp, #(2  * 8)]
+	ldp	x21, x22, [sp, #(4  * 8)]
+	ldp	x23, x24, [sp, #(6  * 8)]
+	ldp	x25, x26, [sp, #(8  * 8)]
+	ldp	x27, x28, [sp, #(10 * 8)]
+	ldp	x29, x30, [sp, #(12 * 8)]
+	add	sp, sp, #(14 * 8)
+
+	ret
+END(efi_rt_arch_call)
+
+ENTRY(efi_rt_fault)
+	/* Clear pcb_onfault */
+	SET_FAULT_HANDLER(xzr, x11)
+	/* Load curthread */
+	ldr	x1, [x18, #PC_CURTHREAD]
+	/* Restore the stack pointer */
+	ldr	x2, [x1, #TD_MD_EFIRT_TMP]
+	mov	sp, x2
+	/* Normal exit returning an error */
+	ldr	x0, =EFAULT
+	b	.Lefi_rt_arch_call_exit
+END(efi_rt_fault)
+
+GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL)

sys/arm64/arm64/genassym.c

@@ -32,6 +32,7 @@
 #include <sys/pcpu.h>
 #include <sys/proc.h>
 
+#include <machine/efi.h>
 #include <machine/frame.h>
 #include <machine/intr.h>
 #include <machine/machdep.h>
@@ -45,6 +46,10 @@ ASSYM(BP_KERN_STACK, offsetof(struct arm64_bootparams, kern_stack));
 ASSYM(BP_KERN_TTBR0, offsetof(struct arm64_bootparams, kern_ttbr0));
 ASSYM(BP_BOOT_EL, offsetof(struct arm64_bootparams, boot_el));
 
+ASSYM(EC_EFI_STATUS, offsetof(struct efirt_callinfo, ec_efi_status));
+ASSYM(EC_FPTR, offsetof(struct efirt_callinfo, ec_fptr));
+ASSYM(EC_ARG1, offsetof(struct efirt_callinfo, ec_arg1));
+
 ASSYM(PCPU_SIZE, sizeof(struct pcpu));
 ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
 ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
@@ -71,6 +76,7 @@ ASSYM(TD_AST, offsetof(struct thread, td_ast));
 ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
 ASSYM(TD_LOCK, offsetof(struct thread, td_lock));
 ASSYM(TD_MD_CANARY, offsetof(struct thread, td_md.md_canary));
+ASSYM(TD_MD_EFIRT_TMP, offsetof(struct thread, td_md.md_efirt_tmp));
 
 ASSYM(TF_SIZE, sizeof(struct trapframe));
 ASSYM(TF_SP, offsetof(struct trapframe, tf_sp));

sys/arm64/include/proc.h

@@ -65,7 +65,11 @@ struct mdthread {
 		struct ptrauth_key apia;
 	} md_ptrauth_kern;
 
-	uint64_t md_reserved[4];
+	uint64_t md_efirt_tmp;
+	int md_efirt_dis_pf;
+
+	int md_reserved0;
+	uint64_t md_reserved[2];
 };
 
 struct mdproc {

sys/conf/files.arm64

@@ -40,6 +40,7 @@ arm64/arm64/debug_monitor.c			standard
 arm64/arm64/disassem.c				optional ddb
 arm64/arm64/dump_machdep.c			standard
 arm64/arm64/efirt_machdep.c			optional efirt
+arm64/arm64/efirt_support.S			optional efirt
 arm64/arm64/elf32_machdep.c			optional compat_freebsd32
 arm64/arm64/elf_machdep.c			standard
 arm64/arm64/exception.S				standard