diff options
Diffstat (limited to '0005-haoc-kernel.patch')
| -rw-r--r-- | 0005-haoc-kernel.patch | 13472 |
1 files changed, 13472 insertions, 0 deletions
diff --git a/0005-haoc-kernel.patch b/0005-haoc-kernel.patch new file mode 100644 index 0000000..2d9484f --- /dev/null +++ b/0005-haoc-kernel.patch @@ -0,0 +1,13472 @@ +From 099672494b5fbd626031f313bbdbc42d1eb3f02d Mon Sep 17 00:00:00 2001 +From: liuzh <liuzhh@zgclab.edu.cn> +Date: Wed, 22 May 2024 16:36:55 +0800 +Subject: [PATCH] Squashed commit of the following: + +commit 909ad06b3bf629d9af4e143347c1d1ef8a3a5808 +Author: liuzh <liuzhh@zgclab.edu.cn> +Date: Wed May 22 16:23:13 2024 +0800 + + fix mte_sync_tags() parameters after rebase to 6.6.0-27.0.0. + +commit 1cbf51371b0539a45f816419b2da82cf36162b4a +Author: ljl <ljl2013@163.com> +Date: Mon Mar 25 08:01:32 2024 +0000 + + IEE SI: Removed redundant codes. + +commit 0178bfc79ad1769a36f4165348a671d2182cff55 +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Mon Mar 25 11:01:11 2024 +0800 + + Fix bugs on qemu when opening CONFIG_CREDP. + +commit 8e714f6e8f2ace5a6fc900b4bce6b03c83c41870 +Author: ljl <ljl2013@163.com> +Date: Thu Mar 21 04:44:26 2024 +0000 + + IEE SI: Remove PAN operations as BTLB BUG is already fixed. + +commit 7b5fc74cb99e377d3bc59da81612cd6f3dd8a4d8 +Author: ljl <lvjinglin2013@163.com> +Date: Wed Mar 20 18:31:47 2024 +0800 + + IEE SI: Migration of iee rwx gate. + +commit aad2c7e89c9c4ad8ff0fb3ee53cd1b974144a283 +Author: liuzh <liuzhh@zgclab.edu.cn> +Date: Mon Mar 18 15:32:43 2024 +0800 + + modify slub.c set_track_prepare() + +commit 7452bac06ec09bf8321dfdbfb8b6a429d2cd8637 +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Thu Mar 21 11:26:19 2024 +0800 + + Set pgd of lm Privileged. + +commit 33934cfc3eed798a3a687bf86c6bd92697e68ba9 +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Tue Mar 19 17:14:32 2024 +0800 + + Delete some redundant code and put trans_pgd into IEE. + +commit 2bfe9008a72f8b8ac237bc7a5f99f9d40e84c247 +Author: zhangshiyang17@mails.ucas.ac.cn <zhangshiyang17@mails.ucas.ac.cn> +Date: Mon Mar 18 11:47:50 2024 +0000 + + Fix bugs on physical when opening CONFIG_IEE and CONFIG_PTP. + +commit dafa2df600757511ce3e8f178e05e28adabdf39b +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Mon Mar 18 10:40:42 2024 +0800 + + Fix bugs on qemu when opening CONFIG_IEE and CONFIG_PTP. + +commit 9231a9f6b34c62090b5f202c9c64a52bfdac7a73 +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Thu Mar 14 16:34:53 2024 +0800 + + Fix compiling bugs of CONFIG_PTP. + +commit 6469df3bcce32896c2cb297d3cd7ead82c33f35d +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Thu Mar 14 11:10:00 2024 +0800 + + Fix bugs on qemu when opening CONFIG_IEE and CONFIG_INTERRUPTABLE. + +commit 5f1773dada622a3514c9ed6aa72dd50e918f2664 +Author: zhangsy <zhangshiyang17@mails.ucas.ac.cn> +Date: Wed Mar 13 17:31:39 2024 +0800 + + Fix bugs on qemu when opening CONFIG_IEE. + +commit 73f433a093fa84cffa5e11e86bed6f17c9b30a39 +Author: liuzh <liuzhh@zgclab.edu.cn> +Date: Tue Mar 12 15:32:29 2024 +0800 + + fix the map of IEE_SI_TEXT. + +commit 9b92deb4b2338093d9b04f4b81f162855b31c983 +Author: liuzh <liuzhh@zgclab.edu.cn> +Date: Sun Mar 10 16:11:13 2024 +0800 + + modified to be able to compile. + can start the kernel with qemu and successfully reach `start_kernel()`. + +commit e892ec4790d72e9433b48b0221e7e6dc4c361dd9 +Author: liuzh <liuzhh@zgclab.edu.cn> +Date: Thu Mar 7 14:27:45 2024 +0800 + + fix some conflicts + +commit fdec7e39345e81e867e01258487f88801b790b02 +Author: liuzh <liuzhh@zgclab.edu.cn> +Date: Wed Mar 6 12:31:11 2024 +0800 + + migrate openeuler-commit code. (need some fix before compiling) +--- + Makefile | 3 +- + arch/arm64/Kconfig | 18 + + arch/arm64/include/asm/assembler.h | 67 + + arch/arm64/include/asm/daifflags.h | 16 + + arch/arm64/include/asm/efi.h | 4 + + arch/arm64/include/asm/fixmap.h | 3 + + arch/arm64/include/asm/hw_breakpoint.h | 12 + + arch/arm64/include/asm/iee-access.h | 36 + + arch/arm64/include/asm/iee-cred.h | 150 ++ + arch/arm64/include/asm/iee-def.h | 74 + + arch/arm64/include/asm/iee-si.h | 64 + + arch/arm64/include/asm/iee-slab.h | 23 + + arch/arm64/include/asm/iee-token.h | 40 + + arch/arm64/include/asm/iee.h | 10 + + arch/arm64/include/asm/kernel-pgtable.h | 21 + + arch/arm64/include/asm/koi.h | 335 +++++ + arch/arm64/include/asm/memory.h | 24 + + arch/arm64/include/asm/mmu_context.h | 20 + + arch/arm64/include/asm/pgalloc.h | 4 + + arch/arm64/include/asm/pgtable-hwdef.h | 11 + + arch/arm64/include/asm/pgtable.h | 304 +++- + arch/arm64/include/asm/pointer_auth.h | 5 + + arch/arm64/include/asm/sysreg.h | 58 + + arch/arm64/include/asm/tlb.h | 9 + + arch/arm64/include/asm/tlbflush.h | 58 +- + arch/arm64/kernel/Makefile | 2 + + arch/arm64/kernel/armv8_deprecated.c | 16 + + arch/arm64/kernel/asm-offsets.c | 11 + + arch/arm64/kernel/cpu_errata.c | 12 + + arch/arm64/kernel/cpufeature.c | 79 + + arch/arm64/kernel/debug-monitors.c | 4 + + arch/arm64/kernel/entry-common.c | 4 + + arch/arm64/kernel/entry.S | 611 ++++++++ + arch/arm64/kernel/fpsimd.c | 4 + + arch/arm64/kernel/head.S | 56 + + arch/arm64/kernel/hibernate.c | 14 + + arch/arm64/kernel/hw_breakpoint.c | 99 ++ + arch/arm64/kernel/iee/Makefile | 1 + + arch/arm64/kernel/iee/iee-func.c | 187 +++ + arch/arm64/kernel/iee/iee-gate.S | 174 +++ + arch/arm64/kernel/iee/iee.c | 1360 +++++++++++++++++ + arch/arm64/kernel/koi/Makefile | 1 + + arch/arm64/kernel/koi/koi.c | 1327 +++++++++++++++++ + arch/arm64/kernel/mte.c | 5 + + arch/arm64/kernel/process.c | 19 +- + arch/arm64/kernel/proton-pack.c | 8 + + arch/arm64/kernel/setup.c | 33 + + arch/arm64/kernel/traps.c | 26 + + arch/arm64/kernel/vmlinux.lds.S | 61 + + arch/arm64/mm/context.c | 91 +- + arch/arm64/mm/fault.c | 9 + + arch/arm64/mm/fixmap.c | 74 +- + arch/arm64/mm/init.c | 34 + + arch/arm64/mm/mmu.c | 1780 +++++++++++++++++++---- + arch/arm64/mm/pgd.c | 39 + + arch/arm64/mm/proc.S | 28 + + arch/arm64/mm/trans_pgd.c | 46 + + drivers/firmware/efi/arm-runtime.c | 4 + + drivers/firmware/efi/memmap.c | 20 + + drivers/tty/serial/earlycon.c | 4 + + drivers/usb/early/ehci-dbgp.c | 4 + + fs/coredump.c | 8 + + fs/exec.c | 20 + + fs/nfs/flexfilelayout/flexfilelayout.c | 9 + + fs/nfs/nfs4idmap.c | 9 + + fs/nfsd/auth.c | 38 + + fs/nfsd/nfs4callback.c | 12 +- + fs/nfsd/nfs4recover.c | 9 + + fs/nfsd/nfsfh.c | 9 + + fs/open.c | 26 + + fs/overlayfs/dir.c | 9 + + fs/overlayfs/super.c | 12 + + fs/smb/client/cifs_spnego.c | 9 + + fs/smb/client/cifsacl.c | 9 + + include/asm-generic/early_ioremap.h | 3 + + include/asm-generic/fixmap.h | 18 + + include/asm-generic/pgalloc.h | 54 + + include/asm-generic/vmlinux.lds.h | 24 +- + include/linux/cred.h | 45 +- + include/linux/efi.h | 9 + + include/linux/iee-func.h | 27 + + include/linux/module.h | 1 + + include/linux/sched.h | 19 + + init/main.c | 28 +- + kernel/cred.c | 182 +++ + kernel/exit.c | 8 + + kernel/fork.c | 316 ++-- + kernel/groups.c | 7 + + kernel/kthread.c | 13 + + kernel/smpboot.c | 9 + + kernel/sys.c | 107 ++ + kernel/umh.c | 10 + + kernel/user_namespace.c | 18 + + mm/Kconfig | 12 + + mm/damon/ops-common.c | 1 + + mm/debug_vm_pgtable.c | 24 + + mm/early_ioremap.c | 57 + + mm/huge_memory.c | 30 +- + mm/init-mm.c | 17 + + mm/memory.c | 14 + + mm/slub.c | 198 ++- + mm/sparse-vmemmap.c | 21 + + mm/vmalloc.c | 2 +- + net/dns_resolver/dns_key.c | 9 + + security/commoncap.c | 169 +++ + security/keys/keyctl.c | 23 + + security/keys/process_keys.c | 53 + + security/security.c | 15 + + 109 files changed, 8945 insertions(+), 397 deletions(-) + create mode 100644 arch/arm64/include/asm/iee-access.h + create mode 100644 arch/arm64/include/asm/iee-cred.h + create mode 100644 arch/arm64/include/asm/iee-def.h + create mode 100644 arch/arm64/include/asm/iee-si.h + create mode 100644 arch/arm64/include/asm/iee-slab.h + create mode 100644 arch/arm64/include/asm/iee-token.h + create mode 100644 arch/arm64/include/asm/iee.h + create mode 100644 arch/arm64/include/asm/koi.h + create mode 100644 arch/arm64/kernel/iee/Makefile + create mode 100644 arch/arm64/kernel/iee/iee-func.c + create mode 100644 arch/arm64/kernel/iee/iee-gate.S + create mode 100644 arch/arm64/kernel/iee/iee.c + create mode 100644 arch/arm64/kernel/koi/Makefile + create mode 100644 arch/arm64/kernel/koi/koi.c + create mode 100644 include/linux/iee-func.h + +diff --git a/Makefile b/Makefile +index 8e6d9b894b1e..20c367b5957d 100644 +--- a/Makefile ++++ b/Makefile +@@ -554,7 +554,7 @@ LINUXINCLUDE := \ + -I$(objtree)/include \ + $(USERINCLUDE) + +-KBUILD_AFLAGS := -D__ASSEMBLY__ -fno-PIE ++KBUILD_AFLAGS := -D__ASSEMBLY__ -fno-PIE -march=armv8.1-a + + KBUILD_CFLAGS := + KBUILD_CFLAGS += -std=gnu11 +@@ -563,6 +563,7 @@ KBUILD_CFLAGS += -funsigned-char + KBUILD_CFLAGS += -fno-common + KBUILD_CFLAGS += -fno-PIE + KBUILD_CFLAGS += -fno-strict-aliasing ++KBUILD_CFLAGS += -march=armv8.1-a + + KBUILD_CPPFLAGS := -D__KERNEL__ + KBUILD_RUSTFLAGS := $(rust_common_flags) \ +diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig +index 2a875546bdc7..7448afc90c0a 100644 +--- a/arch/arm64/Kconfig ++++ b/arch/arm64/Kconfig +@@ -1730,6 +1730,24 @@ config UNMAP_KERNEL_AT_EL0 + + If unsure, say Y. + ++# Config for iee ++config IEE ++ depends on ARM64 ++ depends on ARM64_PAN ++ depends on ARM64_VA_BITS_48 ++ depends on ARM64_4K_PAGES ++ def_bool y ++ ++# Config for support of interruption of iee ++config IEE_INTERRUPTABLE ++ depends on IEE ++ def_bool n ++ ++# Config for credentials isolation ++config CREDP ++ depends on IEE ++ def_bool y ++ + config MITIGATE_SPECTRE_BRANCH_HISTORY + bool "Mitigate Spectre style attacks against branch history" if EXPERT + default y +diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h +index 38b23786aeb4..6af10d509c2e 100644 +--- a/arch/arm64/include/asm/assembler.h ++++ b/arch/arm64/include/asm/assembler.h +@@ -26,6 +26,41 @@ + #include <asm/ptrace.h> + #include <asm/thread_info.h> + ++#ifdef CONFIG_IEE ++ .macro iee_si_restore_daif, flags:req ++ msr daifclr, #0xf ++ tbnz \flags, #6, 114221f ++ tbnz \flags, #7, 114210f ++ tbnz \flags, #8, 114100f ++ msr daifset, #0b000 ++ b 114514f ++114221: ++ tbnz \flags, #7, 114211f ++ tbnz \flags, #8, 114101f ++ msr daifset, #0b001 ++ b 114514f ++114211: ++ tbnz \flags, #8, 114111f ++ msr daifset, #0b011 ++ b 114514f ++114210: ++ tbnz \flags, #8, 114110f ++ msr daifset, #0b010 ++ b 114514f ++114100: ++ msr daifset, #0b100 ++ b 114514f ++114101: ++ msr daifset, #0b101 ++ b 114514f ++114110: ++ msr daifset, #0b110 ++ b 114514f ++114111: ++ msr daifset, #0b111 ++114514: ++ .endm ++#endif + /* + * Provide a wxN alias for each wN register so what we can paste a xN + * reference after a 'w' to obtain the 32-bit version. +@@ -52,7 +87,11 @@ alternative_else_nop_endif + + .macro disable_daif + disable_allint ++// #ifdef CONFIG_IEE ++// msr daifset, #0x7 ++// #else + msr daifset, #0xf ++// #endif + .endm + + .macro enable_daif +@@ -69,7 +108,11 @@ alternative_else_nop_endif + .endm + + .macro restore_irq, flags ++// #ifdef CONFIG_IEE ++// iee_si_restore_daif \flags ++// #else + msr daif, \flags ++// #endif + .endm + + .macro enable_dbg +@@ -77,20 +120,44 @@ alternative_else_nop_endif + .endm + + .macro disable_step_tsk, flgs, tmp ++// #ifdef CONFIG_IEE ++// 1145: ++// tbz \flgs, #TIF_SINGLESTEP, 9990f ++// mrs \tmp, mdscr_el1 ++// bic \tmp, \tmp, #DBG_MDSCR_SS ++// orr \tmp, \tmp, #DBG_MDSCR_MDE ++// msr mdscr_el1, \tmp ++// isb // Synchronise with enable_dbg ++// mrs \tmp, mdscr_el1 ++// tbz \tmp, #15, 1145b ++// #else + tbz \flgs, #TIF_SINGLESTEP, 9990f + mrs \tmp, mdscr_el1 + bic \tmp, \tmp, #DBG_MDSCR_SS + msr mdscr_el1, \tmp + isb // Synchronise with enable_dbg ++// #endif + 9990: + .endm + + /* call with daif masked */ + .macro enable_step_tsk, flgs, tmp ++// #ifdef CONFIG_IEE ++// 1146: ++// tbz \flgs, #TIF_SINGLESTEP, 9990f ++// mrs \tmp, mdscr_el1 ++// orr \tmp, \tmp, #DBG_MDSCR_SS ++// orr \tmp, \tmp, #DBG_MDSCR_MDE ++// msr mdscr_el1, \tmp ++// isb // Synchronise with enable_dbg ++// mrs \tmp, mdscr_el1 ++// tbz \tmp, #15, 1146b ++// #else + tbz \flgs, #TIF_SINGLESTEP, 9990f + mrs \tmp, mdscr_el1 + orr \tmp, \tmp, #DBG_MDSCR_SS + msr mdscr_el1, \tmp ++// #endif + 9990: + .endm + +diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h +index 2417cc6b1631..cb5b4c2e03b8 100644 +--- a/arch/arm64/include/asm/daifflags.h ++++ b/arch/arm64/include/asm/daifflags.h +@@ -26,11 +26,19 @@ static inline void local_daif_mask(void) + (read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF | + GIC_PRIO_PSR_I_SET))); + ++// #ifdef CONFIG_IEE ++// asm volatile( ++// "msr daifset, #0x7 // local_daif_mask\n" ++// : ++// : ++// : "memory"); ++// #else + asm volatile( + "msr daifset, #0xf // local_daif_mask\n" + : + : + : "memory"); ++// #endif + + /* Don't really care for a dsb here, we don't intend to enable IRQs */ + if (system_uses_irq_prio_masking()) +@@ -118,7 +126,11 @@ static inline void local_daif_restore(unsigned long flags) + gic_write_pmr(pmr); + } + ++// #ifdef CONFIG_IEE ++// iee_si_write_daif(flags); ++// #else + write_sysreg(flags, daif); ++// #endif + + /* If we can take asynchronous errors we can take NMIs */ + if (system_uses_nmi()) { +@@ -151,7 +163,11 @@ static inline void local_daif_inherit(struct pt_regs *regs) + * system_has_prio_mask_debugging() won't restore the I bit if it can + * use the pmr instead. + */ ++// #ifdef CONFIG_IEE ++// iee_si_write_daif(flags); ++// #else + write_sysreg(flags, daif); ++// #endif + + /* The ALLINT field is at the same position in pstate and ALLINT */ + if (system_uses_nmi()) { +diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h +index bcd5622aa096..76c4bd6c2b20 100644 +--- a/arch/arm64/include/asm/efi.h ++++ b/arch/arm64/include/asm/efi.h +@@ -58,7 +58,11 @@ void arch_efi_call_virt_teardown(void); + #define arch_efi_save_flags(state_flags) \ + ((void)((state_flags) = read_sysreg(daif))) + ++// #ifdef CONFIG_IEE ++// #define arch_efi_restore_flags(state_flags) iee_si_write_daif(state_flags) ++// #else + #define arch_efi_restore_flags(state_flags) write_sysreg(state_flags, daif) ++// #endif + + + /* arch specific definitions used by the stub code */ +diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h +index 58c294a96676..095a0731dce3 100644 +--- a/arch/arm64/include/asm/fixmap.h ++++ b/arch/arm64/include/asm/fixmap.h +@@ -108,6 +108,9 @@ void __init fixmap_copy(pgd_t *pgdir); + #define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) + + extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); ++#ifdef CONFIG_PTP ++extern void __iee_set_fixmap_pre_init(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); ++#endif + + #include <asm-generic/fixmap.h> + +diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h +index 84055329cd8b..f72d89bb9a32 100644 +--- a/arch/arm64/include/asm/hw_breakpoint.h ++++ b/arch/arm64/include/asm/hw_breakpoint.h +@@ -104,6 +104,18 @@ static inline void decode_ctrl_reg(u32 reg, + write_sysreg(VAL, dbg##REG##N##_el1);\ + } while (0) + ++#ifdef CONFIG_IEE ++#define IEE_SI_AARCH64_DBG_READ(N, REG, VAL) do{\ ++ VAL = this_cpu_read(iee_si_user_##REG##N);\ ++} while (0) ++ ++#define IEE_SI_AARCH64_DBG_WRITE(N, REG, VAL) do{\ ++ u64 __val = (u64)(VAL); \ ++ this_cpu_write(iee_si_user_##REG##N, __val);\ ++ iee_rwx_gate_entry(IEE_WRITE_AFSR0);\ ++} while (0) ++#endif ++ + struct task_struct; + struct notifier_block; + struct perf_event_attr; +diff --git a/arch/arm64/include/asm/iee-access.h b/arch/arm64/include/asm/iee-access.h +new file mode 100644 +index 000000000000..79604c21a510 +--- /dev/null ++++ b/arch/arm64/include/asm/iee-access.h +@@ -0,0 +1,36 @@ ++#ifndef _LINUX_IEE_ACCESS_H ++#define _LINUX_IEE_ACCESS_H ++ ++#include <asm/iee-def.h> ++#include <asm/iee-slab.h> ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++#ifdef CONFIG_IEE ++void iee_write_in_byte(void *ptr, u64 data, int length) ++{ ++ iee_rw_gate(IEE_WRITE_IN_BYTE, ptr, data, length); ++} ++ ++void iee_memset(void *ptr, int data, size_t n) ++{ ++ iee_rw_gate(IEE_MEMSET, ptr, data, n); ++} ++ ++void iee_set_track(struct track *ptr, struct track *data) ++{ ++ iee_rw_gate(IEE_OP_SET_TRACK, ptr, data); ++} ++ ++void iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr) ++{ ++ iee_rw_gate(IEE_OP_SET_FREEPTR, pptr, ptr); ++} ++ ++void iee_write_entry_task(struct task_struct *tsk) ++{ ++ iee_rw_gate(IEE_WRITE_ENTRY_TASK, tsk); ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/arch/arm64/include/asm/iee-cred.h b/arch/arm64/include/asm/iee-cred.h +new file mode 100644 +index 000000000000..b8c3bb53f98a +--- /dev/null ++++ b/arch/arm64/include/asm/iee-cred.h +@@ -0,0 +1,150 @@ ++#ifndef _LINUX_IEE_CRED_H ++#define _LINUX_IEE_CRED_H ++ ++#include <linux/cred.h> ++#include <asm/iee-def.h> ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++#ifdef CONFIG_CREDP ++static void __maybe_unused iee_copy_cred(const struct cred *old, struct cred *new) ++{ ++ iee_rw_gate(IEE_OP_COPY_CRED,old,new); ++} ++ ++static void __maybe_unused iee_set_cred_uid(struct cred *cred, kuid_t uid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_UID,cred,uid); ++} ++ ++static void __maybe_unused iee_set_cred_gid(struct cred *cred, kgid_t gid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_GID,cred,gid); ++} ++ ++static void __maybe_unused iee_set_cred_suid(struct cred *cred, kuid_t suid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_SUID,cred,suid); ++} ++ ++static void __maybe_unused iee_set_cred_sgid(struct cred *cred, kgid_t sgid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_SGID,cred,sgid); ++} ++ ++static void __maybe_unused iee_set_cred_euid(struct cred *cred, kuid_t euid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_EUID,cred,euid); ++} ++ ++static void __maybe_unused iee_set_cred_egid(struct cred *cred, kgid_t egid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_EGID,cred,egid); ++} ++ ++static void __maybe_unused iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_FSUID,cred,fsuid); ++} ++ ++static void __maybe_unused iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_FSGID,cred,fsgid); ++} ++ ++static void __maybe_unused iee_set_cred_user(struct cred *cred, struct user_struct *user) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_USER,cred,user); ++} ++ ++static void __maybe_unused iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_USER_NS,cred,user_ns); ++} ++ ++static void __maybe_unused iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_UCOUNTS,cred,ucounts); ++} ++ ++static void __maybe_unused iee_set_cred_group_info(struct cred *cred, struct group_info *group_info) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_GROUP_INFO,cred,group_info); ++} ++ ++static void __maybe_unused iee_set_cred_securebits(struct cred *cred, unsigned securebits) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_SECUREBITS,cred,securebits); ++} ++ ++static void __maybe_unused iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_INHER,cred,cap_inheritable); ++} ++ ++static void __maybe_unused iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_PERM,cred,cap_permitted); ++} ++ ++static void __maybe_unused iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_EFFECT,cred,cap_effective); ++} ++ ++static void __maybe_unused iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_BSET,cred,cap_bset); ++} ++ ++static void __maybe_unused iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_CAP_AMBIENT,cred,cap_ambient); ++} ++ ++#ifdef CONFIG_KEYS ++static void __maybe_unused iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_JIT_KEYRING,cred,jit_keyring); ++} ++ ++static void __maybe_unused iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_SESS_KEYRING,cred,session_keyring); ++} ++ ++static void __maybe_unused iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_PROC_KEYRING,cred,process_keyring); ++} ++ ++static void __maybe_unused iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_THREAD_KEYRING,cred,thread_keyring); ++} ++ ++static void __maybe_unused iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_REQ_KEYRING,cred,request_key_auth); ++} ++#endif ++ ++static void __maybe_unused iee_set_cred_atomic_set_usage(struct cred *cred, int i) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_ATSET_USAGE,cred,i); ++} ++ ++#ifdef CONFIG_SECURITY ++static void __maybe_unused iee_set_cred_security(struct cred *cred, void *security) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_SECURITY,cred,security); ++} ++#endif ++ ++static void __maybe_unused iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_RCU,cred,rcu); ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/arch/arm64/include/asm/iee-def.h b/arch/arm64/include/asm/iee-def.h +new file mode 100644 +index 000000000000..76e59259e4d1 +--- /dev/null ++++ b/arch/arm64/include/asm/iee-def.h +@@ -0,0 +1,74 @@ ++// Function Identifiers with Parameters Description ++ ++#define IEE_WRITE_IN_BYTE 0 // Parameters: void *ptr, __u64 data, int length ++#define IEE_OP_SET_PTE 1 // Parameters: pte_t *ptep, pte_t pte ++#define IEE_OP_SET_PMD 2 // Parameters: pmd_t *pmdp, pmd_t pmd ++#define IEE_OP_SET_PUD 3 // Parameters: pud_t *pudp, pud_t pud ++#define IEE_OP_SET_P4D 4 // Parameters: p4d_t *p4dp, p4d_t p4d ++#define IEE_OP_SET_BM_PTE 5 // Parameters: pte_t *ptep, pte_t pte ++#define IEE_OP_SET_SWAPPER_PGD 6 // Parameters: pgd_t *pgdp, pgd_t pgd ++#define IEE_OP_SET_TRAMP_PGD 7 // Parameters: pgd_t *pgdp, pgd_t pgd ++#define IEE_OP_SET_CMPXCHG 8 // Parameters: pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval ++#define IEE_OP_SET_XCHG 9 // Parameters: pte_t *ptep, pteval_t pteval ++#define IEE_OP_COPY_CRED 10 // Parameters: struct cred *old, struct cred *new ++#define IEE_OP_SET_CRED_UID 11 // Parameters: struct cred *cred, kuid_t uid ++#define IEE_OP_SET_CRED_GID 12 // Parameters: struct cred *cred, kgid_t gid ++#define IEE_OP_SET_CRED_SUID 13 // Parameters: struct cred *cred, kuid_t suid ++#define IEE_OP_SET_CRED_SGID 14 // Parameters: struct cred *cred, kgid_t sgid ++#define IEE_OP_SET_CRED_EUID 15 // Parameters: struct cred *cred, kuid_t euid ++#define IEE_OP_SET_CRED_EGID 16 // Parameters: struct cred *cred, kgid_t egid ++#define IEE_OP_SET_CRED_FSUID 17 // Parameters: struct cred *cred, kuid_t fsuid ++#define IEE_OP_SET_CRED_FSGID 18 // Parameters: struct cred *cred, kgid_t fsgid ++#define IEE_OP_SET_CRED_USER 19 // Parameters: struct cred *cred, struct user_struct *user ++#define IEE_OP_SET_CRED_USER_NS 20 // Parameters: struct cred *cred, struct user_namespace *user_ns ++#define IEE_OP_SET_CRED_GROUP_INFO 21 // Parameters: struct cred *cred, struct group_info *group_info ++#define IEE_OP_SET_CRED_SECUREBITS 22 // Parameters: struct cred *cred, unsigned securebits ++#define IEE_OP_SET_CRED_CAP_INHER 23 // Parameters: struct cred *cred, kernel_cap_t cap_inheritable ++#define IEE_OP_SET_CRED_CAP_PERM 24 // Parameters: struct cred *cred, kernel_cap_t cap_permitted ++#define IEE_OP_SET_CRED_CAP_EFFECT 25 // Parameters: struct cred *cred, kernel_cap_t cap_effective ++#define IEE_OP_SET_CRED_CAP_BSET 26 // Parameters: struct cred *cred, kernel_cap_t cap_bset ++#define IEE_OP_SET_CRED_CAP_AMBIENT 27 // Parameters: struct cred *cred, kernel_cap_t cap_ambient ++#define IEE_OP_SET_CRED_JIT_KEYRING 28 // Parameters: struct cred *cred, unsigned char jit_keyring ++#define IEE_OP_SET_CRED_SESS_KEYRING 29 // Parameters: struct cred *cred, struct key *session_keyring ++#define IEE_OP_SET_CRED_PROC_KEYRING 30 // Parameters: struct cred *cred, struct key *process_keyring ++#define IEE_OP_SET_CRED_THREAD_KEYRING 31 // Parameters: struct cred *cred, struct key *thread_keyring ++#define IEE_OP_SET_CRED_REQ_KEYRING 32 // Parameters: struct cred *cred, struct key *request_key_auth ++#define IEE_OP_SET_CRED_NON_RCU 33 // Parameters: struct cred *cred, int non_rcu ++#define IEE_OP_SET_CRED_ATSET_USAGE 34 // Parameters: struct cred *cred, int i ++#define IEE_OP_SET_CRED_ATOP_USAGE 35 // Parameters: struct cred *cred, int flag ++#define IEE_OP_SET_CRED_SECURITY 36 // Parameters: struct cred *cred, void *security ++#define IEE_OP_SET_CRED_RCU 37 // Parameters: struct cred *cred, struct rcu_head *rcu ++#define IEE_MEMSET 38 // Parameters: void *ptr, int data, size_t n ++#define IEE_OP_SET_TRACK 39 // Parameters: struct track *ptr, struct track *data ++#define IEE_OP_SET_FREEPTR 40 // Parameters: void **pptr, void *ptr ++#define IEE_OP_SET_PTE_U 41 // Parameters: pte_t *ptep, pte_t pte ++#define IEE_OP_SET_PTE_P 42 // Parameters: pte_t *ptep, pte_t pte ++#define IEE_SET_TOKEN_MM 43 // Parameters: struct task_token *token, struct mm_struct *mm ++#define IEE_SET_TOKEN_PGD 44 // Parameters: struct task_token *token, pgd_t *pgd ++#define IEE_INIT_TOKEN 45 // Parameters: struct task_struct *tsk, void *kernel_stack, void *iee_stack ++#define IEE_FREE_TOKEN 46 // Parameters: struct task_struct *tsk ++#define IEE_READ_TOKEN_STACK 47 // Parameters: struct task_struct *tsk ++#define IEE_WRITE_ENTRY_TASK 48 // Parameters: struct task_struct *tsk ++#define IEE_OP_SET_CRED_UCOUNTS 49 // Parameters: struct cred *cred, struct ucounts *ucounts ++#ifdef CONFIG_KOI ++#define IEE_READ_KOI_STACK 50 // Parameters: struct task_struct *tsk ++#define IEE_WRITE_KOI_STACK 51 // Parameters: struct task_struct *tsk, unsigned long koi_stack ++#define IEE_READ_TOKEN_TTBR1 52 // Parameters: struct task_struct *tsk ++#define IEE_WRITE_TOKEN_TTBR1 53 // Parameters: struct task_struct *tsk, unsigned long current_ttbr1 ++#define IEE_READ_KOI_KERNEL_STACK 54 // Parameters: struct task_struct *tsk ++#define IEE_WRITE_KOI_KERNEL_STACK 55 // Parameters: struct task_struct *tsk, unsigned long kernel_stack ++#define IEE_READ_KOI_STACK_BASE 56 // Parameters: struct task_struct *tsk ++#define IEE_WRITE_KOI_STACK_BASE 57 // Parameters: struct task_struct *tsk, unsigned long koi_stack_base ++#endif ++ ++/* Add new IEE ops here */ ++ ++#define AT_ADD 1 ++#define AT_INC_NOT_ZERO 2 ++#define AT_SUB_AND_TEST 3 ++/* Atomic ops for atomic_t */ ++ ++#ifdef CONFIG_KOI ++#define IEE_SWITCH_TO_KERNEL 7 ++#define IEE_SWITCH_TO_KOI 8 ++#endif +\ No newline at end of file +diff --git a/arch/arm64/include/asm/iee-si.h b/arch/arm64/include/asm/iee-si.h +new file mode 100644 +index 000000000000..e67d81db66a5 +--- /dev/null ++++ b/arch/arm64/include/asm/iee-si.h +@@ -0,0 +1,64 @@ ++#ifndef _LINUX_IEE_SI_H ++#define _LINUX_IEE_SI_H ++ ++#include <asm/sysreg.h> ++#define __iee_si_code __section(".iee.si_text") ++#define __iee_si_data __section(".iee.si_data") ++ ++/* Used for copying globals that iee rwx gate needs. */ ++extern unsigned long iee_base_idmap_pg_dir; ++extern unsigned long iee_base_reserved_pg_dir; ++extern unsigned long iee_base__bp_harden_el1_vectors; ++extern bool iee_init_done; ++extern unsigned long iee_si_tcr; ++ ++/* The following are __init functions used for iee si initialization. */ ++extern void iee_si_prepare_data(void); ++ ++extern unsigned long __iee_si_start[]; ++// Handler function for sensitive inst ++u64 iee_si_handler(int flag, ...); ++/* ++ * TODO: scan a page to check whether it contains sensitive instructions ++ * return 1 when finding sensitive inst, 0 on safe page. ++ */ ++extern int iee_si_scan_page(unsigned long addr); ++ ++ ++#define DBG_MDSCR_SS (1 << 0) ++#define DBG_MDSCR_MDE (1 << 15) ++ ++#define IEE_SI_TEST 0 ++#define IEE_WRITE_SCTLR 1 ++#define IEE_WRITE_TTBR0 2 ++#define IEE_WRITE_VBAR 3 ++#define IEE_WRITE_TCR 4 ++#define IEE_WRITE_MDSCR 5 ++#define IEE_CONTEXT_SWITCH 6 ++// #define IEE_WRITE_AFSR0 10 ++/* Provide ttbr1 switch gate for KOI */ ++#ifdef CONFIG_KOI ++#define IEE_SWITCH_TO_KERNEL 7 ++#define IEE_SWITCH_TO_KOI 8 ++#endif ++/* MASK modify-permitted bits on IEE protected sys registers */ ++#define IEE_SCTLR_MASK (SCTLR_EL1_CP15BEN | SCTLR_EL1_SED | SCTLR_EL1_UCT | SCTLR_EL1_UCI |\ ++ SCTLR_EL1_BT0 | SCTLR_EL1_BT1 | SCTLR_EL1_TCF0_MASK | SCTLR_ELx_DSSBS |\ ++ SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB|\ ++ SCTLR_EL1_SPINTMASK | SCTLR_EL1_NMI | SCTLR_EL1_TIDCP | SCTLR_EL1_MSCEn|\ ++ SCTLR_ELx_ENTP2 | SCTLR_EL1_TCF_MASK) ++#define IEE_TTBR0_MASK ~0 ++#define IEE_TTBR1_MASK ~0 ++#define IEE_TCR_MASK (TCR_HD | TCR_T0SZ_MASK | TCR_E0PD1) ++#define IEE_MDSCR_MASK (DBG_MDSCR_SS | DBG_MDSCR_MDE) ++ ++#define IEE_DBGBCR_BT 0b0000 << 20 ++#define IEE_DBGBCR_SSC 0b00 << 14 ++#define IEE_DBGBCR_HMC 0b1 << 13 ++#define IEE_DBGBCR_BAS 0b1111 << 5 ++#define IEE_DBGBCR_PMC 0b11 << 1 ++#define IEE_DBGBCR_E 0b1 ++#define IEE_DBGBCR IEE_DBGBCR_BT | IEE_DBGBCR_SSC | IEE_DBGBCR_HMC | IEE_DBGBCR_BAS \ ++ | IEE_DBGBCR_PMC | IEE_DBGBCR_E ++ ++#endif +\ No newline at end of file +diff --git a/arch/arm64/include/asm/iee-slab.h b/arch/arm64/include/asm/iee-slab.h +new file mode 100644 +index 000000000000..4f3c17c7da00 +--- /dev/null ++++ b/arch/arm64/include/asm/iee-slab.h +@@ -0,0 +1,23 @@ ++#ifndef _LINUX_IEE_SLAB_H ++#define _LINUX_IEE_SLAB_H ++/* ++ * Tracking user of a slab. ++ */ ++#include <linux/stackdepot.h> ++ ++#define TRACK_ADDRS_COUNT 16 ++struct track { ++ unsigned long addr; /* Called from address */ ++#ifdef CONFIG_STACKDEPOT ++ depot_stack_handle_t handle; ++#endif ++ int cpu; /* Was running on cpu */ ++ int pid; /* Pid context */ ++ unsigned long when; /* When did the operation occur */ ++}; ++ ++enum track_item { TRACK_ALLOC, TRACK_FREE }; ++ ++typedef struct { unsigned long v; } freeptr_t; ++ ++#endif +\ No newline at end of file +diff --git a/arch/arm64/include/asm/iee-token.h b/arch/arm64/include/asm/iee-token.h +new file mode 100644 +index 000000000000..152474e1a187 +--- /dev/null ++++ b/arch/arm64/include/asm/iee-token.h +@@ -0,0 +1,40 @@ ++#ifndef _LINUX_IEE_TOKEN_H ++#define _LINUX_IEE_TOKEN_H ++ ++#include <asm/iee-def.h> ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++struct task_token; ++struct task_struct; ++struct mm_struct; ++ ++#ifdef CONFIG_IEE ++void iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm) ++{ ++ iee_rw_gate(IEE_SET_TOKEN_MM, tsk, mm); ++} ++ ++void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) ++{ ++ iee_rw_gate(IEE_SET_TOKEN_PGD, tsk, pgd); ++} ++ ++void iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack) ++{ ++ iee_rw_gate(IEE_INIT_TOKEN, tsk, kernel_stack, iee_stack); ++} ++ ++void iee_free_token(struct task_struct *tsk) ++{ ++ iee_rw_gate(IEE_FREE_TOKEN, tsk); ++} ++ ++unsigned long iee_read_token_stack(struct task_struct *tsk) ++{ ++ unsigned long ret; ++ ret = iee_rw_gate(IEE_READ_TOKEN_STACK, tsk); ++ return ret; ++} ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/arch/arm64/include/asm/iee.h b/arch/arm64/include/asm/iee.h +new file mode 100644 +index 000000000000..598f6d0b2626 +--- /dev/null ++++ b/arch/arm64/include/asm/iee.h +@@ -0,0 +1,10 @@ ++#ifndef _LINUX_IEE_H ++#define _LINUX_IEE_H ++#define __iee_code __section(".iee.text") ++#define __iee_header __section(".iee.text.header") ++ ++u64 iee_dispatch(int flag, ...); ++ ++#include <asm/iee-def.h> ++ ++#endif +diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h +index 85d26143faa5..e7a3081ce285 100644 +--- a/arch/arm64/include/asm/kernel-pgtable.h ++++ b/arch/arm64/include/asm/kernel-pgtable.h +@@ -118,4 +118,25 @@ + #define SWAPPER_RX_MMUFLAGS (SWAPPER_RW_MMUFLAGS | PTE_RDONLY) + #endif + ++#ifdef CONFIG_IEE ++ ++#ifdef CONFIG_ARM64_4K_PAGES // zgcXXX: it has been deleted in 6.6. ++#define ARM64_SWAPPER_USES_SECTION_MAPS 1 ++#else ++#define ARM64_SWAPPER_USES_SECTION_MAPS 0 ++#endif ++ ++#define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) // zgcXXX: warning: 6.6 delete this macro. should delete this line later. ++ ++#define SWAPPER_PTE_FLAGS_IDMAP (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED | PTE_RDONLY) ++#define SWAPPER_PMD_FLAGS_IDMAP (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S | PMD_SECT_RDONLY) ++ ++#if ARM64_SWAPPER_USES_SECTION_MAPS ++#define SWAPPER_MM_MMUFLAGS_IDMAP (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS_IDMAP) ++#else ++#define SWAPPER_MM_MMUFLAGS_IDMAP (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS_IDMAP) ++#endif ++ ++#endif ++ + #endif /* __ASM_KERNEL_PGTABLE_H */ +diff --git a/arch/arm64/include/asm/koi.h b/arch/arm64/include/asm/koi.h +new file mode 100644 +index 000000000000..48d9a1378a1d +--- /dev/null ++++ b/arch/arm64/include/asm/koi.h +@@ -0,0 +1,335 @@ ++#include "linux/mm.h" ++#include "asm/current.h" ++#include "asm/pgtable-hwdef.h" ++#include "asm/pgtable-types.h" ++#include "asm/pgtable.h" ++#include "linux/mm_types.h" ++#include "linux/pgtable.h" ++#include "linux/printk.h" ++#include "linux/slab.h" ++#include "linux/string.h" ++#include <linux/sched.h> ++#include "linux/hashtable.h" ++#include "linux/module.h" ++#include "linux/vmalloc.h" ++#include "stacktrace.h" ++#include "asm/mmu.h" ++#ifdef CONFIG_IEE ++#include "asm/iee-si.h" ++#include "asm/iee-def.h" ++#endif ++ ++#define HASH_TABLE_BIT 10 ++#define HASH_TABLE_LEN (1 << HASH_TABLE_BIT) ++#define HASH_KEY_MASK ((1 << HASH_TABLE_BIT) - 1) ++ ++#define MAX_VAR_NAME 64 ++#define DRIVER_ISOLATION_VAR_ARRAY_SIZE 32 ++#define DRIVER_ISOLATION_MAX_VAL 256 ++ ++extern struct hlist_head koi_mem_htbl[1024]; ++extern spinlock_t koi_mem_htbl_spin_lock; ++extern unsigned long koi_swapper_ttbr1; ++extern s64 koi_offset; ++ ++#ifdef CONFIG_IEE ++extern unsigned long long iee_rw_gate(int flag, ...); ++#endif ++ ++DECLARE_PER_CPU(unsigned long[PAGE_SIZE / sizeof(unsigned long)], ++ koi_irq_current_ttbr1); ++ ++/** ++* struct koi_mem_hash_node - ++*@mod:pointer to driver module ++*@mem_list_head:free memory list head ++*@ko_mm: mm_struct in each driver ++*@pgdp:entry to Page Global Directory :pgd ++*@node:hash linked list node ++*@addr_htbl[1 << (HASH_TABLE_BIT)]: ++*@rcu: ++*/ ++struct koi_mem_hash_node { ++ struct module *mod; ++ struct list_head mem_list_head; ++ struct mm_struct *ko_mm; ++ pgd_t *pgdp; ++ unsigned long ko_ttbr1; ++ struct hlist_node node; ++ struct hlist_head addr_htbl[1 << (HASH_TABLE_BIT)]; ++ struct rcu_head rcu; ++ // used to protect free mem list ++ spinlock_t spin_lock; ++ // used to protect addr hashtable ++ spinlock_t addr_htbl_spin_lock; ++}; ++//describe the global shared var ++struct shared_variable_descriptor { ++ unsigned int id; ++ unsigned int type; ++ char name[MAX_VAR_NAME]; ++ unsigned long offset; ++ unsigned int size; ++ unsigned int self_ptr_ids[DRIVER_ISOLATION_VAR_ARRAY_SIZE]; ++}; ++ ++int koi_do_switch_to_kernel_pgtbl(void); ++ ++int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir, ++ unsigned long addr, unsigned long end); ++ ++void koi_create_pagetable(struct module *mod); ++ ++void koi_map_kostack(struct module *mod); ++unsigned long koi_mem_alloc(struct module *mod, unsigned long orig_addr, ++ unsigned long size); ++void koi_mem_free(struct module *mod, unsigned long addr, unsigned long size, ++ bool is_const, int count, ...); ++void *koi_mem_lookup(struct module *mod, unsigned long addr); ++void koi_mem_free_callback(struct module *mod, unsigned long addr, ++ unsigned long size, void (*func)(void *)); ++void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size); ++void koi_mem_free_to_user(struct module *mod, unsigned long addr, ++ unsigned long size); ++ ++unsigned long koi_ttbr_ctor(struct module *mod); ++extern void koi_do_switch_to_kernel_stack(void); ++extern void koi_do_switch_to_ko_stack(void); ++ ++#define switch_pgtable(ttbr1) \ ++ do { \ ++ write_sysreg(ttbr1, ttbr1_el1); \ ++ isb(); \ ++ asm volatile(ALTERNATIVE("nop; nop; nop", \ ++ "ic iallu; dsb nsh; isb", \ ++ ARM64_WORKAROUND_CAVIUM_27456)); \ ++ } while (0); ++ ++#ifndef CONFIG_IEE ++#define koi_switch_to_ko() \ ++ do { \ ++ unsigned long flags, ko_ttbr1, cur_sp; \ ++ unsigned long *ptr; \ ++ struct task_token *token; \ ++ asm volatile("mrs %0, daif\n" \ ++ "msr daifset, #2\n" \ ++ "isb\n" \ ++ "mov %1, sp\n" \ ++ : "=r"(flags), "=r"(cur_sp) \ ++ :); \ ++ if (!on_irq_stack(cur_sp, NULL)) { \ ++ koi_do_switch_to_ko_stack(); \ ++ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ ++ token = (struct task_token *)((unsigned long)current + \ ++ koi_offset); \ ++ token->current_ttbr1 = ko_ttbr1 & (~TTBR_ASID_MASK); \ ++ } else { \ ++ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ ++ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ ++ __kern_my_cpu_offset()); \ ++ *ptr = ko_ttbr1 & ~(TTBR_ASID_MASK); \ ++ } \ ++ switch_pgtable(ko_ttbr1); \ ++ asm volatile("msr daif, %0\n" \ ++ "isb\n" \ ++ : \ ++ : "r"(flags)); \ ++ } while (0); ++ ++#define koi_switch_to_kernel() \ ++ do { \ ++ unsigned long cur_sp, flags, asid; \ ++ unsigned long *ptr; \ ++ struct task_token *token; \ ++ asm volatile("mrs %0, daif\n" \ ++ "msr daifset, #2\n" \ ++ "isb\n" \ ++ "mov %1, sp\n" \ ++ "mov %2, ttbr0_el1\n" \ ++ : "=r"(flags), "=r"(cur_sp), "=r"(asid) \ ++ :); \ ++ asid &= ~USER_ASID_FLAG; \ ++ asid &= TTBR_ASID_MASK; \ ++ switch_pgtable(koi_swapper_ttbr1); \ ++ if (!on_irq_stack(cur_sp, NULL)) { \ ++ token = (struct task_token *)((unsigned long)current + \ ++ koi_offset); \ ++ token->current_ttbr1 = koi_swapper_ttbr1; \ ++ koi_do_switch_to_kernel_stack(); \ ++ } else { \ ++ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ ++ __kern_my_cpu_offset()); \ ++ *ptr = koi_swapper_ttbr1; \ ++ } \ ++ asm volatile("msr daif, %0\n" \ ++ "isb\n" \ ++ : \ ++ : "r"(flags)); \ ++ } while (0); ++#else ++#define koi_switch_to_ko() \ ++ do { \ ++ unsigned long cur_sp, flags, ko_ttbr1; \ ++ unsigned long *ptr; \ ++ asm volatile("mrs %0, daif\n" \ ++ "msr daifset, #2\n" \ ++ "isb\n" \ ++ "mov %1, sp\n" \ ++ : "=r"(flags), "=r"(cur_sp) \ ++ :); \ ++ if (!on_irq_stack(cur_sp, NULL)) { \ ++ koi_do_switch_to_ko_stack(); \ ++ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ ++ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, \ ++ ko_ttbr1 &(~TTBR_ASID_MASK)); \ ++ } else { \ ++ ko_ttbr1 = koi_ttbr_ctor(THIS_MODULE); \ ++ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ ++ __kern_my_cpu_offset()); \ ++ *ptr = ko_ttbr1 & (~TTBR_ASID_MASK); \ ++ } \ ++ iee_rwx_gate_entry(IEE_SWITCH_TO_KOI, ko_ttbr1); \ ++ asm volatile("msr daif, %0\n" \ ++ "isb\n" \ ++ : \ ++ : "r"(flags)); \ ++ } while (0); ++ ++#define koi_switch_to_kernel() \ ++ do { \ ++ unsigned long flags, cur_sp; \ ++ unsigned long *ptr; \ ++ asm volatile("mrs %0, daif\n" \ ++ "msr daifset, #2\n" \ ++ "isb\n" \ ++ "mov %1, sp\n" \ ++ : "=r"(flags), "=r"(cur_sp) \ ++ :); \ ++ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); \ ++ if (!on_irq_stack(cur_sp, NULL)) { \ ++ iee_rw_gate(IEE_WRITE_TOKEN_TTBR1, current, \ ++ koi_swapper_ttbr1); \ ++ koi_do_switch_to_kernel_stack(); \ ++ } else { \ ++ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, \ ++ __kern_my_cpu_offset()); \ ++ *ptr = koi_swapper_ttbr1; \ ++ } \ ++ asm volatile("msr daif, %0\n" \ ++ "isb\n" \ ++ : \ ++ : "r"(flags)); \ ++ } while (0); ++#endif ++//kzalloc function in driver space ++static __maybe_unused noinline void * ++koi_kzalloc_wrapper(struct module *mod, size_t size, gfp_t flags) ++{ ++ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; ++ void *addr; ++ struct koi_mem_hash_node *target = NULL; ++ koi_switch_to_kernel(); ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk("mem node for module: %s not found\n", mod->name); ++ return NULL; ++ } ++ ++ addr = kzalloc(size, flags); ++ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, ++ (unsigned long)addr + PAGE_SIZE * cnt); ++ koi_switch_to_ko(); ++ return addr; ++} ++//kmalloc function in driver space ++static __maybe_unused __always_inline void * ++koi_kmalloc_wrapper(struct module *mod, size_t size, gfp_t flags) ++{ ++ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; ++ void *addr; ++ struct koi_mem_hash_node *target = NULL; ++ koi_switch_to_kernel(); ++ ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk("mem node for module: %s not found\n", mod->name); ++ return 0; ++ } ++ ++ addr = kmalloc(cnt * PAGE_SIZE, flags); ++ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, ++ (unsigned long)addr + PAGE_SIZE * cnt); ++ koi_switch_to_ko(); ++ return (void *)addr; ++} ++//vmalloc function in driver space ++static __maybe_unused void *koi_vmalloc_wrapper(struct module *mod, ++ unsigned long size) ++{ ++ int cnt = (size + PAGE_SIZE - 1) / PAGE_SIZE; ++ void *addr; ++ struct koi_mem_hash_node *target = NULL; ++ koi_switch_to_kernel(); ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk("mem node for module: %s not found\n", mod->name); ++ koi_switch_to_ko(); ++ return 0; ++ } ++ addr = vmalloc(cnt * PAGE_SIZE); ++ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, ++ (unsigned long)addr + PAGE_SIZE * cnt); ++ koi_switch_to_ko(); ++ return addr; ++} ++//kmalloc_array function in driver space ++static __maybe_unused void *koi_kmalloc_array_wrapper(struct module *mod, ++ size_t n, size_t size, ++ gfp_t flags) ++{ ++ int kpage; ++ void *addr; ++ struct koi_mem_hash_node *target = NULL; ++ koi_switch_to_kernel(); ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk("mem node for module: %s not found\n", mod->name); ++ koi_switch_to_ko(); ++ return 0; ++ } ++ kpage = (n * size + PAGE_SIZE - 1) / PAGE_SIZE; ++ n = (kpage * PAGE_SIZE) / size; ++ addr = kmalloc_array(n, size, flags); ++ koi_copy_pagetable(target->ko_mm, target->pgdp, (unsigned long)addr, ++ (unsigned long)addr + PAGE_SIZE * kpage); ++ koi_switch_to_ko(); ++ return addr; ++} +\ No newline at end of file +diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h +index fde4186cc387..6309e5514a16 100644 +--- a/arch/arm64/include/asm/memory.h ++++ b/arch/arm64/include/asm/memory.h +@@ -190,6 +190,13 @@ extern u64 vabits_actual; + #endif + + extern s64 memstart_addr; ++ ++#if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++extern s64 memstart_addr_init; ++extern s64 iee_offset; ++#define LOGICAL_RANDOM (long long int)((long unsigned int)__va(memstart_addr_init) & (~PAGE_OFFSET)) ++#endif ++ + /* PHYS_OFFSET - the physical address of the start of memory. */ + #define PHYS_OFFSET ({ VM_BUG_ON(memstart_addr & 1); memstart_addr; }) + +@@ -310,6 +317,23 @@ extern phys_addr_t __phys_addr_symbol(unsigned long x); + #define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET) | PAGE_OFFSET) + #define __phys_to_kimg(x) ((unsigned long)((x) + kimage_voffset)) + ++#ifdef CONFIG_KOI ++#define KOI_OFFSET ((unsigned long)BIT(vabits_actual - 2)) ++#endif ++ ++#ifdef CONFIG_IEE ++#ifdef CONFIG_IEE_OFFSET ++#define IEE_OFFSET ((CONFIG_IEE_OFFSET) - LOGICAL_RANDOM) ++#else ++#define IEE_OFFSET (((unsigned long)BIT(vabits_actual - 2)) - LOGICAL_RANDOM) ++#endif ++#define __phys_to_iee(x) (__phys_to_virt(x) + IEE_OFFSET) ++#define SET_UPAGE(x) __pgprot(pgprot_val(x) | PTE_USER) ++#define SET_PPAGE(x) __pgprot(pgprot_val(x) & (~PTE_USER)) ++#define SET_INVALID(x) __pgprot(pgprot_val(x) & (~PTE_VALID)) ++#define SET_NG(x) __pgprot(pgprot_val(x) | PTE_NG) ++#endif ++ + /* + * Convert a page to/from a physical address + */ +diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h +index a6fb325424e7..cca5994dabfb 100644 +--- a/arch/arm64/include/asm/mmu_context.h ++++ b/arch/arm64/include/asm/mmu_context.h +@@ -24,6 +24,9 @@ + #include <asm/cputype.h> + #include <asm/sysreg.h> + #include <asm/tlbflush.h> ++#ifdef CONFIG_IEE ++#define INIT_ASID 0x2 ++#endif + + extern bool rodata_full; + +@@ -43,7 +46,12 @@ static inline void cpu_set_reserved_ttbr0_nosync(void) + { + unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); + ++#ifdef CONFIG_IEE ++ ttbr |= FIELD_PREP(TTBR_ASID_MASK, 1); ++ iee_rwx_gate_entry(IEE_WRITE_ttbr0_el1, ttbr); ++#else + write_sysreg(ttbr, ttbr0_el1); ++#endif + } + + static inline void cpu_set_reserved_ttbr0(void) +@@ -79,7 +87,11 @@ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) + + tcr &= ~TCR_T0SZ_MASK; + tcr |= t0sz << TCR_T0SZ_OFFSET; ++#ifdef CONFIG_IEE ++ iee_rwx_gate_entry(IEE_WRITE_tcr_el1, tcr); ++#else + write_sysreg(tcr, tcr_el1); ++#endif + isb(); + } + +@@ -144,7 +156,11 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) + __cpu_set_tcr_t0sz(t0sz); + + /* avoid cpu_switch_mm() and its SW-PAN and CNP interactions */ ++ #ifdef CONFIG_IEE ++ iee_rwx_gate_entry(IEE_WRITE_ttbr0_el1, ttbr0); ++ #else + write_sysreg(ttbr0, ttbr0_el1); ++ #endif + isb(); + } + +@@ -174,6 +190,10 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) + ttbr1 |= TTBR_CNP_BIT; + } + ++ #ifdef CONFIG_IEE ++ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, ASID(current->active_mm)); ++ #endif ++ + replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); + + __cpu_install_idmap(idmap); +diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h +index 237224484d0f..4e3304da8421 100644 +--- a/arch/arm64/include/asm/pgalloc.h ++++ b/arch/arm64/include/asm/pgalloc.h +@@ -63,6 +63,10 @@ static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) + extern pgd_t *pgd_alloc(struct mm_struct *mm); + extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp); + ++#ifdef CONFIG_KOI ++pgd_t *koi_pgd_alloc(void); ++#endif ++ + static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep, + pmdval_t prot) + { +diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h +index e4944d517c99..7f60e568c964 100644 +--- a/arch/arm64/include/asm/pgtable-hwdef.h ++++ b/arch/arm64/include/asm/pgtable-hwdef.h +@@ -84,6 +84,13 @@ + #define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE) + #define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1)) + ++#ifdef CONFIG_IEE ++#define PGD_APT_RO (_AT(pudval_t, 1) << 62) ++#endif ++#define PGD_APT (_AT(pudval_t, 1) << 61) ++#define PGD_PXN (_AT(pudval_t, 1) << 59) ++#define PGD_UXN (_AT(pudval_t, 1) << 60) ++ + /* + * Hardware page table definitions. + * +@@ -285,6 +292,10 @@ + #define TCR_TCMA0 (UL(1) << 57) + #define TCR_TCMA1 (UL(1) << 58) + ++#ifdef CONFIG_IEE ++#define TCR_HPD1 (UL(1) << 42) ++#endif ++ + /* + * TTBR. + */ +diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h +index 8d68d00de0a4..6f8d5b85bfd7 100644 +--- a/arch/arm64/include/asm/pgtable.h ++++ b/arch/arm64/include/asm/pgtable.h +@@ -34,6 +34,9 @@ + #include <linux/mm_types.h> + #include <linux/sched.h> + #include <linux/page_table_check.h> ++#ifdef CONFIG_PTP ++#include <asm/iee.h> ++#endif + + #ifdef CONFIG_TRANSPARENT_HUGEPAGE + #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE +@@ -156,6 +159,30 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) + #define pud_access_permitted(pud, write) \ + (pte_access_permitted(pud_pte(pud), (write))) + ++#ifdef CONFIG_PTP ++static inline bool in_tramp_pgdir(void *addr); ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++static void iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) ++{ ++ iee_rw_gate(IEE_OP_SET_TRAMP_PGD, pgdp, pgd); ++} ++ ++static noinline pteval_t iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval) ++{ ++ pteval_t ret; ++ ret = iee_rw_gate(IEE_OP_SET_XCHG, ptep, pteval); ++ return (pteval_t)ret; ++} ++ ++static noinline pteval_t iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval) ++{ ++ pteval_t ret; ++ ret = iee_rw_gate(IEE_OP_SET_CMPXCHG, ptep, old_pteval, new_pteval); ++ return (pteval_t)ret; ++} ++#endif ++ + static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) + { + pte_val(pte) &= ~pgprot_val(prot); +@@ -262,6 +289,64 @@ static inline pte_t pte_mkdevmap(pte_t pte) + + static inline void __set_pte(pte_t *ptep, pte_t pte) + { ++#ifdef CONFIG_KOI ++ if (!pte_none(pte)) { ++ pte = __pte(pte_val(pte) | PTE_NG); ++ } ++#endif ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_PTE, ptep, pte); ++ dsb(ishst); ++ isb(); ++#else ++ WRITE_ONCE(*ptep, pte); ++ ++ /* ++ * Only if the new pte is valid and kernel, otherwise TLB maintenance ++ * or update_mmu_cache() have the necessary barriers. ++ */ ++ if (pte_valid_not_user(pte)) { ++ dsb(ishst); ++ isb(); ++ } ++#endif ++} ++ ++ ++#ifdef CONFIG_PTP ++static inline void iee_set_bm_pte(pte_t *ptep, pte_t pte) ++{ ++ // If it is pre init, write once. ++ // Else, write once will cause exception. So it is safe. ++ unsigned long flags; ++ unsigned long res; ++ local_irq_save(flags); ++ asm volatile("at s1e1r, %0"::"r"(__phys_to_iee(__pa_symbol(ptep)))); ++ isb(); ++ res = read_sysreg(par_el1); ++ local_irq_restore(flags); ++ if(res & 0x1) ++ WRITE_ONCE(*ptep,pte); ++ else ++ iee_rw_gate(IEE_OP_SET_BM_PTE, ptep, pte); ++ ++ /* ++ * Only if the new pte is valid and kernel, otherwise TLB maintenance ++ * or update_mmu_cache() have the necessary barriers. ++ */ ++ if (pte_valid_not_user(pte)) { ++ dsb(ishst); ++ isb(); ++ } ++} ++ ++static inline void iee_set_fixmap_pte_pre_init(pte_t *ptep, pte_t pte) ++{ ++#ifdef CONFIG_KOI ++ if (!pte_none(pte)) { ++ pte = __pte(pte_val(pte) | PTE_NG); ++ } ++#endif + WRITE_ONCE(*ptep, pte); + + /* +@@ -273,6 +358,7 @@ static inline void __set_pte(pte_t *ptep, pte_t pte) + isb(); + } + } ++#endif + + static inline pte_t __ptep_get(pte_t *ptep) + { +@@ -546,6 +632,95 @@ static inline void __set_pte_at(struct mm_struct *mm, + __set_pte(ptep, pte); + } + ++#ifdef CONFIG_IEE ++static inline void iee_set_pte_upage(pte_t *ptep, pte_t pte) ++{ ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_PTE_U, ptep, pte); ++ dsb(ishst); ++ isb(); ++#else ++ WRITE_ONCE(*ptep, pte); ++ if (pte_valid_not_user(pte)) { ++ dsb(ishst); ++ isb(); ++ } ++#endif ++} ++ ++static inline void iee_set_pte_ppage(pte_t *ptep, pte_t pte) ++{ ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_PTE_P, ptep, pte); ++#else ++ WRITE_ONCE(*ptep, pte); ++#endif ++ if (pte_valid_not_user(pte)) { ++ dsb(ishst); ++ isb(); ++ } ++} ++#endif ++ ++#ifdef CONFIG_PTP ++static inline void set_pmd(pmd_t *pmdp, pmd_t pmd); ++static inline void __set_pmd_at(struct mm_struct *mm, unsigned long addr, ++ pmd_t *pmdp, pmd_t pmd) ++{ ++ if (pte_present(pmd_pte(pmd)) && pte_user_exec(pmd_pte(pmd)) && !pte_special(pmd_pte(pmd))) ++ __sync_icache_dcache(pmd_pte(pmd)); ++ ++ /* ++ * If the PTE would provide user space access to the tags associated ++ * with it then ensure that the MTE tags are synchronised. Although ++ * pte_access_permitted() returns false for exec only mappings, they ++ * don't expose tags (instruction fetches don't check tags). ++ */ ++ if (system_supports_mte() && pte_access_permitted(pmd_pte(pmd), false) && ++ !pte_special(pmd_pte(pmd)) && pte_tagged(pmd_pte(pmd))) ++ mte_sync_tags(pmd_pte(pmd), PMD_SIZE >> PAGE_SHIFT); ++ ++ __check_safe_pte_update(mm, (pte_t *)pmdp, pmd_pte(pmd)); ++ ++ set_pmd(pmdp, pmd); ++} ++ ++static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, ++ pmd_t *pmdp, pmd_t pmd) ++{ ++ page_table_check_pmd_set(mm, pmdp, pmd); ++ return __set_pmd_at(mm, addr, pmdp, pmd); ++} ++ ++static inline void set_pud(pud_t *pudp, pud_t pud); ++static inline void __set_pud_at(struct mm_struct *mm, unsigned long addr, ++ pud_t *pudp, pud_t pud) ++{ ++ if (pte_present(pud_pte(pud)) && pte_user_exec(pud_pte(pud)) && !pte_special(pud_pte(pud))) ++ __sync_icache_dcache(pud_pte(pud)); ++ ++ /* ++ * If the PTE would provide user space access to the tags associated ++ * with it then ensure that the MTE tags are synchronised. Although ++ * pte_access_permitted() returns false for exec only mappings, they ++ * don't expose tags (instruction fetches don't check tags). ++ */ ++ if (system_supports_mte() && pte_access_permitted(pud_pte(pud), false) && ++ !pte_special(pud_pte(pud)) && pte_tagged(pud_pte(pud))) ++ mte_sync_tags(pud_pte(pud), PUD_SIZE >> PAGE_SHIFT); ++ ++ __check_safe_pte_update(mm, (pte_t *)pudp, pud_pte(pud)); ++ ++ set_pud(pudp, pud); ++} ++ ++static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, ++ pud_t *pudp, pud_t pud) ++{ ++ page_table_check_pud_set(mm, pudp, pud); ++ return __set_pud_at(mm, addr, pudp, pud); ++} ++#else + static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) + { +@@ -561,7 +736,7 @@ static inline void set_pud_at(struct mm_struct *mm, unsigned long addr, + return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud), + PUD_SIZE >> PAGE_SHIFT); + } +- ++#endif + #define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d)) + #define __phys_to_p4d_val(phys) __phys_to_pte_val(phys) + +@@ -640,7 +815,14 @@ static inline bool in_swapper_pgdir(void *addr) + ((unsigned long)swapper_pg_dir & PAGE_MASK); + } + +-static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) ++#ifdef CONFIG_PTP ++static inline bool in_tramp_pgdir(void *addr) ++{ ++ return ((unsigned long)addr & PAGE_MASK) == ++ ((unsigned long)tramp_pg_dir & PAGE_MASK); ++} ++ ++static inline void iee_set_fixmap_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) + { + #ifdef __PAGETABLE_PMD_FOLDED + if (in_swapper_pgdir(pmdp)) { +@@ -648,7 +830,6 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) + return; + } + #endif /* __PAGETABLE_PMD_FOLDED */ +- + WRITE_ONCE(*pmdp, pmd); + + if (pmd_valid(pmd)) { +@@ -656,6 +837,32 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) + isb(); + } + } ++#endif ++ ++static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) ++{ ++#ifdef __PAGETABLE_PMD_FOLDED ++ if (in_swapper_pgdir(pmdp)) { ++ set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd))); ++ return; ++ } ++#endif /* __PAGETABLE_PMD_FOLDED */ ++#ifdef CONFIG_KOI ++ pmdval_t val = pmd_val(pmd); ++ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { ++ pmd = __pmd(val | PMD_SECT_NG); ++ } ++#endif ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_PMD, pmdp, pmd); ++#else ++ WRITE_ONCE(*pmdp, pmd); ++#endif ++ if (pmd_valid(pmd)) { ++ dsb(ishst); ++ isb(); ++ } ++} + + static inline void pmd_clear(pmd_t *pmdp) + { +@@ -675,6 +882,12 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) + /* Find an entry in the third-level page table. */ + #define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t)) + ++#ifdef CONFIG_PTP ++#define pte_set_fixmap_init(addr) ((pte_t *)iee_set_fixmap_offset_pre_init(FIX_PTE, addr)) ++#define pte_set_fixmap_offset_init(pmd, addr) pte_set_fixmap_init(pte_offset_phys(pmd, addr)) ++#define pte_clear_fixmap_init() clear_fixmap_init(FIX_PTE) ++#endif ++ + #define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) + #define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) + #define pte_clear_fixmap() clear_fixmap(FIX_PTE) +@@ -703,7 +916,9 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) + #define pud_user(pud) pte_user(pud_pte(pud)) + #define pud_user_exec(pud) pte_user_exec(pud_pte(pud)) + +-static inline void set_pud(pud_t *pudp, pud_t pud) ++ ++#ifdef CONFIG_PTP ++static inline void iee_set_fixmap_pud_pre_init(pud_t *pudp, pud_t pud) + { + #ifdef __PAGETABLE_PUD_FOLDED + if (in_swapper_pgdir(pudp)) { +@@ -711,7 +926,6 @@ static inline void set_pud(pud_t *pudp, pud_t pud) + return; + } + #endif /* __PAGETABLE_PUD_FOLDED */ +- + WRITE_ONCE(*pudp, pud); + + if (pud_valid(pud)) { +@@ -719,6 +933,33 @@ static inline void set_pud(pud_t *pudp, pud_t pud) + isb(); + } + } ++#endif ++ ++static inline void set_pud(pud_t *pudp, pud_t pud) ++{ ++#ifdef __PAGETABLE_PUD_FOLDED ++ if (in_swapper_pgdir(pudp)) { ++ set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud))); ++ return; ++ } ++#endif /* __PAGETABLE_PUD_FOLDED */ ++#ifdef CONFIG_KOI ++ pudval_t val = pud_val(pud); ++ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) { ++ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead. ++ pud = __pud(val | PMD_SECT_NG); ++ } ++#endif ++#ifdef CONFIG_PTP ++ iee_rw_gate(IEE_OP_SET_PUD, pudp, pud); ++#else ++ WRITE_ONCE(*pudp, pud); ++#endif ++ if (pud_valid(pud)) { ++ dsb(ishst); ++ isb(); ++ } ++} + + static inline void pud_clear(pud_t *pudp) + { +@@ -738,6 +979,12 @@ static inline pmd_t *pud_pgtable(pud_t pud) + /* Find an entry in the second-level page table. */ + #define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t)) + ++#ifdef CONFIG_PTP ++#define pmd_set_fixmap_init(addr) ((pmd_t *)iee_set_fixmap_offset_pre_init(FIX_PMD, addr)) ++#define pmd_set_fixmap_offset_init(pud, addr) pmd_set_fixmap_init(pmd_offset_phys(pud, addr)) ++#define pmd_clear_fixmap_init() clear_fixmap_init(FIX_PMD) ++#endif ++ + #define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) + #define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) + #define pmd_clear_fixmap() clear_fixmap(FIX_PMD) +@@ -769,15 +1016,26 @@ static inline pmd_t *pud_pgtable(pud_t pud) + #define p4d_none(p4d) (!p4d_val(p4d)) + #define p4d_bad(p4d) (!(p4d_val(p4d) & 2)) + #define p4d_present(p4d) (p4d_val(p4d)) ++#define p4d_valid(p4d) pte_valid(p4d_pte(p4d)) + + static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) + { +- if (in_swapper_pgdir(p4dp)) { ++ if (in_swapper_pgdir(p4dp)) ++ { + set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); + return; + } + ++#ifdef CONFIG_PTP ++ if(in_tramp_pgdir(p4dp)) ++ { ++ iee_set_tramp_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); ++ return; ++ } ++ iee_rw_gate(IEE_OP_SET_P4D, p4dp, p4d); ++#else + WRITE_ONCE(*p4dp, p4d); ++#endif + dsb(ishst); + isb(); + } +@@ -800,6 +1058,12 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) + /* Find an entry in the first-level page table. */ + #define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t)) + ++#ifdef CONFIG_PTP ++#define pud_set_fixmap_init(addr) ((pud_t *)iee_set_fixmap_offset_pre_init(FIX_PUD, addr)) ++#define pud_set_fixmap_offset_init(p4d, addr) pud_set_fixmap_init(pud_offset_phys(p4d, addr)) ++#define pud_clear_fixmap_init() clear_fixmap_init(FIX_PUD) ++#endif ++ + #define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) + #define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr)) + #define pud_clear_fixmap() clear_fixmap(FIX_PUD) +@@ -826,6 +1090,10 @@ static inline pud_t *p4d_pgtable(p4d_t p4d) + #define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) + ++#ifdef CONFIG_PTP ++#define pgd_set_fixmap_init(addr) ((pgd_t *)iee_set_fixmap_offset_pre_init(FIX_PGD, addr)) ++#define pgd_clear_fixmap_init() clear_fixmap_init(FIX_PGD) ++#endif + #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) + #define pgd_clear_fixmap() clear_fixmap(FIX_PGD) + +@@ -912,8 +1180,13 @@ static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma, + do { + old_pte = pte; + pte = pte_mkold(pte); ++ #ifdef CONFIG_PTP ++ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep, ++ pte_val(old_pte), pte_val(pte)); ++ #else + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), + pte_val(old_pte), pte_val(pte)); ++ #endif + } while (pte_val(pte) != pte_val(old_pte)); + + return pte_young(pte); +@@ -952,8 +1225,12 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + static inline pte_t __ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, pte_t *ptep) + { ++ #ifdef CONFIG_PTP ++ pteval_t pteval= iee_set_xchg_relaxed((pte_t *)&pte_val(*ptep), (pteval_t)0); ++ pte_t pte = __pte(pteval); ++ #else + pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0)); +- ++ #endif + page_table_check_pte_clear(mm, pte); + + return pte; +@@ -995,7 +1272,12 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, + static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) + { ++ #ifdef CONFIG_PTP ++ pteval_t pteval= iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pteval_t)0); ++ pmd_t pmd = __pmd(pteval); ++ #else + pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0)); ++ #endif + + page_table_check_pmd_clear(mm, pmd); + +@@ -1012,8 +1294,12 @@ static inline void ___ptep_set_wrprotect(struct mm_struct *mm, + do { + old_pte = pte; + pte = pte_wrprotect(pte); ++ #ifdef CONFIG_PTP ++ pte_val(pte) = iee_set_cmpxchg_relaxed(ptep,pte_val(old_pte), pte_val(pte)); ++ #else + pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep), + pte_val(old_pte), pte_val(pte)); ++ #endif + } while (pte_val(pte) != pte_val(old_pte)); + } + +@@ -1049,7 +1335,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) + { + page_table_check_pmd_set(vma->vm_mm, pmdp, pmd); ++ #ifdef CONFIG_PTP ++ return __pmd((pmdval_t)iee_set_xchg_relaxed((pte_t *)&pmd_val(*pmdp), (pmdval_t)pmd_val(pmd))); ++ #else + return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd))); ++ #endif + } + #endif + +diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h +index d2e0306e65d3..8352e92d4536 100644 +--- a/arch/arm64/include/asm/pointer_auth.h ++++ b/arch/arm64/include/asm/pointer_auth.h +@@ -108,8 +108,13 @@ static __always_inline void ptrauth_enable(void) + { + if (!system_supports_address_auth()) + return; ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, 0, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | ++ SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)); ++ #else + sysreg_clear_set(sctlr_el1, 0, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | + SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)); ++ #endif + isb(); + } + +diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h +index 42358b8d678e..1e2d11e57fe3 100644 +--- a/arch/arm64/include/asm/sysreg.h ++++ b/arch/arm64/include/asm/sysreg.h +@@ -1116,6 +1116,64 @@ + write_sysreg_s(__scs_new, sysreg); \ + } while (0) + ++ ++#ifdef CONFIG_IEE ++ ++#define SYS_TCR_IEE_SI TCR_HPD1 | TCR_A1 ++ ++extern void iee_rwx_gate_entry(int flag, ...); ++#define IEE_SI_TEST 0 ++#define IEE_WRITE_sctlr_el1 1 ++#define IEE_WRITE_ttbr0_el1 2 ++#define IEE_WRITE_vbar_el1 3 ++#define IEE_WRITE_tcr_el1 4 ++#define IEE_WRITE_mdscr_el1 5 ++#define IEE_WRITE_AFSR0 10 ++ ++#define sysreg_clear_set_iee_si(sysreg, clear, set) do { \ ++ u64 __scs_val = read_sysreg(sysreg); \ ++ u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \ ++ if (__scs_new != __scs_val) \ ++ iee_rwx_gate_entry(IEE_WRITE_##sysreg, __scs_new); \ ++} while (0) ++ ++#define IEE_SI_WRITE_DAIF_SEL "msr daifclr, #0xf\n\t" \ ++ "tbnz %x0, #6, 114221f\n\t" \ ++ "tbnz %x0, #7, 114210f\n\t" \ ++ "tbnz %x0, #8, 114100f\n\t" \ ++ "msr daifset, #0b000\n\t" \ ++ "b 114514f\n\t" \ ++"114221:\n\t" \ ++ "tbnz %x0, #7, 114211f\n\t" \ ++ "tbnz %x0, #8, 114101f\n\t" \ ++ "msr daifset, #0b001\n\t" \ ++ "b 114514f\n\t" \ ++"114211:\n\t" \ ++ "tbnz %x0, #8, 114111f\n\t" \ ++ "msr daifset, #0b011\n\t" \ ++ "b 114514f\n\t" \ ++"114210:\n\t" \ ++ "tbnz %x0, #8, 114110f\n\t" \ ++ "msr daifset, #0b010\n\t" \ ++ "b 114514f\n\t" \ ++"114100:\n\t" \ ++ "msr daifset, #0b100\n\t" \ ++ "b 114514f\n\t" \ ++"114101:\n\t" \ ++ "msr daifset, #0b101\n\t" \ ++ "b 114514f\n\t" \ ++"114110:\n\t" \ ++ "msr daifset, #0b110\n\t" \ ++ "b 114514f\n\t" \ ++"114111:\n\t" \ ++ "msr daifset, #0b111\n\t" \ ++"114514:\n\t" ++ ++#define iee_si_write_daif(v) do { \ ++ u64 __val = (u64)(v); \ ++ asm volatile(IEE_SI_WRITE_DAIF_SEL: : "rZ" (__val));} while (0) ++#endif ++ + #define read_sysreg_par() ({ \ + u64 par; \ + asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412)); \ +diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h +index 2c29239d05c3..955f99317790 100644 +--- a/arch/arm64/include/asm/tlb.h ++++ b/arch/arm64/include/asm/tlb.h +@@ -11,8 +11,17 @@ + #include <linux/pagemap.h> + #include <linux/swap.h> + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + static inline void __tlb_remove_table(void *_table) + { ++#ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(page_to_phys((struct page *)_table)); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)page_address((struct page *)_table)); ++#endif + free_page_and_swap_cache((struct page *)_table); + } + +diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h +index 831c314d75ff..7775628528c6 100644 +--- a/arch/arm64/include/asm/tlbflush.h ++++ b/arch/arm64/include/asm/tlbflush.h +@@ -49,6 +49,7 @@ + + #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) + ++ + #define __tlbi_user(op, arg) do { \ + if (arm64_kernel_unmapped_at_el0()) \ + __tlbi(op, (arg) | USER_ASID_FLAG); \ +@@ -258,6 +259,10 @@ static inline void flush_tlb_mm(struct mm_struct *mm) + asid = __TLBI_VADDR(0, ASID(mm)); + __tlbi(aside1is, asid); + __tlbi_user(aside1is, asid); ++ #if defined(CONFIG_IEE) || defined (CONFIG_KOI) ++ if (!arm64_kernel_unmapped_at_el0()) ++ __tlbi(aside1is, asid | USER_ASID_FLAG); ++ #endif + dsb(ish); + mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); + } +@@ -273,6 +278,10 @@ static inline void __flush_tlb_page_nosync(struct mm_struct *mm, + __tlbi_user(vale1is, addr); + mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, + (uaddr & PAGE_MASK) + PAGE_SIZE); ++ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ if (!arm64_kernel_unmapped_at_el0()) ++ __tlbi(vale1is, addr | USER_ASID_FLAG); ++ #endif + } + + static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, +@@ -366,6 +375,45 @@ static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) + * 2. If there is 1 page remaining, flush it through non-range operations. Range + * operations can only span an even number of pages. + */ ++#if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++#define __flush_tlb_range_op(op, start, pages, stride, \ ++ asid, tlb_level, tlbi_user) \ ++do { \ ++ int num = 0; \ ++ int scale = 0; \ ++ unsigned long addr; \ ++ \ ++ while (pages > 0) { \ ++ if (!system_supports_tlb_range() || \ ++ pages % 2 == 1) { \ ++ addr = __TLBI_VADDR(start, asid); \ ++ __tlbi_level(op, addr, tlb_level); \ ++ if (!arm64_kernel_unmapped_at_el0()) /* added for IEE */ \ ++ __tlbi_level(op, addr | USER_ASID_FLAG, tlb_level); \ ++ if (tlbi_user) \ ++ __tlbi_user_level(op, addr, tlb_level); \ ++ start += stride; \ ++ pages -= stride >> PAGE_SHIFT; \ ++ continue; \ ++ } \ ++ \ ++ num = __TLBI_RANGE_NUM(pages, scale); \ ++ if (num >= 0) { \ ++ addr = __TLBI_VADDR_RANGE(start, asid, scale, \ ++ num, tlb_level); \ ++ __tlbi(r##op, addr); \ ++ if (!arm64_kernel_unmapped_at_el0()) /* added for IEE */ \ ++ __tlbi(r##op, addr | USER_ASID_FLAG); \ ++ if (tlbi_user) \ ++ __tlbi_user(r##op, addr); \ ++ start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ ++ pages -= __TLBI_RANGE_PAGES(num, scale); \ ++ } \ ++ scale++; \ ++ } \ ++} while (0) ++ ++#else + #define __flush_tlb_range_op(op, start, pages, stride, \ + asid, tlb_level, tlbi_user) \ + do { \ +@@ -399,6 +447,8 @@ do { \ + } \ + } while (0) + ++#endif //if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ + #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ + __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false) + +@@ -467,7 +517,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end + return; + } + +- start = __TLBI_VADDR(start, 0); ++ start = __TLBI_VADDR(start, 0); + end = __TLBI_VADDR(end, 0); + + dsb(ishst); +@@ -483,9 +533,9 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end + */ + static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) + { +- unsigned long addr = __TLBI_VADDR(kaddr, 0); +- +- dsb(ishst); ++ unsigned long addr = __TLBI_VADDR(kaddr, 0); ++ ++ dsb(ishst); + __tlbi(vaae1is, addr); + dsb(ish); + isb(); +diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile +index 21ef9c21a400..44eb76cc54d7 100644 +--- a/arch/arm64/kernel/Makefile ++++ b/arch/arm64/kernel/Makefile +@@ -36,6 +36,8 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ + syscall.o proton-pack.o idreg-override.o idle.o \ + patching.o + ++obj-y += iee/ ++obj-$(CONFIG_KOI) += koi/ + obj-$(CONFIG_AARCH32_EL0) += binfmt_elf32.o sys32.o signal32.o \ + sys_compat.o + obj-$(CONFIG_AARCH32_EL0) += sigreturn32.o +diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c +index fd0f291e215e..c008e46b5fc0 100644 +--- a/arch/arm64/kernel/armv8_deprecated.c ++++ b/arch/arm64/kernel/armv8_deprecated.c +@@ -306,11 +306,19 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr) + + static int cp15_barrier_set_hw_mode(bool enable) + { ++#ifdef CONFIG_IEE ++ if (enable) ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_CP15BEN); ++ else ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_CP15BEN, 0); ++ return 0; ++#else + if (enable) + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN); + else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0); + return 0; ++#endif + } + + static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn) +@@ -341,11 +349,19 @@ static int setend_set_hw_mode(bool enable) + if (!cpu_supports_mixed_endian_el0()) + return -EINVAL; + ++#ifdef CONFIG_IEE ++ if (enable) ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_CP15BEN); ++ else ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_CP15BEN, 0); ++ return 0; ++#else + if (enable) + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0); + else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED); + return 0; ++#endif + } + + static int __a32_setend_handler(struct pt_regs *regs, u32 big_endian) +diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c +index e997ad275afb..e105f633355a 100644 +--- a/arch/arm64/kernel/asm-offsets.c ++++ b/arch/arm64/kernel/asm-offsets.c +@@ -97,6 +97,17 @@ int main(void) + DEFINE(FREGS_DIRECT_TRAMP, offsetof(struct ftrace_regs, direct_tramp)); + #endif + DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs)); ++#ifdef CONFIG_IEE ++ DEFINE(iee_from_token_offset, offsetof(struct task_token, iee_stack)); ++ DEFINE(kernel_from_token_offset, offsetof(struct task_token, kernel_stack)); ++ DEFINE(mm_from_task_offset, offsetof(struct task_struct, mm)); ++#endif ++#ifdef CONFIG_KOI ++ DEFINE(koi_kernel_from_token_offset, offsetof(struct task_token, koi_kernel_stack)); ++ DEFINE(koi_from_token_offset, offsetof(struct task_token, koi_stack)); ++ DEFINE(ttbr1_from_token_offset, offsetof(struct task_token, current_ttbr1)); ++ DEFINE(koi_stack_base_from_token_offset, offsetof(struct task_token, koi_stack_base)); ++#endif + BLANK(); + #endif + #ifdef CONFIG_AARCH32_EL0 +diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c +index 46813132a09f..412006aa323c 100644 +--- a/arch/arm64/kernel/cpu_errata.c ++++ b/arch/arm64/kernel/cpu_errata.c +@@ -80,7 +80,11 @@ hisilicon_1980005_enable(const struct arm64_cpu_capabilities *__unused) + __set_bit(ARM64_HAS_CACHE_IDC, system_cpucaps); + arm64_ftr_reg_ctrel0.sys_val |= BIT(CTR_EL0_IDC_SHIFT); + arm64_ftr_reg_ctrel0.strict_mask &= ~BIT(CTR_EL0_IDC_SHIFT); ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); ++#endif + } + #endif + +@@ -132,7 +136,11 @@ cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *cap) + enable_uct_trap = true; + + if (enable_uct_trap) ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); ++#endif + } + + #ifdef CONFIG_ARM64_ERRATUM_1463225 +@@ -147,7 +155,11 @@ has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry, + static void __maybe_unused + cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) + { ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCI, 0); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0); ++#endif + } + + #ifdef CONFIG_HISILICON_ERRATUM_HIP08_RU_PREFETCH +diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c +index 74e445251b51..ebc1e776b175 100644 +--- a/arch/arm64/kernel/cpufeature.c ++++ b/arch/arm64/kernel/cpufeature.c +@@ -94,6 +94,10 @@ + #include <asm/vectors.h> + #include <asm/virt.h> + ++#ifdef CONFIG_IEE ++#include <asm/iee-si.h> ++#endif ++ + /* Kernel representation of AT_HWCAP and AT_HWCAP2 */ + static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly; + +@@ -1612,7 +1616,11 @@ static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unu + * value. + */ + if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT))) ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_UCT, 0); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0); ++#endif + } + + static bool has_cache_dic(const struct arm64_cpu_capabilities *entry, +@@ -1873,7 +1881,11 @@ static inline void __cpu_enable_hw_dbm(void) + { + u64 tcr = read_sysreg(tcr_el1) | TCR_HD; + ++#ifdef CONFIG_IEE ++ iee_rwx_gate_entry(IEE_WRITE_tcr_el1, tcr); ++#else + write_sysreg(tcr, tcr_el1); ++#endif + isb(); + local_flush_tlb_all(); + } +@@ -2056,7 +2068,9 @@ static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) + */ + WARN_ON_ONCE(in_interrupt()); + ++ #ifndef CONFIG_IEE + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0); ++ #endif + set_pstate_pan(1); + } + #endif /* CONFIG_ARM64_PAN */ +@@ -2121,7 +2135,11 @@ static bool has_generic_auth(const struct arm64_cpu_capabilities *entry, + static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap) + { + if (this_cpu_has_cap(ARM64_HAS_E0PD)) ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(tcr_el1, 0, TCR_E0PD1); ++#else + sysreg_clear_set(tcr_el1, 0, TCR_E0PD1); ++#endif + } + #endif /* CONFIG_ARM64_E0PD */ + +@@ -2214,7 +2232,11 @@ static void nmi_enable(const struct arm64_cpu_capabilities *__unused) + * avoid leaving things masked. + */ + _allint_clear(); ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI); ++ #else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPINTMASK, SCTLR_EL1_NMI); ++ #endif + isb(); + } + #endif +@@ -2229,7 +2251,11 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) + * So, be strict and forbid other BRs using other registers to + * jump onto a PACIxSP instruction: + */ ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); ++#else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1); ++#endif + isb(); + } + #endif /* CONFIG_ARM64_BTI */ +@@ -2237,7 +2263,11 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) + #ifdef CONFIG_ARM64_MTE + static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) + { ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); ++ #else + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); ++ #endif + + mte_cpu_setup(); + +@@ -2271,7 +2301,11 @@ static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, in + + static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused) + { ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_TIDCP); ++ #else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP); ++ #endif + } + + static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) +@@ -2281,7 +2315,11 @@ static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused) + + static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused) + { ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_EL1_MSCEn); ++ #else + sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn); ++ #endif + } + + /* Internal helper functions to match cpu capability type */ +@@ -3475,6 +3513,43 @@ static void __init setup_system_capabilities(void) + enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU); + } + ++#ifdef CONFIG_IEE ++ ++static void iee_si_test_end(void) ++{ ++ pr_info("IEE: testing iee_exec_entry sctlr...\n"); ++ iee_rwx_gate_entry(IEE_WRITE_SCTLR, read_sysreg(sctlr_el1)& ~SCTLR_ELx_M); ++ pr_info("IEE: testing iee_exec_entry ttbr0_el1...\n"); ++ iee_rwx_gate_entry(IEE_WRITE_TTBR0, read_sysreg(ttbr0_el1)); ++ pr_info("IEE: testing iee_exec_entry vbar...\n"); ++ iee_rwx_gate_entry(IEE_WRITE_VBAR, read_sysreg(vbar_el1)); ++ pr_info("IEE: testing iee_exec_entry tcr...\n"); ++ iee_rwx_gate_entry(IEE_WRITE_TCR, read_sysreg(tcr_el1)); ++ // pr_info("IEE: testing iee_exec_entry mdscr...\n"); ++ // iee_rwx_gate_entry(IEE_WRITE_MDSCR, read_sysreg(mdscr_el1)); ++ // pr_info("IEE: testing iee_exec_entry afsr0...\n"); ++ // iee_rwx_gate_entry(IEE_WRITE_AFSR0); ++ #ifdef CONFIG_KOI ++ write_sysreg(read_sysreg(ttbr0_el1)+0x3000000000000, ttbr0_el1); ++ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1)); ++ pr_info("IEE: testing iee_exec_entry switch to koi...\n"); ++ iee_rwx_gate_entry(IEE_SWITCH_TO_KOI, phys_to_ttbr(__pa_symbol(swapper_pg_dir))); ++ pr_info("IEE: current TTBR1_EL1:%llx, TTBR0:%llx\n", read_sysreg(ttbr1_el1), read_sysreg(ttbr0_el1)); ++ pr_info("IEE: testing iee_exec_entry switch to kernel...\n"); ++ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); ++ #endif ++} ++ ++/* Finish iee rwx gate initializations. */ ++static void __init iee_si_init_done(void) ++{ ++ // Prepare data for iee rwx gate ++ iee_si_prepare_data(); ++ // All initialization is done. Do some simple tests. ++ iee_si_test_end(); ++} ++#endif ++ + void __init setup_cpu_features(void) + { + u32 cwg; +@@ -3502,6 +3577,10 @@ void __init setup_cpu_features(void) + if (!cwg) + pr_warn("No Cache Writeback Granule information, assuming %d\n", + ARCH_DMA_MINALIGN); ++ ++ #ifdef CONFIG_IEE ++ iee_si_init_done(); ++ #endif + } + + static int enable_mismatched_32bit_el0(unsigned int cpu) +diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c +index 745aefddd9a3..265417e0ad81 100644 +--- a/arch/arm64/kernel/debug-monitors.c ++++ b/arch/arm64/kernel/debug-monitors.c +@@ -36,10 +36,14 @@ u8 debug_monitors_arch(void) + */ + static void mdscr_write(u32 mdscr) + { ++// #ifdef CONFIG_IEE ++// iee_rwx_gate_entry(IEE_WRITE_mdscr_el1, mdscr); ++// #else + unsigned long flags; + flags = local_daif_save(); + write_sysreg(mdscr, mdscr_el1); + local_daif_restore(flags); ++// #endif + } + NOKPROBE_SYMBOL(mdscr_write); + +diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c +index 08274e4317b2..0d259e355c90 100644 +--- a/arch/arm64/kernel/entry-common.c ++++ b/arch/arm64/kernel/entry-common.c +@@ -156,7 +156,11 @@ asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) + * mode. Before this function is called it is not safe to call regular kernel + * code, instrumentable code, or any code which may trigger an exception. + */ ++#ifdef CONFIG_IEE ++void noinstr arm64_enter_nmi(struct pt_regs *regs) ++#else + static void noinstr arm64_enter_nmi(struct pt_regs *regs) ++#endif + { + regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); + +diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S +index 7fcbee0f6c0e..dee813ee6aad 100644 +--- a/arch/arm64/kernel/entry.S ++++ b/arch/arm64/kernel/entry.S +@@ -29,12 +29,391 @@ + #include <asm/asm-uaccess.h> + #include <asm/unistd.h> + ++#ifdef CONFIG_IEE ++#include <asm/iee-def.h> ++ ++#define BAD_SP_EL0 0 ++#define BAD_ELR_EL1 1 ++#define BAD_TCR_EL1 2 ++#define BAD_IEE_SI 4 ++#endif ++ + .macro clear_gp_regs + .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29 + mov x\n, xzr + .endr + .endm + ++#ifdef CONFIG_KOI ++#ifdef CONFIG_IEE ++/* ++ * This function is used to switch to ko stack in glue code ++ */ ++SYM_FUNC_START(koi_do_switch_to_ko_stack) ++ sub sp, sp, #48 ++ stp x29, x30, [sp] ++ str x2, [sp, #16] ++ stp x0, x1, [sp, #32] ++ ++ // iee_rw_gate(IEE_WRITE_KERNEL_STACK, current, sp) ++ mov x0, #IEE_WRITE_KOI_KERNEL_STACK ++ mrs x1, sp_el0 ++ add x2, sp, #48 ++ ++ bl iee_rw_gate ++ ++ // iee_rw_gate(IEE_READ_KOI_STACK, current) ++ mov x0, #IEE_READ_KOI_STACK ++ mrs x1, sp_el0 ++ bl iee_rw_gate ++ ++ ldp x29, x30, [sp] ++ ldr x2, [sp, #16] ++ add x1, sp, #32 ++ mov sp, x0 ++ ldp x0, x1, [x1] ++ ++ isb ++ ret ++SYM_FUNC_END(koi_do_switch_to_ko_stack) ++ ++/* ++ * This fucntion is used to switch to kernel stack in glue code ++ */ ++SYM_FUNC_START(koi_do_switch_to_kernel_stack) ++ sub sp, sp, #48 ++ stp x29, x30, [sp] ++ str x2, [sp, #16] ++ stp x0, x1, [sp, #32] ++ // iee_rw_gate(IEE_WRITE_KOI_STACK, current, sp) ++ mov x0, #IEE_WRITE_KOI_STACK ++ mrs x1, sp_el0 ++ add x2, sp, #48 ++ bl iee_rw_gate ++ ++ // iee_rw_gate(IEE_READ_KOI_KERNEL_STACK, current) ++ mov x0, #IEE_READ_KOI_KERNEL_STACK ++ mrs x1, sp_el0 ++ bl iee_rw_gate ++ ++ ldp x29, x30, [sp] ++ ldr x2, [sp, #16] ++ add x1, sp, #32 ++ mov sp, x0 ++ ldp x0, x1, [x1] ++ isb ++ ret ++SYM_FUNC_END(koi_do_switch_to_kernel_stack) ++ ++/* ++ * Before switch to ko's pgtable, we must switch current stack to ko's stack. ++ * We have stored registers to kernel stack, and we need to restore them from ko's stack after switching, ++ * so we need to copy from kernel stack to ko stack ++ * the memory region to copy is [sp, stack_top) ++ * void koi_switch_to_ko_stack(void); ++ */ ++SYM_FUNC_START(koi_switch_to_ko_stack) ++ mrs x17, pan ++ msr pan, 0x0 ++ ++ sub sp, sp, #32 ++ str x17, [sp, #16] ++ stp x30, x29, [sp] ++ ++ // current sp stores in x1 ++ add x1, x1, #176 ++ // current sp_el0 stores in x0 ++ bl _iee_write_koi_kernel_stack ++ ++ mrs x0, sp_el0 ++ bl _iee_read_koi_stack ++ ++ ldr x17, [sp, #16] ++ ldp x30, x29, [sp] ++ add sp, sp, #32 ++ ++ msr pan, x17 ++ ++ sub x0, x0, #176 ++ mov x1, sp ++ mov x2, #176 ++ ++ // memcpy(current->driver_stack, current->kernel_stack, 176) ++ mov x16, lr ++ bl memcpy ++ mov lr, x16 ++ ++ mov sp, x0 ++ isb ++ ret ++SYM_FUNC_END(koi_switch_to_ko_stack) ++ ++SYM_FUNC_START(koi_switch_to_kernel_stack) ++ /* ++ * current sp belongs to driver stack, and the bottom 160 bytes saves registers when exception occurred, ++ * so we should add 160 to current sp, and store it in task_struct ++ * also, fetch kernel sp from task_struct, copy the bottom 160 bytes from driver stack to kernel stack ++ */ ++ mrs x17, pan ++ msr pan, 0x0 ++ ++ sub sp, sp, #32 ++ stp x30, x29, [sp] ++ str x17, [sp, #16] ++ ++ mrs x0, sp_el0 ++ add x1, sp, #192 ++ bl _iee_write_koi_stack ++ ++ mrs x0, sp_el0 ++ bl _iee_read_koi_kernel_stack ++ ++ ldr x17, [sp, #16] ++ ldp x30, x29, [sp] ++ add sp, sp, #32 ++ ++ msr pan, x17 ++ ++ // x0 = kernel_stack ++ sub x0, x0, #160 ++ mov x1, sp ++ // x2 = 160 ++ mov x2, #160 ++ ++ mov x16, lr ++ bl memcpy ++ mov lr, x16 ++ ++ mov sp, x0 ++ isb ++ ret ++SYM_FUNC_END(koi_switch_to_kernel_stack) ++#else ++/* ++ * This function is used to switch to ko stack in glue code ++ */ ++SYM_FUNC_START(koi_do_switch_to_ko_stack) ++ sub sp, sp, #16 ++ stp x16, x17, [sp] ++ mrs x17, sp_el0 ++ adrp x16, koi_offset ++ ldr x16, [x16, #:lo12:koi_offset] ++ add x17, x17, x16 ++ add x16, sp, #16 ++ str x16, [x17, #koi_kernel_from_token_offset] ++ ldr x16, [x17, #koi_from_token_offset] ++ mov x17, sp ++ mov sp, x16 ++ ldp x16, x17, [x17] ++ isb ++ ret ++SYM_FUNC_END(koi_do_switch_to_ko_stack) ++ ++/* ++ * This fucntion is used to switch to kernel stack in glue code ++ */ ++SYM_FUNC_START(koi_do_switch_to_kernel_stack) ++ sub sp, sp, #16 ++ stp x16, x17, [sp] ++ mrs x17, sp_el0 ++ adrp x16, koi_offset ++ ldr x16, [x16, #:lo12:koi_offset] ++ add x17, x17, x16 ++ add x16, sp, #16 ++ str x16, [x17, #koi_from_token_offset] ++ ldr x16, [x17, #koi_kernel_from_token_offset] ++ mov x17, sp ++ mov sp, x16 ++ ldp x16, x17, [x17] ++ isb ++ ret ++SYM_FUNC_END(koi_do_switch_to_kernel_stack) ++ ++/* ++ * Before switch to ko's pgtable, we must switch current stack to ko's stack. ++ * We have stored registers to kernel stack, and we need to restore them from ko's stack after switching, ++ * so we need to copy from kernel stack to ko stack ++ * the memory region to copy is [sp, stack_top) ++ * void koi_switch_to_ko_stack(unsigned long stack_top); ++ */ ++SYM_FUNC_START(koi_switch_to_ko_stack) ++ // current sp stores in x1 ++ add x3, x1, #176 ++ adrp x4, koi_offset ++ ldr x4, [x4, #:lo12:koi_offset] ++ add x4, x0, x4 ++ // current sp_el0 stores in x0 ++ str x3, [x4, #koi_kernel_from_token_offset] ++ ldr x0, [x4, #koi_from_token_offset] ++ sub x0, x0, #176 ++ mov x2, #176 ++ ++ // memcpy(current->driver_stack, current->kernel_stack, 176) ++ mov x16, lr ++ bl memcpy ++ mov lr, x16 ++ ++ mov sp, x0 ++ isb ++ ret ++SYM_FUNC_END(koi_switch_to_ko_stack) ++ ++SYM_FUNC_START(koi_switch_to_kernel_stack) ++ /* ++ * current sp belongs to driver stack, and the bottom 176 bytes saves registers when exception occurred, ++ * so we should add 176 to current sp, and store it in task_struct ++ * also, fetch kernel sp from task_struct, copy the bottom 176 bytes from driver stack to kernel stack ++ */ ++ mov x1, sp ++ add x3, sp, #160 ++ ++ mrs x16, sp_el0 ++ adrp x2, koi_offset ++ ldr x2, [x2, #:lo12:koi_offset] ++ add x16, x16, x2 ++ str x3, [x16, #koi_from_token_offset] ++ // sp points to kernel_stack ++ ldr x0, [x16, #koi_kernel_from_token_offset] ++ ++ // x0 = kernel_stack ++ sub x0, x0, #160 ++ // x2 = 160 ++ mov x2, #160 ++ mov x16, lr ++ // memcpy(kernel_stack, driver_stack, 160) ++ bl memcpy ++ mov lr, x16 ++ mov sp, x0 ++ isb ++ ret ++SYM_FUNC_END(koi_switch_to_kernel_stack) ++#endif ++ ++SYM_FUNC_START(koi_switch_to_ko_pgtbl) ++ stp x0, x1, [sp, #16 * 1] ++ stp x2, x3, [sp, #16 * 2] ++ stp x4, x5, [sp, #16 * 3] ++ stp x6, x7, [sp, #16 * 4] ++ stp x8, x9, [sp, #16 * 5] ++ stp x10, x11, [sp, #16 * 6] ++ stp x12, x13, [sp, #16 * 7] ++ stp x14, x15, [sp, #16 * 8] ++ stp x16, x17, [sp, #16 * 9] ++ stp x18, x30, [sp, #16 * 10] ++ ++ adrp x0, koi_swapper_ttbr1 ++ ldr x0, [x0, #:lo12:koi_swapper_ttbr1] ++ cbz x0, 0f ++ bl koi_do_switch_to_ko_pgtbl ++ // if x0 == 0, don't need to switch pgtable and stack, jump to 0 ++ cbz x0, 0f ++ mov x19, x0 ++ // if current on task's kernel stack， switch to ko stack ++ mrs x0, sp_el0 ++ mov x1, sp ++ ldr x2, [x0, TSK_STACK] ++ eor x2, x2, x1 ++ and x2, x2, #~(THREAD_SIZE - 1) ++ cbnz x2, 1f ++ ++ bl koi_switch_to_ko_stack ++1: ++#ifndef CONFIG_IEE ++ msr ttbr1_el1, x19 ++ isb ++ nop ++ nop ++ nop ++#else ++ mov x0, #IEE_SWITCH_TO_KOI ++ mov x1, x19 ++ bl iee_rwx_gate_entry ++#endif ++0: ++ ++ ldp x0, x1, [sp, #16 * 1] ++ ldp x2, x3, [sp, #16 * 2] ++ ldp x4, x5, [sp, #16 * 3] ++ ldp x6, x7, [sp, #16 * 4] ++ ldp x8, x9, [sp, #16 * 5] ++ ldp x10, x11, [sp, #16 * 6] ++ ldp x12, x13, [sp, #16 * 7] ++ ldp x14, x15, [sp, #16 * 8] ++ ldp x16, x17, [sp, #16 * 9] ++ ldp x18, x30, [sp, #16 * 10] ++ ret ++SYM_FUNC_END(koi_switch_to_ko_pgtbl) ++ ++.pushsection ".koi.text", "ax" ++SYM_FUNC_START(koi_switch_to_kernel_pgtbl) ++ sub sp, sp, #160 ++ stp x0, x1, [sp, #16 * 0] ++ stp x2, x3, [sp, #16 * 1] ++ stp x4, x5, [sp, #16 * 2] ++ stp x6, x7, [sp, #16 * 3] ++ stp x8, x9, [sp, #16 * 4] ++ stp x10, x11, [sp, #16 * 5] ++ ++ stp x12, x13, [sp, #16 * 6] ++ stp x14, x15, [sp, #16 * 7] ++ stp x16, x17, [sp, #16 * 8] ++ stp x18, x30, [sp, #16 * 9] ++ // check whether paging init finished ++ adrp x0, koi_swapper_ttbr1 ++ ldr x0, [x0, #:lo12:koi_swapper_ttbr1] ++ cbz x0, 0f ++ ++ bl koi_do_switch_to_kernel_pgtbl ++ /* ++ * koi_do_switch_to_kernel_pgtbl return 0 indicates ++ * that when exception occurred, the isolated ko is executing under koi pgtbl, ++ * so we need to switch stack to kernel stack after switch pgtbl back to koi_swapper_ttbr1. ++ */ ++ cbz x0, 0f ++#ifndef CONFIG_IEE ++ mrs x0, sp_el0 ++ adrp x1, koi_offset ++ ldr x1, [x1, #:lo12:koi_offset] ++ add x0, x0, x1 ++ mov x16, sp ++ ldr x17, [x0, koi_stack_base_from_token_offset] ++ eor x17, x17, x16 ++ and x17, x17, #~(THREAD_SIZE - 1) ++ cbnz x17, 0f ++#else ++ // save current pan ++ mrs x17, pan ++ // disable pan ++ msr pan, 0x0 ++ mrs x0, sp_el0 ++ bl _iee_read_koi_stack_base ++ // restore pan ++ msr pan, x17 ++ ++ mov x16, sp ++ eor x0, x0, x16 ++ and x0, x0, #~(THREAD_SIZE - 1) ++ cbnz x0, 0f ++#endif ++ bl koi_switch_to_kernel_stack ++0: ++ ++ ldp x0, x1, [sp, #16 * 0] ++ ldp x2, x3, [sp, #16 * 1] ++ ldp x4, x5, [sp, #16 * 2] ++ ldp x6, x7, [sp, #16 * 3] ++ ldp x8, x9, [sp, #16 * 4] ++ ldp x10, x11, [sp, #16 * 5] ++ ldp x12, x13, [sp, #16 * 6] ++ ldp x14, x15, [sp, #16 * 7] ++ ldp x16, x17, [sp, #16 * 8] ++ ldp x18, x30, [sp, #16 * 9] ++ add sp, sp, #160 ++ ret ++SYM_FUNC_END(koi_switch_to_kernel_pgtbl) ++.popsection ++#endif ++ + .macro kernel_ventry, el:req, ht:req, regsize:req, label:req + .align 7 + .Lventry_start\@: +@@ -151,6 +530,17 @@ alternative_else_nop_endif + #endif + .endm + ++#ifdef CONFIG_IEE ++// SP_EL0 check failed. ++SYM_FUNC_START_LOCAL(sp_el0_check_failed) ++ mov x0, sp ++ mov x1, #BAD_SP_EL0 ++ mrs x2, esr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++SYM_FUNC_END(sp_el0_check_failed) ++#endif ++ + /* Clear the MTE asynchronous tag check faults */ + .macro clear_mte_async_tcf thread_sctlr + #ifdef CONFIG_ARM64_MTE +@@ -224,6 +614,14 @@ alternative_cb_end + ldr_this_cpu tsk, __entry_task, x20 + msr sp_el0, tsk + ++#ifdef CONFIG_IEE ++ // tsk check. ++ ldr_this_cpu x19, __entry_task, x20 ++ mrs x20, sp_el0 ++ cmp x19, x20 ++ b.ne sp_el0_check_failed ++#endif ++ + /* + * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions + * when scheduling. +@@ -276,6 +674,13 @@ alternative_else_nop_endif + + scs_load_current + .else ++#ifdef CONFIG_IEE ++ // tsk check. ++ ldr_this_cpu x19, __entry_task, x20 ++ mrs x20, sp_el0 ++ cmp x19, x20 ++ b.ne sp_el0_check_failed ++#endif + add x21, sp, #PT_REGS_SIZE + get_current_task tsk + .endif /* \el == 0 */ +@@ -333,9 +738,11 @@ alternative_else_nop_endif + .endm + + .macro kernel_exit, el ++ #ifndef CONFIG_IEE + .if \el != 0 + disable_daif + .endif ++ #endif + + #ifdef CONFIG_ARM64_PSEUDO_NMI + alternative_if_not ARM64_HAS_GIC_PRIO_MASKING +@@ -411,6 +818,41 @@ alternative_else_nop_endif + + msr elr_el1, x21 // set up the return data + msr spsr_el1, x22 ++ ++#ifdef CONFIG_IEE ++ ++ .if \el == 0 ++ ++ #ifndef CONFIG_UNMAP_KERNEL_AT_EL0 ++ // SET hpd1 = 0 start ++ mrs x0, tcr_el1 ++ and x0, x0, #0xFFFFFBFFFFFFFFFF ++ and x0, x0, #0xFFFFFFFFFFBFFFFF ++ msr tcr_el1, x0 ++ // SET hpd1 = 0 end ++ ++ disable_daif ++ ++ // Check ELR_EL1 ++ mrs x0, elr_el1 ++ lsr x0, x0, #48 ++ tst x0, #0xffff ++ b.ne 5f ++ #endif ++ ++ .endif ++ ++#else ++#ifdef CONFIG_KOI ++ .if \el==0 ++ mrs x0, tcr_el1 ++ and x0, x0, #0xFFFFFFFFFFBFFFFF ++ msr tcr_el1,x0 ++ .endif ++#endif ++ ++#endif ++ + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] +@@ -569,12 +1011,167 @@ SYM_CODE_START_LOCAL(__bad_stack) + SYM_CODE_END(__bad_stack) + #endif /* CONFIG_VMAP_STACK */ + ++/* ++ * iee exception entry ++ */ ++ .macro iee_exception_entry, el ++ ++ /* Check whether exception is permmited. */ ++ ldr x1, =__iee_si_no_irq ++ cmp x1, x22 ++ b.hi 1148f ++ ldr x1, =__iee_si_end ++ cmp x1, x22 ++ b.lo 1148f ++ /* ELR check fail */ ++ mov x0, sp ++ mov x1, #BAD_IEE_SI ++ mrs x2, esr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++1148: ++ ++ /* el0 set hpds */ ++ .if \el == 0 ++ ++ #ifndef CONFIG_UNMAP_KERNEL_AT_EL0 ++ /* SET hpd1 = 1 start */ ++ mrs x0, tcr_el1 ++ orr x0, x0, #0x0000040000000000 ++ orr x0, x0, #0x0000000000400000 ++ msr tcr_el1, x0 ++ /* SET hpd1 = 1 end */ ++ ++ disable_daif ++ ++ /* Check TCR_EL1 */ ++ mrs x0, tcr_el1 ++ tst x0, #0x0000040000000000 ++ b.eq 5f ++ tst x0, #0x0000000000400000 ++ b.ne 6f ++ ++5: ++ /* TCR_EL1 check fail */ ++ mov x0, sp ++ mov x1, #BAD_TCR_EL1 ++ mrs x2, esr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++ ++6: ++ nop ++ #endif ++ ++ .else ++#ifdef CONFIG_IEE_INTERRUPTABLE ++ /* el1 save elr_el1 and set pan */ ++ /* Check ELR_EL1 */ ++ ldr x1, =__iee_code_start ++ cmp x1, x22 ++ b.hi 7f ++ ldr x1, =__iee_code_end ++ cmp x1, x22 ++ b.lo 7f ++ /* Exception from iee code */ ++ /* Switch to kernel stack */ ++ mrs x0, sp_el0 /* x0 -> task_struct(VA) */ ++ adrp x2, iee_offset ++ ldr x2, [x2, #:lo12:iee_offset] ++ add x1, x0, x2 /* x1 -> task_token(IEE) */ ++ // store iee stack ++ mov x3, sp ++ str x3, [x1, #iee_from_token_offset] ++ // load kernel stack ++ ldr x3, [x1, #kernel_from_token_offset] ++ mov sp, x3 ++ sub sp, sp, #PT_REGS_SIZE ++ /* Enable PAN */ ++ msr pan, #0x1 ++ ++7: ++ /* Exception from kernel code */ ++ mov x0, #0x0 ++ mov x1, #0x0 ++ mov x2, #0x0 ++ mov x3, #0x0 ++#endif ++ .endif ++ .endm ++ ++/* ++ * iee exception exit ++ */ ++ .macro iee_exception_exit, el ++ // Disable daif ++ disable_daif ++ ++ .if \el == 1 ++#ifdef CONFIG_IEE_INTERRUPTABLE ++ /* el1 pop elr_el1 and set pan */ ++ /* Check ELR_EL1 */ ++ ldr x1, =__iee_code_start ++ cmp x1, x22 ++ b.hi 9f ++ ldr x1, =__iee_code_end ++ cmp x1, x22 ++ b.lo 9f ++ /* Eret iee code */ ++ /* Disable PAN */ ++ msr pan, #0x0 ++ /* Switch to iee stack */ ++ add sp, sp, #PT_REGS_SIZE ++ mrs x0, sp_el0 /* x0 -> task_struct */ ++ adrp x2, iee_offset ++ ldr x2, [x2, #:lo12:iee_offset] ++ add x1, x0, x2 /* x1 -> task_token(IEE) */ ++ // store kernel stack ++ mov x3, sp ++ str x3, [x1, #kernel_from_token_offset] ++ // load iee stack ++ ldr x2, [x1, #iee_from_token_offset] ++ mov sp, x2 ++ /* Load ELR_EL1 from iee stack */ ++ ldr x21, [sp, #S_PC] ++ /* Check the modify of ELR_EL1 */ ++ cmp x21, x22 ++ b.ne 8f ++ /* ELR_EL1 not modified */ ++ b 9f ++ ++8: ++ // ELR_EL1 modified ++ mov x0, sp ++ mov x1, #BAD_ELR_EL1 ++ mrs x2, esr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++ ++9: ++ // Eret kernel code ++ mov x0, #0x0 ++ mov x1, #0x0 ++ mov x2, #0x0 ++ mov x3, #0x0 ++#endif ++ .endif ++ .endm + + .macro entry_handler el:req, ht:req, regsize:req, label:req + SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) + kernel_entry \el, \regsize ++ ++ #ifdef CONFIG_IEE ++ iee_exception_entry \el ++ #endif ++ + mov x0, sp + bl el\el\ht\()_\regsize\()_\label\()_handler ++ ++ #ifdef CONFIG_IEE ++ iee_exception_exit \el ++ #endif ++ + .if \el == 0 + b ret_to_user + .else +@@ -844,6 +1441,13 @@ SYM_FUNC_START(cpu_switch_to) + ldr lr, [x8] + mov sp, x9 + msr sp_el0, x1 ++#ifdef CONFIG_IEE ++ // tsk check. ++ ldr_this_cpu x8, __entry_task, x9 ++ mrs x9, sp_el0 ++ cmp x8, x9 ++ b.ne sp_el0_check_failed ++#endif + ptrauth_keys_install_kernel x1, x8, x9, x10 + scs_save x0 + scs_load_current +@@ -1033,6 +1637,13 @@ SYM_CODE_START(__sdei_asm_handler) + mrs x28, sp_el0 + ldr_this_cpu dst=x0, sym=__entry_task, tmp=x1 + msr sp_el0, x0 ++#ifdef CONFIG_IEE ++ // tsk check. ++ ldr_this_cpu x0, __entry_task, x1 ++ mrs x1, sp_el0 ++ cmp x0, x1 ++ b.ne sp_el0_check_failed ++#endif + + /* If we interrupted the kernel point to the previous stack/frame. */ + and x0, x3, #0xc +diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c +index 5cdfcc9e3e54..c0af965bd92e 100644 +--- a/arch/arm64/kernel/fpsimd.c ++++ b/arch/arm64/kernel/fpsimd.c +@@ -1309,7 +1309,11 @@ void sme_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p) + isb(); + + /* Allow EL0 to access TPIDR2 */ ++ #ifdef CONFIG_IEE ++ iee_rwx_gate_entry(IEE_WRITE_sctlr_el1, read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2); ++ #else + write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1); ++ #endif + isb(); + } + +diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S +index 6517bf2644a0..5c75e069d877 100644 +--- a/arch/arm64/kernel/head.S ++++ b/arch/arm64/kernel/head.S +@@ -464,6 +464,42 @@ SYM_FUNC_END(create_kernel_mapping) + set_this_cpu_offset \tmp1 + .endm + ++#ifdef CONFIG_IEE ++ .macro init_cpu_task_checked tsk, tmp1, tmp2 ++ msr sp_el0, \tsk ++ // tsk check. ++ adrp x29, __per_cpu_offset ++ mrs \tmp1, sp_el0 ++ ldr \tmp2, [\tmp1, #TSK_TI_CPU] /* cpu number */ ++1: ++ cmp \tmp2, #0 ++ b.eq 2f ++ add x29, x29, #8 ++ sub \tmp2, \tmp2, #1 ++ b 1b ++2: ++ ldr \tmp2, [x29, #:lo12:__per_cpu_offset] /* cpu offset */ ++ adr_l x29, __entry_task ++ ldr x29, [x29, \tmp2] ++ cmp x29, \tmp1 ++ b.ne sp_el0_check_failed ++ ++ ldr \tmp1, [\tsk, #TSK_STACK] ++ add sp, \tmp1, #THREAD_SIZE ++ sub sp, sp, #PT_REGS_SIZE ++ ++ stp xzr, xzr, [sp, #S_STACKFRAME] ++ add x29, sp, #S_STACKFRAME ++ ++ scs_load_current ++ ++ adr_l \tmp1, __per_cpu_offset ++ ldr w\tmp2, [\tsk, #TSK_TI_CPU] ++ ldr \tmp1, [\tmp1, \tmp2, lsl #3] ++ set_this_cpu_offset \tmp1 ++ .endm ++#endif ++ + /* + * The following fragment of code is executed with the MMU enabled. + * +@@ -661,6 +697,18 @@ SYM_FUNC_START_LOCAL(secondary_startup) + SYM_FUNC_END(secondary_startup) + + .text ++#ifdef CONFIG_IEE ++// SP_EL0 check failed. ++SYM_FUNC_START_LOCAL(sp_el0_check_failed) ++ 1: ++ nop ++ nop ++ nop ++ nop ++ b 1f ++SYM_FUNC_END(sp_el0_check_failed) ++#endif ++ + SYM_FUNC_START_LOCAL(__secondary_switched) + mov x0, x20 + bl set_cpu_boot_mode_flag +@@ -677,7 +725,11 @@ SYM_FUNC_START_LOCAL(__secondary_switched) + ldr x2, [x0, #CPU_BOOT_TASK] + cbz x2, __secondary_too_slow + ++#ifdef CONFIG_IEE ++ init_cpu_task_checked x2, x1, x3 ++#else + init_cpu_task x2, x1, x3 ++#endif + + #ifdef CONFIG_ARM64_PTR_AUTH + ptrauth_keys_init_cpu x2, x3, x4, x5 +@@ -746,6 +798,10 @@ SYM_FUNC_START(__enable_mmu) + cmp x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX + b.gt __no_granule_support + phys_to_ttbr x2, x2 ++#ifdef CONFIG_IEE ++ mov x3, #1 ++ bfi x2, x3, #48, #16 // ASID 1 is used by IEE rwx gate. ++#endif + msr ttbr0_el1, x2 // load TTBR0 + load_ttbr1 x1, x1, x3 + +diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c +index 02870beb271e..1c14428a3ed4 100644 +--- a/arch/arm64/kernel/hibernate.c ++++ b/arch/arm64/kernel/hibernate.c +@@ -34,6 +34,10 @@ + #include <asm/trans_pgd.h> + #include <asm/virt.h> + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + /* + * Hibernate core relies on this value being 0 on resume, and marks it + * __nosavedata assuming it will keep the resume kernel's '0' value. This +@@ -197,12 +201,22 @@ static int create_safe_exec_page(void *src_start, size_t length, + phys_addr_t trans_ttbr0; + unsigned long t0sz; + int rc; ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + + if (!page) + return -ENOMEM; + + memcpy(page, src_start, length); + caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length); ++ ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(page_address(page))); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)page_address(page)); ++ #endif ++ + rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page); + if (rc) + return rc; +diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c +index d39a8787edf2..b5ac4b7670bc 100644 +--- a/arch/arm64/kernel/hw_breakpoint.c ++++ b/arch/arm64/kernel/hw_breakpoint.c +@@ -26,6 +26,10 @@ + #include <asm/cputype.h> + #include <asm/system_misc.h> + ++#ifdef CONFIG_IEE ++#include <asm/iee-si.h> ++#endif ++ + /* Breakpoint currently in use for each BRP. */ + static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); + +@@ -102,13 +106,68 @@ int hw_breakpoint_slots(int type) + WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ + WRITE_WB_REG_CASE(OFF, 15, REG, VAL) + ++#ifdef CONFIG_IEE ++ ++#define IEE_SI_READ_WB_REG_CASE(OFF, N, REG, VAL) \ ++ case (OFF + N): \ ++ IEE_SI_AARCH64_DBG_READ(N, REG, VAL); \ ++ break ++ ++#define IEE_SI_WRITE_WB_REG_CASE(OFF, N, REG, VAL) \ ++ case (OFF + N): \ ++ IEE_SI_AARCH64_DBG_WRITE(N, REG, VAL); \ ++ break ++ ++#define IEE_SI_GEN_READ_REG_CASES(OFF, REG, VAL) \ ++ IEE_SI_READ_WB_REG_CASE(OFF, 0, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 15, REG, VAL) ++ ++#define IEE_SI_GEN_WRITE_REG_CASES(OFF, REG, VAL) \ ++ IEE_SI_WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ ++ WRITE_WB_REG_CASE(OFF, 15, REG, VAL) ++ ++#endif ++ + static u64 read_wb_reg(int reg, int n) + { + u64 val = 0; + + switch (reg + n) { ++// #ifdef CONFIG_IEE ++// IEE_SI_GEN_READ_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); ++// IEE_SI_GEN_READ_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); ++// #else + GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); + GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); ++// #endif + GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); + GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); + default: +@@ -122,8 +181,13 @@ NOKPROBE_SYMBOL(read_wb_reg); + static void write_wb_reg(int reg, int n, u64 val) + { + switch (reg + n) { ++// #ifdef CONFIG_IEE ++// IEE_SI_GEN_WRITE_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); ++// IEE_SI_GEN_WRITE_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); ++// #else + GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); + GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); ++// #endif + GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); + GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); + default: +@@ -171,6 +235,10 @@ static int is_a32_compat_bp(struct perf_event *bp) + return tsk && is_a32_compat_thread(task_thread_info(tsk)); + } + ++#ifdef CONFIG_IEE ++int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); ++#endif ++ + /** + * hw_breakpoint_slot_setup - Find and setup a perf slot according to + * operations +@@ -191,6 +259,37 @@ static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, + { + int i; + struct perf_event **slot; ++// reserve hw breakpoint 0 for iee rwx gate in kernel sapce. ++// #ifdef CONFIG_IEE ++// struct arch_hw_breakpoint *info = counter_arch_bp(bp); ++// if (arch_check_bp_in_kernelspace(info)){ ++// for (i = 1; i < max_slots; ++i) { // search from hw breakpoint 1 ++// slot = &slots[i]; ++// switch (ops) { ++// case HW_BREAKPOINT_INSTALL: ++// if (!*slot) { ++// *slot = bp; ++// return i; ++// } ++// break; ++// case HW_BREAKPOINT_UNINSTALL: ++// if (*slot == bp) { ++// *slot = NULL; ++// return i; ++// } ++// break; ++// case HW_BREAKPOINT_RESTORE: ++// if (*slot == bp) ++// return i; ++// break; ++// default: ++// pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); ++// return -EINVAL; ++// } ++// } ++// return -ENOSPC; ++// } ++// #endif + + for (i = 0; i < max_slots; ++i) { + slot = &slots[i]; +diff --git a/arch/arm64/kernel/iee/Makefile b/arch/arm64/kernel/iee/Makefile +new file mode 100644 +index 000000000000..123c68c5cc4e +--- /dev/null ++++ b/arch/arm64/kernel/iee/Makefile +@@ -0,0 +1 @@ ++obj-$(CONFIG_IEE) += iee.o iee-gate.o iee-func.o +\ No newline at end of file +diff --git a/arch/arm64/kernel/iee/iee-func.c b/arch/arm64/kernel/iee/iee-func.c +new file mode 100644 +index 000000000000..7764dbd41555 +--- /dev/null ++++ b/arch/arm64/kernel/iee/iee-func.c +@@ -0,0 +1,187 @@ ++#include "asm/pgtable.h" ++#include <linux/memory.h> ++#include <linux/mm.h> ++#include <asm/tlb.h> ++#include <asm/tlbflush.h> ++#include <asm/pgalloc.h> ++ ++void set_iee_page_valid(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1)))) ++ return; ++ ++ pte = __pte(pte_val(pte) | 0x1); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ isb(); ++} ++ ++void set_iee_page_invalid(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1)))) ++ return; ++ ++ pte = __pte(pte_val(pte) & ~0x1); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ isb(); ++} ++ ++void iee_set_logical_mem_ro(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if(addr < PAGE_OFFSET) ++ return; ++ ++ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ isb(); ++} ++ ++void iee_set_logical_mem_rw(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if((addr < PAGE_OFFSET) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 2)))) ++ return; ++ ++ pte = __pte(pte_val(pte) | PTE_DBM); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ isb(); ++} ++ ++void iee_set_token_page_valid(void *token, void *new) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); ++ ++ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); ++ ++ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); ++ pte_t pte = READ_ONCE(*ptep); ++ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE)); ++ isb(); ++} ++ ++void iee_set_token_page_invalid(void *token) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); ++ ++ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); ++ ++ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); ++ pte_t pte = READ_ONCE(*ptep); ++ pte = __pte(((pte_val(pte) & ~((unsigned long)0x1)) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(token - IEE_OFFSET))); ++ set_pte(ptep, pte); ++ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE)); ++ isb(); ++} ++ ++void iee_set_kernel_ppage(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ ++ int i; ++ for(i = 0; i < 4; i++) ++ { ++ pte_t pte = READ_ONCE(*ptep); ++ pte = __pte(pte_val(pte) & ~PTE_USER & ~PTE_NG); ++ iee_set_pte_ppage(ptep, pte); ++ ptep++; ++ } ++ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE); ++ isb(); ++} ++ ++void iee_set_kernel_upage(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ ++ int i; ++ for(i = 0; i < 4; i++) ++ { ++ pte_t pte = READ_ONCE(*ptep); ++ pte = __pte(pte_val(pte) | PTE_USER | PTE_NG); ++ iee_set_pte_upage(ptep, pte); ++ ptep++; ++ } ++ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE); ++ isb(); ++} +\ No newline at end of file +diff --git a/arch/arm64/kernel/iee/iee-gate.S b/arch/arm64/kernel/iee/iee-gate.S +new file mode 100644 +index 000000000000..6de99a018bde +--- /dev/null ++++ b/arch/arm64/kernel/iee/iee-gate.S +@@ -0,0 +1,174 @@ ++#include <asm/asm-offsets.h> ++#include <linux/linkage.h> ++#include <asm/bug.h> ++#include <asm-generic/export.h> ++ ++#ifdef CONFIG_IEE ++ ++SYM_FUNC_START(iee_rw_gate) ++ /* save daif, close irq */ ++ mrs x13, daif ++ msr daifset, #0x2 ++ isb ++ /* save lr */ ++ sub sp, sp, #16 ++ stp x29, x30, [sp] ++ bl iee_protected_rw_gate ++ /* restore lr */ ++ ldp x29, x30, [sp] ++ add sp, sp, #16 ++ /* restore daif */ ++ msr daif, x13 ++ ret ++SYM_FUNC_END(iee_rw_gate) ++#if defined(CONFIG_CREDP) || defined(CONFIG_KOI) ++EXPORT_SYMBOL(iee_rw_gate) ++#endif ++ ++ .pushsection ".iee.text.header", "ax" ++ ++SYM_FUNC_START(iee_protected_rw_gate) ++ mrs x9, pan ++ /* disable PAN */ ++ msr pan, #0x0 ++ /* switch to iee stack */ ++ mrs x9, sp_el0 /* x9 -> task_struct */ ++ adrp x12, iee_offset ++ ldr x12, [x12, #:lo12:iee_offset] ++ add x11, x9, x12 /* x11 -> task_token(IEE) */ ++ // store kernel stack ++ mov x10, sp ++ str x10, [x11, #kernel_from_token_offset] ++ // load iee stack ++ ldr x10, [x11, #iee_from_token_offset] ++ mov sp, x10 ++#ifdef CONFIG_IEE_INTERRUPTABLE ++ isb ++ /* restore daif */ ++ msr daif, x13 ++ sub sp, sp, #16 ++ stp x29, x30, [sp] ++#else ++ sub sp, sp, #16 ++ stp x13, x30, [sp] ++#endif ++ /* call iee func */ ++ bl iee_dispatch ++#ifdef CONFIG_IEE_INTERRUPTABLE ++ ldp x29, x30, [sp] ++ add sp, sp, #16 ++ /* store and disable daif */ ++ mrs x13, daif ++ msr daifset, #0x2 ++ isb ++#else ++ ldp x13, x30, [sp] ++ add sp, sp, #16 ++#endif ++ /* switch to kernel stack */ ++ mrs x9, sp_el0 /* x9 -> task_struct(VA) */ ++ adrp x12, iee_offset ++ ldr x12, [x12, #:lo12:iee_offset] ++ add x11, x9, x12 /* x11 -> task_token(IEE) */ ++ // store iee stack ++ mov x10, sp ++ str x10, [x11, #iee_from_token_offset] ++ // load kernel stack ++ ldr x10, [x11, #kernel_from_token_offset] ++ mov sp, x10 ++ /* enable PAN */ ++ msr pan, #0x1 ++ ret ++SYM_FUNC_END(iee_protected_rw_gate) ++ ++ .popsection ++ ++#include <asm/asm-bug.h> ++#define BAD_IEE 4 ++#define BAD_IEE_SI 5 ++ ++#define SYS_TCR_EL1_HPD1 0x40000000000 ++#define SYS_TCR_EL1_A1 0x400000 ++ ++ .pushsection ".iee.exec_entry", "ax" ++ ++SYM_FUNC_START(iee_rwx_gate_entry) ++ /* Disable irq first. */ ++ mrs x15, daif // use x15 to restore daif ++ msr DAIFSet, #0xf ++ isb ++ ++ /* Set HPD1 = 0 to exec follwing codes in U RWX page */ ++ mrs x9, tcr_el1 ++ bic x9, x9, #SYS_TCR_EL1_HPD1 ++ bic x9, x9, #SYS_TCR_EL1_A1 ++ msr tcr_el1, x9 ++ isb ++ ++ b iee_rwx_gate_tramp ++SYM_FUNC_END(iee_rwx_gate_entry) ++ .popsection ++ ++ .pushsection ".iee.si_text", "awx" ++ ++SYM_FUNC_START(iee_rwx_gate_tramp) ++ /* Check tcr val. */ ++ mrs x10, tcr_el1 ++ adrp x12, iee_si_tcr // tcr val shall be const after init ++ ldr x12, [x12, #:lo12:iee_si_tcr] ++ cbz x12, 1f ++ cmp x12, x10 ++ b.ne 3f ++1: ++ mov x13, sp ++ /* If iee hasn't been initialized, skip stack switch. */ ++ ldr x11, =iee_init_done ++ ldr x10, [x11] ++ cbz x10, 2f ++ ++ /* Switch to iee stack */ ++ mrs x9, sp_el0 // x9 -> task_struct ++ adrp x12, iee_offset ++ ldr x12, [x12, #:lo12:iee_offset] ++ add x11, x9, x12 // x11 -> task_token(IEE) ++ // load iee stack ++ ldr x10, [x11, #iee_from_token_offset] ++ mov sp, x10 ++ ++ /* x15 stores daif and x13 stores previous sp */ ++2: ++ stp x15, x13, [sp, #-32]! ++ stp x29, x30, [sp, #16] ++ bl iee_si_handler // enter actual handler ++ ldp x29, x30, [sp, #16] ++ ++ b iee_rwx_gate_exit // jump to iee exit ++3: ++ mov x0, sp ++ mov x1, #BAD_IEE_SI ++ mrs x2, esr_el1 ++ bl iee_bad_mode ++ ASM_BUG() ++SYM_FUNC_END(iee_rwx_gate_tramp) ++ ++ .popsection ++ ++ .pushsection ".iee.exec_exit", "ax" ++ ++SYM_FUNC_START(iee_rwx_gate_exit) ++ ldp x15, x13, [sp], #32 ++ mov sp, x13 // switch to kernel stack ++ mrs x9, tcr_el1 ++ orr x9, x9, #SYS_TCR_EL1_HPD1 ++ orr x9, x9, #SYS_TCR_EL1_A1 ++ msr tcr_el1, x9 ++/* --------Page boundary-------- */ ++ isb ++ msr daif, x15 ++ isb ++ ret ++SYM_FUNC_END(iee_rwx_gate_exit) ++ ++ .popsection ++ ++#endif +diff --git a/arch/arm64/kernel/iee/iee.c b/arch/arm64/kernel/iee/iee.c +new file mode 100644 +index 000000000000..6b9f7d40df67 +--- /dev/null ++++ b/arch/arm64/kernel/iee/iee.c +@@ -0,0 +1,1360 @@ ++#include "linux/sched.h" ++#include <linux/stdarg.h> ++#include <asm/pgtable-types.h> ++#include <asm/iee.h> ++#include <asm/iee-si.h> ++#include <asm/sysreg.h> ++#include <linux/pgtable.h> ++#include <linux/cred.h> ++#include <asm/iee-slab.h> ++#include <asm/percpu.h> ++ ++#ifdef CONFIG_IEE ++extern struct cred init_cred; ++extern s64 memstart_addr; ++ ++void __iee_code _iee_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd); ++void __iee_code _iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd); ++void __iee_code _iee_set_pte(pte_t *ptep, pte_t pte); ++void __iee_code _iee_set_pmd(pmd_t *pmdp, pmd_t pmd); ++void __iee_code _iee_set_pud(pud_t *pudp, pud_t pud); ++void __iee_code _iee_set_p4d(p4d_t *p4dp, p4d_t p4d); ++void __iee_code _iee_set_bm_pte(pte_t *ptep, pte_t pte); ++pteval_t __iee_code _iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval); ++pteval_t __iee_code _iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval); ++void __iee_code _iee_write_in_byte(void *ptr, __u64 data, int length); ++void __iee_code _iee_set_cred_uid(struct cred *cred, kuid_t uid); ++void __iee_code _iee_set_cred_gid(struct cred *cred, kgid_t gid); ++void __iee_code _iee_copy_cred(struct cred *old, struct cred *new); ++void __iee_code _iee_set_cred_suid(struct cred *cred, kuid_t suid); ++void __iee_code _iee_set_cred_sgid(struct cred *cred, kgid_t sgid); ++void __iee_code _iee_set_cred_euid(struct cred *cred, kuid_t euid); ++void __iee_code _iee_set_cred_egid(struct cred *cred, kgid_t egid); ++void __iee_code _iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid); ++void __iee_code _iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid); ++void __iee_code _iee_set_cred_user(struct cred *cred, struct user_struct *user); ++void __iee_code _iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns); ++void __iee_code _iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts); ++void __iee_code _iee_set_cred_group_info(struct cred *cred, struct group_info *group_info); ++void __iee_code _iee_set_cred_securebits(struct cred *cred, unsigned securebits); ++void __iee_code _iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable); ++void __iee_code _iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted); ++void __iee_code _iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective); ++void __iee_code _iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset); ++void __iee_code _iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient); ++void __iee_code _iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring); ++void __iee_code _iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring); ++void __iee_code _iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring); ++void __iee_code _iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring); ++void __iee_code _iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth); ++void __iee_code _iee_set_cred_non_rcu(struct cred *cred, int non_rcu); ++void __iee_code _iee_set_cred_atomic_set_usage(struct cred *cred, int i); ++bool __iee_code _iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr); ++void __iee_code _iee_set_cred_security(struct cred *cred, void *security); ++void __iee_code _iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu); ++void __iee_code _iee_memset(void *ptr, int data, size_t n); ++void __iee_code _iee_set_track(struct track *ptr, struct track *data); ++void __iee_code _iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr); ++void __iee_code _iee_set_pte_upage(pte_t *ptep, pte_t pte); ++void __iee_code _iee_set_pte_ppage(pte_t *ptep, pte_t pte); ++void __iee_code _iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm); ++void __iee_code _iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd); ++void __iee_code _iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack); ++void __iee_code _iee_free_token(struct task_struct *tsk); ++unsigned long __iee_code _iee_read_token_stack(struct task_struct *tsk); ++void __iee_code _iee_write_entry_task(struct task_struct *tsk); ++#ifdef CONFIG_KOI ++unsigned long __iee_code _iee_read_koi_stack(struct task_struct *tsk); ++void __iee_code _iee_write_koi_stack(struct task_struct *tsk, unsigned long koi_stack); ++unsigned long __iee_code _iee_read_token_ttbr1(struct task_struct *tsk); ++void __iee_code _iee_write_token_ttbr1(struct task_struct *tsk, unsigned long current_ttbr1); ++unsigned long __iee_code _iee_read_koi_kernel_stack(struct task_struct *tsk); ++void __iee_code _iee_write_koi_kernel_stack(struct task_struct *tsk, unsigned long kernel_stack); ++unsigned long __iee_code _iee_read_koi_stack_base(struct task_struct *tsk); ++void __iee_code _iee_write_koi_stack_base(struct task_struct *tsk, unsigned long koi_stack_base); ++#endif ++ ++/* wrapper functions */ ++void __iee_code iee_wrapper_write_in_byte(va_list args) { ++ void *ptr = va_arg(args, void *); ++ __u64 data = va_arg(args, __u64); ++ int length = va_arg(args, int); ++ _iee_write_in_byte(ptr, data, length); ++} ++ ++void __iee_code iee_wrapper_set_pte(va_list args) { ++ pte_t *ptep = va_arg(args, pte_t *); ++ pte_t pte = va_arg(args, pte_t); ++ _iee_set_pte(ptep, pte); ++} ++ ++void __iee_code iee_wrapper_set_pmd(va_list args) { ++ pmd_t *pmdp = va_arg(args, pmd_t *); ++ pmd_t pmd = va_arg(args, pmd_t); ++ _iee_set_pmd(pmdp, pmd); ++} ++ ++void __iee_code iee_wrapper_set_p4d(va_list args) { ++ p4d_t *p4dp = va_arg(args, p4d_t *); ++ p4d_t p4d = va_arg(args, p4d_t); ++ _iee_set_p4d(p4dp, p4d); ++} ++ ++void __iee_code iee_wrapper_set_pud(va_list args) { ++ pud_t *pudp = va_arg(args, pud_t *); ++ pud_t pud = va_arg(args, pud_t); ++ _iee_set_pud(pudp, pud); ++} ++ ++void __iee_code iee_wrapper_set_bm_pte(va_list args) { ++ pte_t *ptep = va_arg(args, pte_t *); ++ pte_t pte = va_arg(args, pte_t); ++ _iee_set_bm_pte(ptep, pte); ++} ++ ++void __iee_code iee_wrapper_set_swapper_pgd(va_list args) { ++ pgd_t *pgdp = va_arg(args, pgd_t *); ++ pgd_t pgd = va_arg(args, pgd_t); ++ _iee_set_swapper_pgd(pgdp, pgd); ++} ++ ++void __iee_code iee_wrapper_set_tramp_pgd(va_list args) { ++ pgd_t *pgdp = va_arg(args, pgd_t *); ++ pgd_t pgd = va_arg(args, pgd_t); ++ _iee_set_tramp_pgd_pre_init(pgdp, pgd); ++} ++ ++pteval_t __iee_code iee_wrapper_set_xchg(va_list args) { ++ pteval_t ret; ++ pte_t *ptep = va_arg(args, pte_t *); ++ pteval_t pteval = va_arg(args, pteval_t); ++ ret = _iee_set_xchg_relaxed(ptep, pteval); ++ return (u64)ret; ++} ++ ++pteval_t __iee_code iee_wrapper_set_cmpxchg(va_list args) { ++ pteval_t ret; ++ pte_t *ptep = va_arg(args, pte_t *); ++ pteval_t old_pteval = va_arg(args, pteval_t); ++ pteval_t new_pteval = va_arg(args, pteval_t); ++ ret = _iee_set_cmpxchg_relaxed(ptep, old_pteval, new_pteval); ++ return (u64)ret; ++} ++ ++void __iee_code iee_wrapper_set_cred_uid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kuid_t uid = va_arg(args, kuid_t); ++ _iee_set_cred_uid(cred, uid); ++} ++ ++void __iee_code iee_wrapper_set_cred_gid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kgid_t gid = va_arg(args, kgid_t); ++ _iee_set_cred_gid(cred, gid); ++} ++ ++void __iee_code iee_wrapper_copy_cred(va_list args) { ++ struct cred *old = va_arg(args, struct cred *); ++ struct cred *new = va_arg(args, struct cred *); ++ _iee_copy_cred(old, new); ++} ++ ++void __iee_code iee_wrapper_set_cred_suid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kuid_t suid = va_arg(args, kuid_t); ++ _iee_set_cred_suid(cred, suid); ++} ++ ++void __iee_code iee_wrapper_set_cred_sgid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kgid_t sgid = va_arg(args, kgid_t); ++ _iee_set_cred_sgid(cred, sgid); ++} ++ ++void __iee_code iee_wrapper_set_cred_euid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kuid_t euid = va_arg(args, kuid_t); ++ _iee_set_cred_euid(cred, euid); ++} ++ ++void __iee_code iee_wrapper_set_cred_egid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kgid_t egid = va_arg(args, kgid_t); ++ _iee_set_cred_egid(cred, egid); ++} ++ ++void __iee_code iee_wrapper_set_cred_fsuid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kuid_t fsuid = va_arg(args, kuid_t); ++ _iee_set_cred_fsuid(cred, fsuid); ++} ++ ++void __iee_code iee_wrapper_set_cred_fsgid(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kgid_t fsgid = va_arg(args, kgid_t); ++ _iee_set_cred_fsgid(cred, fsgid); ++} ++ ++void __iee_code iee_wrapper_set_cred_user(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct user_struct *user = va_arg(args, struct user_struct *); ++ _iee_set_cred_user(cred, user); ++} ++ ++void __iee_code iee_wrapper_set_cred_user_ns(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct user_namespace *user_ns = va_arg(args, struct user_namespace *); ++ _iee_set_cred_user_ns(cred, user_ns); ++} ++ ++void __iee_code iee_wrapper_set_cred_ucounts(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct ucounts *ucounts = va_arg(args, struct ucounts *); ++ _iee_set_cred_ucounts(cred, ucounts); ++} ++ ++void __iee_code iee_wrapper_set_cred_group_info(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct group_info *group_info = va_arg(args, struct group_info *); ++ _iee_set_cred_group_info(cred, group_info); ++} ++ ++void __iee_code iee_wrapper_set_cred_securebits(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ unsigned securebits = va_arg(args, unsigned); ++ _iee_set_cred_securebits(cred, securebits); ++} ++ ++void __iee_code iee_wrapper_set_cred_cap_inheritable(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kernel_cap_t cap_inheritable = va_arg(args, kernel_cap_t); ++ _iee_set_cred_cap_inheritable(cred, cap_inheritable); ++} ++ ++void __iee_code iee_wrapper_set_cred_cap_permitted(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kernel_cap_t cap_permitted = va_arg(args, kernel_cap_t); ++ _iee_set_cred_cap_permitted(cred, cap_permitted); ++} ++ ++void __iee_code iee_wrapper_set_cred_cap_effective(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kernel_cap_t cap_effective = va_arg(args, kernel_cap_t); ++ _iee_set_cred_cap_effective(cred, cap_effective); ++} ++ ++void __iee_code iee_wrapper_set_cred_cap_bset(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kernel_cap_t cap_bset = va_arg(args, kernel_cap_t); ++ _iee_set_cred_cap_bset(cred, cap_bset); ++} ++ ++void __iee_code iee_wrapper_set_cred_cap_ambient(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ kernel_cap_t cap_ambient = va_arg(args, kernel_cap_t); ++ _iee_set_cred_cap_ambient(cred, cap_ambient); ++} ++ ++void __iee_code iee_wrapper_set_cred_jit_keyring(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ unsigned long jit_keyring = va_arg(args, unsigned long); ++ _iee_set_cred_jit_keyring(cred, (unsigned char)jit_keyring); ++} ++ ++void __iee_code iee_wrapper_set_cred_session_keyring(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct key *session_keyring = va_arg(args, struct key *); ++ _iee_set_cred_session_keyring(cred, session_keyring); ++} ++ ++void __iee_code iee_wrapper_set_cred_process_keyring(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct key *process_keyring = va_arg(args, struct key *); ++ _iee_set_cred_process_keyring(cred, process_keyring); ++} ++ ++void __iee_code iee_wrapper_set_cred_thread_keyring(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct key *thread_keyring = va_arg(args, struct key *); ++ _iee_set_cred_thread_keyring(cred, thread_keyring); ++} ++ ++void __iee_code iee_wrapper_set_cred_request_key_auth(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct key *request_key_auth = va_arg(args, struct key *); ++ _iee_set_cred_request_key_auth(cred, request_key_auth); ++} ++ ++void __iee_code iee_wrapper_set_cred_non_rcu(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ int non_rcu = va_arg(args, int); ++ _iee_set_cred_non_rcu(cred, non_rcu); ++} ++ ++void __iee_code iee_wrapper_set_cred_atomic_set_usage(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ int i = va_arg(args, int); ++ _iee_set_cred_atomic_set_usage(cred, i); ++} ++ ++u64 __iee_code iee_wrapper_set_cred_atomic_op_usage(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ int flag = va_arg(args, int); ++ int nr = va_arg(args, int); ++ return (u64)_iee_set_cred_atomic_op_usage(cred, flag, nr); ++} ++ ++void __iee_code iee_wrapper_set_cred_security(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ void *security = va_arg(args, void *); ++ _iee_set_cred_security(cred, security); ++} ++ ++void __iee_code iee_wrapper_set_cred_rcu(va_list args) { ++ struct cred *cred = va_arg(args, struct cred *); ++ struct rcu_head *rcu = va_arg(args, struct rcu_head *); ++ _iee_set_cred_rcu(cred, rcu); ++} ++ ++void __iee_code iee_wrapper_memset(va_list args) { ++ void *ptr = va_arg(args, void *); ++ int data = va_arg(args, int); ++ size_t n = va_arg(args, size_t); ++ _iee_memset(ptr, data, n); ++} ++ ++void __iee_code iee_wrapper_set_track(va_list args) { ++ struct track *ptr = va_arg(args, struct track *); ++ struct track *data = va_arg(args, struct track *); ++ _iee_set_track(ptr, data); ++} ++ ++void __iee_code iee_wrapper_set_freeptr(va_list args) { ++ freeptr_t *pptr = va_arg(args, freeptr_t *); ++ freeptr_t ptr = va_arg(args, freeptr_t); ++ _iee_set_freeptr(pptr, ptr); ++} ++ ++void __iee_code iee_wrapper_set_pte_upage(va_list args) { ++ pte_t *ptep = va_arg(args, pte_t *); ++ pte_t pte = va_arg(args, pte_t); ++ _iee_set_pte_upage(ptep, pte); ++} ++ ++void __iee_code iee_wrapper_set_pte_ppage(va_list args) { ++ pte_t *ptep = va_arg(args, pte_t *); ++ pte_t pte = va_arg(args, pte_t); ++ _iee_set_pte_ppage(ptep, pte); ++} ++ ++void __iee_code iee_wrapper_set_token_mm(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ struct mm_struct *mm = va_arg(args, struct mm_struct *); ++ _iee_set_token_mm(tsk, mm); ++} ++ ++void __iee_code iee_wrapper_set_token_pgd(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ pgd_t *pgd = va_arg(args, pgd_t *); ++ _iee_set_token_pgd(tsk, pgd); ++} ++ ++void __iee_code iee_wrapper_init_token(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ void *kernel_stack = va_arg(args, void *); ++ void *iee_stack = va_arg(args, void *); ++ _iee_init_token(tsk, kernel_stack, iee_stack); ++} ++ ++void __iee_code iee_wrapper_free_token(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ _iee_free_token(tsk); ++} ++ ++u64 __iee_code iee_wrapper_read_token_stack(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ return (u64)_iee_read_token_stack(tsk); ++} ++ ++void __iee_code iee_wrapper_write_entry_task(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ _iee_write_entry_task(tsk); ++} ++ ++#ifdef CONFIG_KOI ++u64 __iee_code iee_wrapper_read_koi_stack(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ return (u64)_iee_read_koi_stack(tsk); ++} ++ ++void __iee_code iee_wrapper_write_koi_stack(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ unsigned long koi_stack = va_arg(args, unsigned long); ++ _iee_write_koi_stack(tsk, koi_stack); ++} ++ ++u64 __iee_code iee_wrapper_read_token_ttbr1(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ return (u64)_iee_read_token_ttbr1(tsk); ++} ++ ++void __iee_code iee_wrapper_write_token_ttbr1(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ unsigned long current_ttbr1 = va_arg(args, unsigned long); ++ _iee_write_token_ttbr1(tsk, current_ttbr1); ++} ++ ++u64 __iee_code iee_wrapper_read_koi_kernel_stack(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ return (u64)_iee_read_koi_kernel_stack(tsk); ++} ++ ++void __iee_code iee_wrapper_write_koi_kernel_stack(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ unsigned long kernel_stack = va_arg(args, unsigned long); ++ _iee_write_koi_kernel_stack(tsk, kernel_stack); ++} ++ ++u64 __iee_code iee_wrapper_read_koi_stack_base(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ return (u64)_iee_read_koi_stack_base(tsk); ++} ++ ++void __iee_code iee_wrapper_write_koi_stack_base(va_list args) { ++ struct task_struct *tsk = va_arg(args, struct task_struct *); ++ unsigned long koi_stack_base = va_arg(args, unsigned long); ++ _iee_write_koi_stack_base(tsk, koi_stack_base); ++} ++#endif ++// Define the function pointer type for wrapper functions. ++// Each function pointer conforms to a standardized calling convention ++// using a variable argument list (va_list) as its parameter. ++// This allows dynamic invocation of different functions with various arguments. ++typedef void (*iee_wrapper_func)(va_list args); ++iee_wrapper_func iee_wrappers[] = { ++ iee_wrapper_write_in_byte, ++ iee_wrapper_set_pte, ++ iee_wrapper_set_pmd, ++ iee_wrapper_set_pud, ++ iee_wrapper_set_p4d, ++ iee_wrapper_set_bm_pte, ++ iee_wrapper_set_swapper_pgd, ++ iee_wrapper_set_tramp_pgd, ++ (iee_wrapper_func)iee_wrapper_set_cmpxchg, ++ (iee_wrapper_func)iee_wrapper_set_xchg, ++ iee_wrapper_copy_cred, ++ iee_wrapper_set_cred_uid, ++ iee_wrapper_set_cred_gid, ++ iee_wrapper_set_cred_suid, ++ iee_wrapper_set_cred_sgid, ++ iee_wrapper_set_cred_euid, ++ iee_wrapper_set_cred_egid, ++ iee_wrapper_set_cred_fsuid, ++ iee_wrapper_set_cred_fsgid, ++ iee_wrapper_set_cred_user, ++ iee_wrapper_set_cred_user_ns, ++ iee_wrapper_set_cred_group_info, ++ iee_wrapper_set_cred_securebits, ++ iee_wrapper_set_cred_cap_inheritable, ++ iee_wrapper_set_cred_cap_permitted, ++ iee_wrapper_set_cred_cap_effective, ++ iee_wrapper_set_cred_cap_bset, ++ iee_wrapper_set_cred_cap_ambient, ++ iee_wrapper_set_cred_jit_keyring, ++ iee_wrapper_set_cred_session_keyring, ++ iee_wrapper_set_cred_process_keyring, ++ iee_wrapper_set_cred_thread_keyring, ++ iee_wrapper_set_cred_request_key_auth, ++ iee_wrapper_set_cred_non_rcu, ++ iee_wrapper_set_cred_atomic_set_usage, ++ (iee_wrapper_func)iee_wrapper_set_cred_atomic_op_usage, ++ iee_wrapper_set_cred_security, ++ iee_wrapper_set_cred_rcu, ++ iee_wrapper_memset, ++ iee_wrapper_set_track, ++ iee_wrapper_set_freeptr, ++ iee_wrapper_set_pte_upage, ++ iee_wrapper_set_pte_ppage, ++ iee_wrapper_set_token_mm, ++ iee_wrapper_set_token_pgd, ++ iee_wrapper_init_token, ++ iee_wrapper_free_token, ++ (iee_wrapper_func)iee_wrapper_read_token_stack, ++ iee_wrapper_write_entry_task, ++ iee_wrapper_set_cred_ucounts, ++#ifdef CONFIG_KOI ++ (iee_wrapper_func)iee_wrapper_read_koi_stack, ++ iee_wrapper_write_koi_stack, ++ (iee_wrapper_func)iee_wrapper_read_token_ttbr1, ++ iee_wrapper_write_token_ttbr1, ++ (iee_wrapper_func)iee_wrapper_read_koi_kernel_stack, ++ iee_wrapper_write_koi_kernel_stack, ++ (iee_wrapper_func)iee_wrapper_read_koi_stack_base, ++ iee_wrapper_write_koi_stack_base ++#endif ++}; ++ ++u64 __iee_code iee_dispatch(int flag, ...){ ++ va_list pArgs; ++ ++ va_start(pArgs, flag); ++ ++ switch(flag) ++ { ++ case IEE_OP_SET_CMPXCHG: ++ { ++ pteval_t ret = iee_wrapper_set_cmpxchg(pArgs); ++ va_end(pArgs); ++ return (u64)ret; ++ } ++ case IEE_OP_SET_XCHG: ++ { ++ pteval_t ret = iee_wrapper_set_xchg(pArgs); ++ va_end(pArgs); ++ return (u64)ret; ++ } ++ case IEE_OP_SET_CRED_ATOP_USAGE: ++ { ++ u64 ret = iee_wrapper_set_cred_atomic_op_usage(pArgs); ++ va_end(pArgs); ++ return ret; ++ } ++ case IEE_READ_TOKEN_STACK: ++ { ++ u64 ret = iee_wrapper_read_token_stack(pArgs); ++ va_end(pArgs); ++ return ret; ++ } ++#ifdef CONFIG_KOI ++ case IEE_READ_KOI_STACK: ++ { ++ u64 ret = iee_wrapper_read_koi_stack(pArgs); ++ va_end(pArgs); ++ return ret; ++ } ++ case IEE_READ_TOKEN_TTBR1: ++ { ++ u64 ret = iee_wrapper_read_token_ttbr1(pArgs); ++ va_end(pArgs); ++ return ret; ++ } ++ case IEE_READ_KOI_KERNEL_STACK: ++ { ++ u64 ret = iee_wrapper_read_koi_kernel_stack(pArgs); ++ va_end(pArgs); ++ return ret; ++ } ++ case IEE_READ_KOI_STACK_BASE: ++ { ++ u64 ret = iee_wrapper_read_koi_stack_base(pArgs); ++ va_end(pArgs); ++ return ret; ++ } ++#endif ++ default: ++ { ++ #ifndef CONFIG_KOI ++ if((flag < IEE_WRITE_IN_BYTE) | (flag > IEE_OP_SET_CRED_UCOUNTS)) ++ panic("Invalid iee flag.\n"); ++ #else ++ if((flag < IEE_WRITE_IN_BYTE) | (flag > IEE_WRITE_KOI_STACK_BASE)) ++ panic("Invalid iee flag.\n"); ++ #endif ++ iee_wrappers[flag](pArgs); ++ break; ++ } ++ } ++ ++ va_end(pArgs); ++ return 0; ++} ++ ++#ifdef CONFIG_KOI ++unsigned long __iee_code _iee_read_koi_stack(struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ return (unsigned long)token->koi_stack; ++} ++ ++void __iee_code _iee_write_koi_stack(struct task_struct *tsk, unsigned long koi_stack) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->koi_stack = koi_stack; ++} ++ ++unsigned long __iee_code _iee_read_token_ttbr1(struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ return token->current_ttbr1; ++} ++ ++void __iee_code _iee_write_token_ttbr1(struct task_struct *tsk, unsigned long current_ttbr1) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->current_ttbr1 = current_ttbr1; ++} ++ ++unsigned long __iee_code _iee_read_koi_kernel_stack(struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ return token->koi_kernel_stack; ++} ++ ++void __iee_code _iee_write_koi_kernel_stack(struct task_struct *tsk, unsigned long kernel_stack) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->koi_kernel_stack = kernel_stack; ++} ++ ++unsigned long __iee_code _iee_read_koi_stack_base(struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ return (unsigned long)token->koi_stack_base; ++} ++ ++void __iee_code _iee_write_koi_stack_base(struct task_struct *tsk, unsigned long koi_stack_base) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->koi_stack_base = koi_stack_base; ++} ++#endif ++ ++// Protect the __entry_task. ++__attribute__((aligned(PAGE_SIZE))) DECLARE_PER_CPU(struct task_struct *[PAGE_SIZE/sizeof(struct task_struct *)], __entry_task); ++void __iee_code _iee_write_entry_task(struct task_struct *tsk) ++{ ++ // Add check of tsk. ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ ++ unsigned long flags; ++ unsigned long res; ++ struct task_struct **entry_addr; ++ local_irq_save(flags); ++ asm volatile("at s1e1r, %0"::"r"(token)); ++ isb(); ++ res = read_sysreg(par_el1); ++ local_irq_restore(flags); ++ ++ // If it is logical map, that means it is not a token. ++ if(__phys_to_iee(res & PTE_ADDR_MASK) == (((unsigned long)token) & PTE_ADDR_MASK)) ++ panic("Trying to forge a token.\n"); ++ ++ if(!token->valid) ++ panic("Trying to write a wrong task into __entry_task.\n"); ++ entry_addr = (struct task_struct **)__phys_to_iee(__pa(SHIFT_PERCPU_PTR(__entry_task,__kern_my_cpu_offset()))); ++ *entry_addr = tsk; ++} ++ ++unsigned long __iee_code _iee_read_token_stack(struct task_struct *tsk) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ return (unsigned long)token->iee_stack; ++} ++ ++void __iee_code _iee_free_token(struct task_struct *tsk) ++{ ++ _iee_memset(tsk, 0, sizeof(struct task_token)); ++} ++ ++#ifdef CONFIG_KOI ++extern unsigned long koi_swapper_ttbr1; ++#endif ++void __iee_code _iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->kernel_stack = kernel_stack; ++ token->iee_stack = iee_stack; ++ token->valid = true; ++#ifdef CONFIG_KOI ++ token->koi_kernel_stack = NULL; ++ token->koi_stack = NULL; ++ token->koi_stack_base = NULL; ++ token->current_ttbr1 = 0; ++#endif ++} ++ ++void __iee_code _iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->mm = mm; ++} ++ ++void __iee_code _iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd) ++{ ++ struct task_token *token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ token->pgd = pgd; ++} ++ ++void __iee_code _iee_set_freeptr(freeptr_t *pptr, freeptr_t ptr) ++{ ++ pptr = (freeptr_t *)__phys_to_iee(__pa(pptr)); ++ *pptr = ptr; ++} ++ ++#pragma GCC push_options ++#pragma GCC optimize("O0") ++void __iee_code _iee_memset(void *ptr, int data, size_t n) ++{ ++ char *_ptr = (char *)__phys_to_iee(__pa(ptr)); ++ ++ while (n--) ++ *_ptr++ = data; ++} ++ ++void __iee_code _iee_memcpy(void *dst, void *src, size_t n) ++{ ++ char *_dst = (char *)__phys_to_iee(__pa(dst)); ++ char *_src = (char *)src; ++ ++ while(n--) ++ *_dst++ = *_src++; ++} ++#pragma GCC pop_options ++ ++void __iee_code _iee_set_track(struct track *ptr, struct track *data) ++{ ++ _iee_memcpy(ptr, data, sizeof(struct track)); ++} ++ ++void __iee_code _iee_set_cred_rcu(struct cred *cred, struct rcu_head *rcu) ++{ ++ if(cred == &init_cred) ++ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred)); ++ else ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ #ifdef CONFIG_CREDP ++ *((struct rcu_head **)(&(cred->rcu.func))) = rcu; ++ #endif ++} ++ ++void __iee_code _iee_set_cred_security(struct cred *cred, void *security) ++{ ++ if(cred == &init_cred) ++ cred = (struct cred *)__phys_to_iee(__pa_symbol(cred)); ++ else ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->security = security; ++} ++ ++bool __iee_code _iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ switch (flag) ++ { ++ case AT_ADD: { ++ atomic_long_add(nr, &cred->usage); ++ return 0; ++ } ++ case AT_INC_NOT_ZERO: { ++ return atomic_long_inc_not_zero(&cred->usage); ++ } ++ case AT_SUB_AND_TEST: { ++ return atomic_long_sub_and_test(nr, &cred->usage); ++ } ++ } ++ return 0; ++} ++ ++void __iee_code _iee_set_cred_atomic_set_usage(struct cred *cred, int i) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ atomic_long_set(&cred->usage,i); ++} ++ ++void __iee_code _iee_set_cred_non_rcu(struct cred *cred, int non_rcu) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->non_rcu = non_rcu; ++} ++ ++void __iee_code _iee_set_cred_session_keyring(struct cred *cred, struct key *session_keyring) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->session_keyring = session_keyring; ++} ++ ++void __iee_code _iee_set_cred_process_keyring(struct cred *cred, struct key *process_keyring) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->process_keyring = process_keyring; ++} ++ ++void __iee_code _iee_set_cred_thread_keyring(struct cred *cred, struct key *thread_keyring) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->thread_keyring = thread_keyring; ++} ++ ++void __iee_code _iee_set_cred_request_key_auth(struct cred *cred, struct key *request_key_auth) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->request_key_auth = request_key_auth; ++} ++ ++void __iee_code _iee_set_cred_jit_keyring(struct cred *cred, unsigned char jit_keyring) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->jit_keyring = jit_keyring; ++} ++ ++void __iee_code _iee_set_cred_cap_inheritable(struct cred *cred, kernel_cap_t cap_inheritable) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->cap_inheritable = cap_inheritable; ++} ++ ++void __iee_code _iee_set_cred_cap_permitted(struct cred *cred, kernel_cap_t cap_permitted) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->cap_permitted = cap_permitted; ++} ++ ++void __iee_code _iee_set_cred_cap_effective(struct cred *cred, kernel_cap_t cap_effective) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->cap_effective = cap_effective; ++} ++ ++void __iee_code _iee_set_cred_cap_bset(struct cred *cred, kernel_cap_t cap_bset) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->cap_bset = cap_bset; ++} ++ ++void __iee_code _iee_set_cred_cap_ambient(struct cred *cred, kernel_cap_t cap_ambient) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->cap_ambient = cap_ambient; ++} ++ ++void __iee_code _iee_set_cred_securebits(struct cred *cred, unsigned securebits) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->securebits = securebits; ++} ++ ++void __iee_code _iee_set_cred_group_info(struct cred *cred, struct group_info *group_info) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->group_info = group_info; ++} ++ ++void __iee_code _iee_set_cred_ucounts(struct cred *cred, struct ucounts *ucounts) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->ucounts = ucounts; ++} ++ ++void __iee_code _iee_set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->user_ns = user_ns; ++} ++ ++void __iee_code _iee_set_cred_user(struct cred *cred, struct user_struct *user) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->user = user; ++} ++ ++void __iee_code _iee_set_cred_fsgid(struct cred *cred, kgid_t fsgid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->fsgid = fsgid; ++} ++ ++void __iee_code _iee_set_cred_fsuid(struct cred *cred, kuid_t fsuid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->fsuid = fsuid; ++} ++ ++void __iee_code _iee_set_cred_egid(struct cred *cred, kgid_t egid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->egid = egid; ++} ++ ++void __iee_code _iee_set_cred_euid(struct cred *cred, kuid_t euid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->euid = euid; ++} ++ ++void __iee_code _iee_set_cred_sgid(struct cred *cred, kgid_t sgid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->sgid = sgid; ++} ++ ++void __iee_code _iee_set_cred_suid(struct cred *cred, kuid_t suid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->suid = suid; ++} ++ ++void __iee_code _iee_copy_cred(struct cred *old, struct cred *new) ++{ ++ #ifdef CONFIG_CREDP ++ struct rcu_head *rcu = (struct rcu_head *)(new->rcu.func); ++ struct cred *_new = (struct cred *)__phys_to_iee(__pa(new)); ++ _iee_memcpy(new,old,sizeof(struct cred)); ++ *(struct rcu_head **)(&(_new->rcu.func)) = rcu; ++ *(struct rcu_head *)(_new->rcu.func) = *(struct rcu_head *)(old->rcu.func); ++ #endif ++} ++ ++void __iee_code _iee_set_cred_gid(struct cred *cred, kgid_t gid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->gid = gid; ++} ++ ++void __iee_code _iee_set_cred_uid(struct cred *cred, kuid_t uid) ++{ ++ cred = (struct cred *)__phys_to_iee(__pa(cred)); ++ cred->uid = uid; ++} ++ ++void __iee_code _iee_write_in_byte(void *ptr, __u64 data, int length) ++{ ++ ptr = (void *)__phys_to_iee(__pa(ptr)); ++ switch(length) { ++ case 8: { ++ *(__u64 *)ptr = data; ++ break; ++ } ++ case 4: { ++ *(__u32 *)ptr = (__u32)data; ++ break; ++ } ++ case 2: { ++ *(__u16 *)ptr = (__u16)data; ++ break; ++ } ++ case 1: { ++ *(__u8 *)ptr = (__u8)data; ++ break; ++ } ++ } ++} ++ ++pteval_t __iee_code _iee_set_xchg_relaxed(pte_t *ptep, pteval_t pteval) ++{ ++ pteval_t ret = xchg_relaxed((pteval_t *)(__phys_to_iee(__pa(ptep))), pteval); ++ return ret; ++} ++ ++pteval_t __iee_code _iee_set_cmpxchg_relaxed(pte_t *ptep, pteval_t old_pteval, pteval_t new_pteval) ++{ ++ pteval_t pteval = cmpxchg_relaxed((pteval_t *)(__phys_to_iee(__pa(ptep))), old_pteval, new_pteval); ++ return pteval; ++} ++ ++/* Check if addr is allocated in IEE page */ ++static inline bool check_addr_in_iee_valid(unsigned long addr) ++{ ++ unsigned long flags; ++ unsigned long res; ++ local_irq_save(flags); ++ asm volatile("at s1e1r, %0"::"r"(addr)); ++ isb(); ++ res = read_sysreg(par_el1); ++ local_irq_restore(flags); ++ ++ // If it is not logical map, that means it is a token. ++ if(__phys_to_iee(res & PTE_ADDR_MASK) != addr) ++ return false; ++ ++ return !(res & 0x1); ++} ++ ++void __iee_code _iee_set_tramp_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) ++{ ++ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); ++} ++ ++void __iee_code _iee_set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) ++{ ++ if(!(pgd_val(pgd) & PMD_SECT_VALID)) ++ { ++ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); ++ return; ++ } ++ ++ if ((pgd_val(pgd) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pgd_to_phys(pgd)))) ++ panic("You can't use non-iee-pgtable\n"); ++ ++ if((pgdp >= pgd_offset_pgd((pgd_t *)swapper_pg_dir, PAGE_OFFSET + BIT(vabits_actual - 2))) && (pgdp < pgd_offset_pgd((pgd_t *)swapper_pg_dir, PAGE_OFFSET + BIT(vabits_actual - 1))) && !(pgd_val(pgd) & PGD_APT)) ++ panic("Set IEE pgd U page.\n"); ++ ++ WRITE_ONCE(*((pgd_t *)(__phys_to_iee(__pa_symbol(pgdp)))), pgd); ++} ++ ++void __iee_code _iee_set_p4d(p4d_t *p4dp, p4d_t p4d) ++{ ++ if(!(p4d_val(p4d) & PMD_SECT_VALID)) ++ { ++ WRITE_ONCE(*((p4d_t *)(__phys_to_iee(__pa(p4dp)))), p4d); ++ return; ++ } ++ ++ if ((p4d_val(p4d) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__p4d_to_phys(p4d)))) ++ panic("You can't use non-iee-pgtable\n"); ++ ++ WRITE_ONCE(*((p4d_t *)(__phys_to_iee(__pa(p4dp)))), p4d); ++} ++ ++void __iee_code _iee_set_pud(pud_t *pudp, pud_t pud) ++{ ++ if(!(pud_val(pud) & PMD_SECT_VALID)) ++ { ++ WRITE_ONCE(*((pud_t *)(__phys_to_iee(__pa(pudp)))), pud); ++ return; ++ } ++ ++ if ((pud_val(pud) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pud_to_phys(pud)))) ++ panic("You can't use non-iee-pgtable\n"); ++ ++ WRITE_ONCE(*((pud_t *)(__phys_to_iee(__pa(pudp)))), pud); ++} ++ ++// Return true if the modify does not break DEP. ++static inline bool check_pmd_dep(char *addr, pmd_t pmd) ++{ ++ // DEP for kernel code and readonly data ++ // _text: .text start addr, __init_begin: .rodata end addr ++ if (addr >= _stext && addr < _etext) ++ { ++ if ((PTE_WRITE & pmd_val(pmd)) || // DBM == 1 --> writable ++ !(PTE_RDONLY & pmd_val(pmd))) // DBM == 0 && AP[2] = 0 --> writable ++ { ++ panic("Can't make kernel's text/readonly page as writable!\n" ++ "addr = 0x%16llx, pmd_val = 0x%16llx", ++ (u64)addr, pmd_val(pmd)); ++ } ++ } ++ return true; ++} ++ ++// Return true if the pmd table is a part of kernel page table. ++// TODO : Optimize to get lower overhead. ++static inline bool is_kernel_pmd_table(pmd_t *pmdp, pmd_t pmd) ++{ ++ int i = 0,j = 0; ++ for(i = 0; i < PAGE_SIZE/sizeof(pgd_t); i++) ++ { ++ pgd_t *pgdp = (pgd_t *)swapper_pg_dir + i; ++ if((pgd_val(*pgdp) & PMD_SECT_VALID) && (pgd_val(*pgdp) & PMD_TABLE_BIT)) ++ { ++ for(j = 0; j < PAGE_SIZE/sizeof(pud_t); j++) ++ { ++ pud_t *pudp = (pud_t *)__va(__pgd_to_phys(*pgdp)) + i; ++ if((pud_val(*pudp) & PMD_SECT_VALID) && (pud_val(*pudp) & PMD_TABLE_BIT)) ++ { ++ pmd_t *current_pmdp = __va(__pud_to_phys(*pudp)); ++ if((unsigned long)current_pmdp == ((unsigned long)pmdp & PAGE_MASK)) ++ return true; ++ } ++ } ++ } ++ } ++ return false; ++} ++ ++// Return true if it is mapped to a physical range containing IEE page. ++// TODO : Optimize to get lower overhead. ++static inline bool check_addr_range_in_iee_valid(pmd_t pmd) ++{ ++ int i = 0; ++ unsigned long addr = __phys_to_iee(__pmd_to_phys(pmd)); ++ for(i = 0; i < PAGE_SIZE/sizeof(pmd_t); i++) ++ { ++ if(check_addr_in_iee_valid(addr + PAGE_SIZE * i)) ++ return true; ++ } ++ return false; ++} ++ ++void __iee_code _iee_set_pmd(pmd_t *pmdp, pmd_t pmd) ++{ ++ char * addr = (char *)__phys_to_kimg(__pmd_to_phys(pmd)); ++ ++ if(!(pmd_val(pmd) & PMD_SECT_VALID)) ++ { ++ WRITE_ONCE(*((pmd_t *)(__phys_to_iee(__pa(pmdp)))), pmd); ++ return; ++ } ++ ++ // Check if the pte table is legally allocated. ++ if ((pmd_val(pmd) & PMD_TABLE_BIT) && !check_addr_in_iee_valid(__phys_to_iee(__pmd_to_phys(pmd)))) ++ panic("You can't use non-iee-pgtable\n"); ++ ++ // Avoid mapping a huge pmd as U page. ++ if(!(pmd_val(pmd) & PMD_TABLE_BIT) && (pmd_val(pmd) & PMD_SECT_USER) && is_kernel_pmd_table(pmdp, pmd)) ++ panic("Set a block descriptor in kernel space U page.\n"); ++ ++ // Avoid mapping a huge pmd to IEE physical page. ++ if(!(pmd_val(pmd) & PMD_TABLE_BIT) && check_addr_range_in_iee_valid(pmd)) ++ panic("Mapping IEE physical page to a huge pmd.\n"); ++ ++ if(!check_pmd_dep(addr, pmd)) ++ return; ++ ++ WRITE_ONCE(*((pmd_t *)(__phys_to_iee(__pa(pmdp)))), pmd); ++} ++ ++// Return true if the pte table is a part of kernel page table. ++// TODO : Optimize to get lower overhead. ++static inline bool is_kernel_pte_table(pte_t *ptep, pte_t pte) ++{ ++ return false; ++} ++ ++// Return true if it does not change the privilage or add new U page in kernel. ++static inline bool check_privilage_safe(pte_t *ptep, pte_t pte) ++{ ++ if(!(pte_val(pte) & PTE_VALID)) ++ return true; ++ ++ if((pte_val(*ptep) & PTE_VALID)) ++ { ++ if((pte_val(*ptep) & PTE_USER) != (pte_val(pte) & PTE_USER)) ++ panic("Incorrectly change privilage.\n"); ++ } ++ else ++ { ++ if((pte_val(pte) & PTE_USER) && is_kernel_pte_table(ptep, pte)) ++ panic("Add new U page in kernel space.\n"); ++ } ++ return true; ++} ++ ++// TODO : When adding a new executable page, check it for DEP. ++static inline bool safely_adding_new_exec_page(pte_t *ptep, pte_t pte) ++{ ++ return true; ++} ++ ++// Return true if it is only changing prot of a pte. ++static inline bool is_changing_pte_prot(pte_t *ptep, pte_t pte) ++{ ++ if(((pte_val(*ptep) ^ pte_val(pte)) & PTE_ADDR_MASK) == 0) ++ return true; ++ else ++ return false; ++} ++ ++// Return true if the modify does not break DEP. ++static inline bool check_pte_dep(char *addr, pte_t pte) ++{ ++ // DEP for kernel code and readonly data ++ // _text: .text start addr, __init_begin: .rodata end addr ++ if (addr >= _stext && addr < _etext) ++ { ++ if ((PTE_WRITE & pte_val(pte)) // DBM == 1 --> writable ++ || !(PTE_RDONLY & pte_val(pte))) // DBM == 0 && AP[2] = 0 --> writable ++ { ++ panic("Can't make kernel's text/readonly page as writable!\n" ++ "addr = 0x%16llx, pte_val = 0x%16llx", ++ (u64)addr, pte_val(pte)); ++ } ++ } ++ return true; ++} ++ ++void __iee_code _iee_set_pte(pte_t *ptep, pte_t pte) ++{ ++ char * addr = (char *)__phys_to_kimg(__pte_to_phys(pte)); ++ ++ if(!(pte_val(pte) & PTE_VALID)) ++ { ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); ++ return; ++ } ++ ++ // Avoid modify privilage unsafely. ++ if(!check_privilage_safe(ptep, pte)) ++ panic("You are modify privilage unsafely.\n"); ++ ++ // Avoid mapping a new executable page. ++ if(!safely_adding_new_exec_page(ptep, pte)) ++ panic("You are adding a new executable page unsafely.\n"); ++ ++ // Avoid mapping a new VA to IEE PA. ++ if(!is_changing_pte_prot(ptep, pte) && ++ check_addr_in_iee_valid(__phys_to_iee(__pte_to_phys(pte)))) ++ panic("You are remmaping IEE page to other VA.\n"); ++ ++ // Avoid mapping a writable VA to kernel code PA. ++ if(!check_pte_dep(addr, pte)) ++ return; ++ ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); ++} ++ ++// Return true if it only sets U page and modify NG. ++static inline bool is_setting_upage(pte_t *ptep, pte_t pte) ++{ ++ if(((pte_val(*ptep) ^ pte_val(pte)) & ~(PTE_USER | PTE_NG)) != 0) ++ panic("Incorrectly setting U page.\n"); ++ if((pte_val(pte) & PTE_USER) != PTE_USER) ++ panic("Using error interface to set P page.\n"); ++ return true; ++} ++ ++void __iee_code _iee_set_pte_upage(pte_t *ptep, pte_t pte) ++{ ++ // Check if it only change the prot. ++ if(!is_setting_upage(ptep,pte)) ++ panic("Incorrectly setting U page.\n"); ++ ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); ++} ++ ++// Return true if it only sets P page and modify NG. ++static inline bool is_setting_ppage(pte_t *ptep, pte_t pte) ++{ ++ if(((pte_val(*ptep) ^ pte_val(pte)) & ~(PTE_USER | PTE_NG)) != 0) ++ panic("Incorrectly setting P page.\n"); ++ if((pte_val(pte) & PTE_USER) != 0) ++ panic("Using error interface to set U page.\n"); ++ return true; ++} ++ ++void __iee_code _iee_set_pte_ppage(pte_t *ptep, pte_t pte) ++{ ++ // Check if it only change the prot. ++ if(!is_setting_ppage(ptep,pte)) ++ panic("Incorrectly setting P page.\n"); ++ ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); ++} ++ ++void __iee_code _iee_set_bm_pte(pte_t *ptep, pte_t pte) ++{ ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa_symbol(ptep)))), pte); ++} ++ ++/* Data in iee_si_base is visible to all pgd while iee_si_data is private. */ ++unsigned long iee_base_idmap_pg_dir __iee_si_data; ++unsigned long iee_base_reserved_pg_dir __iee_si_data; ++unsigned long iee_base__bp_harden_el1_vectors __iee_si_data; ++bool iee_init_done __iee_si_data; ++unsigned long iee_si_tcr __iee_si_data; ++ ++static u64 __iee_si_code inline iee_si_mask(unsigned long mask, unsigned long new_val, unsigned long old_val) ++{ ++ return (new_val & mask) | (old_val & ~mask); ++} ++/* ++ * handler function for requests of executing sensitive instrutions. ++ */ ++u64 __iee_si_code iee_si_handler(int flag, ...) ++{ ++ va_list pArgs; ++ u64 old_val, new_val; ++ ++ // BUG_ON(flag > IEE_WRITE_MDSCR); ++ va_start(pArgs, flag); ++ switch (flag) { ++ case IEE_SI_TEST: ++ break; ++ case IEE_WRITE_SCTLR: { ++ old_val = read_sysreg(sctlr_el1); ++ new_val = va_arg(pArgs, u64); ++ new_val = iee_si_mask(IEE_SCTLR_MASK, new_val, old_val); ++ write_sysreg(new_val, sctlr_el1); ++ break; ++ } ++ case IEE_WRITE_TTBR0: ++ case IEE_CONTEXT_SWITCH: { ++ u64 new_asid, new_phys, old_phys, token_phys; ++ struct task_struct *tsk; ++ struct task_token *token; ++ new_val = va_arg(pArgs, u64); ++ new_phys = (new_val & PAGE_MASK) & ~TTBR_ASID_MASK; ++ new_asid = new_val >> 48; ++ ++ // Check ASID first ++ if (new_phys == iee_base_reserved_pg_dir){ ++ if (new_asid != 1) ++ panic("IEE SI warning: reserved_pg_dir ASID invalid: %llx:%llx", new_asid, new_val); ++ } ++ // Already reserved asid 1 for iee rwx gate. ++ else if (new_asid == 0){ ++ new_val |= FIELD_PREP(TTBR_ASID_MASK, 1); ++ printk("IEE SI: Modify ASID of %llx to 1.", new_val); ++ } ++ // TO DO: operations to protect idmap_pg_dir ++ else if (new_phys == iee_base_idmap_pg_dir) ++ { ++ // printk("IEE SI: switch to idmap_pg_dir.); ++ } ++ else if (new_asid % 2 ==0) ++ panic("IEE SI warning: TTBR0 ASID invalid: %llx:%llx", new_asid, new_val); ++ ++ /* Skip verification if iee hasn't been initialized. */ ++ if (iee_init_done){ ++ // Verify current sp_el0 with iee token info ++ asm volatile("mrs %x0, sp_el0":"=r"(tsk)); ++ token = (struct task_token *)__phys_to_iee(__pa(tsk)); ++ ++ /* ++ * token->pgd != NULL means it is a user task, then we need to check whether current ttbr0 is correct. ++ */ ++ if (token->pgd){ ++ old_val = read_sysreg(ttbr0_el1); ++ // When TTBR0 is reserved_pg_dir then no checking is available. ++ if (old_val != iee_base_reserved_pg_dir){ ++ old_phys = (old_val & PAGE_MASK) & ~TTBR_ASID_MASK; ++ token_phys = __pa(token->pgd); ++ if (old_phys != token_phys) ++ panic("IEE SI warning: Pgd set error. old ttbr0:%lx, token ttbr0:%lx, token pgd:%lx", ++ (unsigned long)old_phys, (unsigned long)token_phys, (unsigned long)(token->pgd)); ++ } ++ } ++ } ++ // all checks are done. ++ write_sysreg(new_val, ttbr0_el1); ++ ++ // SET ASID in TTBR1 when context switch ++ if (flag == IEE_CONTEXT_SWITCH){ ++ new_val = (read_sysreg(ttbr1_el1) & ~TTBR_ASID_MASK) | FIELD_PREP(TTBR_ASID_MASK, new_asid-1); ++ write_sysreg(new_val, ttbr1_el1); ++ } ++ break; ++ } ++ case IEE_WRITE_VBAR: { ++ u64 el1_vector; ++ new_val = va_arg(pArgs, u64); ++ el1_vector = iee_base__bp_harden_el1_vectors; ++ if(new_val == el1_vector || new_val == el1_vector+SZ_2K || ++ new_val == el1_vector+SZ_2K*2 || new_val == el1_vector+SZ_2K*3) ++ write_sysreg(new_val, vbar_el1); ++ break; ++ } ++ case IEE_WRITE_TCR: { ++ old_val = read_sysreg(tcr_el1); ++ new_val = va_arg(pArgs, u64); ++ new_val = iee_si_mask(IEE_TCR_MASK, new_val, old_val); ++ write_sysreg(new_val, tcr_el1); ++ break; ++ } ++ case IEE_WRITE_MDSCR: { ++ old_val = read_sysreg(mdscr_el1); ++ new_val = va_arg(pArgs, u64); ++ new_val = iee_si_mask(IEE_MDSCR_MASK, new_val, old_val); ++ write_sysreg(new_val, mdscr_el1); ++ break; ++ } ++ } ++ va_end(pArgs); ++ return 0; ++} ++/* ++ * TODO: scan a page to check whether it contains sensitive instructions ++ * return 1 when finding sensitive inst, 0 on safe page. ++ */ ++int iee_si_scan_page(unsigned long addr); ++#endif +\ No newline at end of file +diff --git a/arch/arm64/kernel/koi/Makefile b/arch/arm64/kernel/koi/Makefile +new file mode 100644 +index 000000000000..9be8710b714a +--- /dev/null ++++ b/arch/arm64/kernel/koi/Makefile +@@ -0,0 +1 @@ ++obj-y += koi.o +\ No newline at end of file +diff --git a/arch/arm64/kernel/koi/koi.c b/arch/arm64/kernel/koi/koi.c +new file mode 100644 +index 000000000000..716ba16ab358 +--- /dev/null ++++ b/arch/arm64/kernel/koi/koi.c +@@ -0,0 +1,1327 @@ ++#include "asm/koi.h" ++#include "linux/compiler_attributes.h" ++#include "linux/compiler_types.h" ++#include "asm/barrier.h" ++#include "asm-generic/bug.h" ++#include "asm-generic/errno-base.h" ++#include "asm-generic/memory_model.h" ++#include "asm-generic/pgtable-nop4d.h" ++#include "asm-generic/rwonce.h" ++#include "asm/pgalloc.h" ++#include "asm/memory.h" ++#include "linux/bitfield.h" ++#include "linux/compiler.h" ++#include "linux/types.h" ++#include "linux/spinlock.h" ++#include "linux/spinlock_types.h" ++#include "linux/kernel.h" ++#include "linux/rculist.h" ++#include "linux/rcupdate.h" ++#include "linux/list.h" ++#include "asm/current.h" ++#include "linux/compiler_types.h" ++#include "asm-generic/barrier.h" ++#include "asm-generic/rwonce.h" ++#include "asm-generic/pgalloc.h" ++#include "asm/cpufeature.h" ++#include "asm/kvm_hyp.h" ++#include "asm/mmu.h" ++#include "asm/mmu_context.h" ++#include "asm/page-def.h" ++#include "asm/pgalloc.h" ++#include "asm/pgtable-hwdef.h" ++#include "asm/pgtable-types.h" ++#include "asm/pgtable.h" ++#include "asm/string.h" ++#include "asm/sysreg.h" ++#include "linux/bitfield.h" ++#include "linux/compiler.h" ++#include "linux/export.h" ++#include "linux/gfp.h" ++#include "linux/huge_mm.h" ++#include "linux/kallsyms.h" ++#include "linux/kconfig.h" ++#include "linux/kern_levels.h" ++#include "linux/kernel.h" ++#include "linux/list.h" ++#include "linux/lockdep.h" ++#include "linux/mm.h" ++#include "linux/mm_types.h" ++#include "linux/pgtable.h" ++#include "linux/printk.h" ++#include "linux/rculist.h" ++#include "linux/rcupdate.h" ++#include "linux/rmap.h" ++#include "linux/sched.h" ++#include "linux/stddef.h" ++#include "linux/string.h" ++#include "linux/swap.h" ++#include "linux/swapops.h" ++#include "linux/types.h" ++#include "linux/slab.h" ++#include "linux/string.h" ++#include "linux/hashtable.h" ++ ++#define __koi_code __section(".koi.text") ++#define __koi_data __section(".data..koi") ++ ++extern unsigned long __koi_code_start[]; ++extern unsigned long __koi_code_end[]; ++extern unsigned long __koi_data_start[]; ++extern unsigned long __koi_data_end[]; ++#ifdef CONFIG_IEE ++extern unsigned long __iee_si_base_start[]; ++extern unsigned long __iee_exec_entry_start[]; ++extern unsigned long __iee_exec_entry_end[]; ++#endif ++ ++__koi_data unsigned long koi_swapper_ttbr1 = 0; ++EXPORT_SYMBOL(koi_swapper_ttbr1); ++#define KOI_SWAPPER_MASK 0x0000fffffffffff0 ++ ++__attribute__((aligned(PAGE_SIZE))) ++DEFINE_PER_CPU(unsigned long[PAGE_SIZE / sizeof(unsigned long)], ++ koi_irq_current_ttbr1); ++EXPORT_SYMBOL(koi_irq_current_ttbr1); ++ ++extern void koi_switch_to_ko_stack(unsigned long stack_top); ++extern void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp); ++extern void koi_check_and_switch_context(struct mm_struct *mm); ++extern int koi_add_page_mapping(unsigned long dst, unsigned long src); ++extern unsigned long _iee_read_token_ttbr1(struct task_struct *tsk); ++/** ++*struct koi_mem_list - maintain a linked list of free memory in the kernel ++*@addr: stating address of this memory ++*@size: the size of the memory ++*@list: the head of the koi_mem_list ++*@rcu: for rcu ++*/ ++struct koi_mem_list { ++ unsigned long addr; ++ unsigned long size; ++ struct list_head list; ++ struct rcu_head rcu; ++}; ++//mapping parameter pointer to copy ++struct koi_addr_map { ++ unsigned long buffer_addr; ++ unsigned long orig_addr; ++ int offset; ++ struct hlist_node node; ++ struct rcu_head rcu; ++}; ++ ++DEFINE_HASHTABLE(koi_mem_htbl, HASH_TABLE_BIT); ++EXPORT_SYMBOL(koi_mem_htbl); ++DEFINE_SPINLOCK(koi_mem_htbl_spin_lock); ++EXPORT_SYMBOL(koi_mem_htbl_spin_lock); ++ ++EXPORT_SYMBOL(koi_do_switch_to_ko_stack); ++EXPORT_SYMBOL(koi_do_switch_to_kernel_stack); ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++ ++/** ++* koi_ttbr_ctor - return ttbr1 for the given driver module ++*/ ++unsigned long koi_ttbr_ctor(struct module *mod) ++{ ++ struct koi_mem_hash_node *ko; ++ struct mm_struct *ko_mm; ++ unsigned long ttbr1; ++ unsigned long asid; ++ int bkt; ++ rcu_read_lock(); ++ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { ++ if (ko->mod == mod) { ++ ko_mm = ko->ko_mm; ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (!ko_mm) { ++ printk(KERN_ERR "cannot found module %s in koi_mem_htbl", ++ mod->name); ++ return 0; ++ } ++ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):); ++ asid &= TTBR_ASID_MASK; ++ ttbr1 = ko->ko_ttbr1 | asid; ++ // koi_check_and_switch_context(ko_mm); ++ // asid = ASID(ko_mm); ++ // ttbr1 = ko->ko_ttbr1 | FIELD_PREP(TTBR_ASID_MASK, asid); ++ return ttbr1; ++} ++EXPORT_SYMBOL(koi_ttbr_ctor); ++//release the hash node ++static __maybe_unused void koi_mem_hash_node_free(struct rcu_head *rcu) ++{ ++ struct koi_mem_hash_node *node = ++ container_of(rcu, struct koi_mem_hash_node, rcu); ++ kfree(node); ++} ++//release free memory linked list nodes ++static void koi_mem_node_free(struct rcu_head *rcu) ++{ ++ struct koi_mem_list *mem_node = ++ container_of(rcu, struct koi_mem_list, rcu); ++ kfree(mem_node); ++} ++//release the node in koi_addr_map ++static void koi_addr_map_node_free(struct rcu_head *rcu) ++{ ++ struct koi_addr_map *addr_map_node = ++ container_of(rcu, struct koi_addr_map, rcu); ++ kfree(addr_map_node); ++} ++ ++#ifndef CONFIG_IEE ++/* ++ * This function is used to switch to ko's pgtable. ++ */ ++__koi_code noinline unsigned long koi_do_switch_to_ko_pgtbl(void) ++{ ++ struct koi_mem_hash_node *ko; ++ // struct mm_struct *ko_mm; ++ unsigned long addr; ++ unsigned long ttbr1, asid; ++ unsigned long *ptr; ++ struct task_token *token_addr = ++ (struct task_token *)((unsigned long)current + ++ (unsigned long)koi_offset); ++ int bkt; ++ asm volatile(" mrs %0, elr_el1\n" : "=r"(addr)); ++ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, __kern_my_cpu_offset()); ++ rcu_read_lock(); ++ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { ++ if (ko->mod->init_layout.base != NULL) { ++ if (addr >= (unsigned long)ko->mod->init_layout.base && ++ addr < (unsigned long)(ko->mod->init_layout.base + ++ ko->mod->init_layout.size)) { ++ if (token_addr->current_ttbr1 == ko->ko_ttbr1 || ++ *ptr == ko->ko_ttbr1) { ++ // ko_mm = ko->ko_mm; ++ // koi_check_and_switch_context(ko_mm); ++ // asid = ASID(ko_mm); ++ // ttbr1 = ko->ko_ttbr1; ++ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); ++ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):); ++ asid &= TTBR_ASID_MASK; ++ ttbr1 = ko->ko_ttbr1 | asid; ++ rcu_read_unlock(); ++ return ttbr1; ++ } ++ rcu_read_unlock(); ++ return 0; ++ } ++ } ++ if (addr >= (unsigned long)ko->mod->core_layout.base && ++ addr < (unsigned long)ko->mod->core_layout.base + ++ ko->mod->core_layout.size) { ++ if (token_addr->current_ttbr1 == ko->ko_ttbr1 || ++ *ptr == ko->ko_ttbr1) { ++ // ko_mm = ko->ko_mm; ++ // koi_check_and_switch_context(ko_mm); ++ // asid = ASID(ko_mm); ++ // ttbr1 = ko->ko_ttbr1; ++ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); ++ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):); ++ asid &= TTBR_ASID_MASK; ++ ttbr1 = ko->ko_ttbr1 | asid; ++ rcu_read_unlock(); ++ return ttbr1; ++ } ++ rcu_read_unlock(); ++ return 0; ++ } ++ } ++ rcu_read_unlock(); ++ return 0; ++} ++/** ++* koi_do_switch_to_kernel_pgtbl - switch to kernel pagetable ++*/ ++__koi_code noinline int koi_do_switch_to_kernel_pgtbl(void) ++{ ++ unsigned long curr_ttbr1, asid; ++ // if (!cpu_online(smp_processor_id())) ++ // return 0; ++ asm volatile(" mrs %0, ttbr1_el1\n" : "=r"(curr_ttbr1)); ++ if ((curr_ttbr1 & KOI_SWAPPER_MASK) == ++ (koi_swapper_ttbr1 & KOI_SWAPPER_MASK)) { ++ return 0; ++ } ++ if (((curr_ttbr1 & TTBR_ASID_MASK) >> 48) <= 1) { ++ return 0; ++ } ++ asm volatile("mrs %0, ttbr0_el1\n":"=r"(asid):); ++ asid &= ~USER_ASID_FLAG; ++ asid &= TTBR_ASID_MASK; ++ write_sysreg(koi_swapper_ttbr1 | asid, ttbr1_el1); ++ isb(); ++ asm volatile(ALTERNATIVE("nop; nop; nop", "ic iallu; dsb nsh; isb", ++ ARM64_WORKAROUND_CAVIUM_27456)); ++ return 1; ++} ++#else ++__koi_code noinline unsigned long koi_do_switch_to_ko_pgtbl(void) ++{ ++ struct koi_mem_hash_node *ko; ++ struct mm_struct *ko_mm; ++ unsigned long addr, ttbr1, asid, pan_flag, current_ttbr1; ++ unsigned long *ptr; ++ int bkt; ++ asm volatile("mrs %0, pan\n" ++ "msr pan, 0x0\n" ++ : "=r"(pan_flag) ++ :); ++ current_ttbr1 = _iee_read_token_ttbr1(current); ++ asm volatile("msr pan, %0\n" : : "r"(pan_flag)); ++ ptr = SHIFT_PERCPU_PTR(koi_irq_current_ttbr1, __kern_my_cpu_offset()); ++ if (current_ttbr1 == 0 && *ptr == 0) ++ return 0; ++ asm volatile(" mrs %0, elr_el1\n" : "=r"(addr)); ++ rcu_read_lock(); ++ hash_for_each_rcu (koi_mem_htbl, bkt, ko, node) { ++ if (ko->mod->init_layout.base != NULL) { ++ if (addr >= (unsigned long)ko->mod->init_layout.base && ++ addr < (unsigned long)(ko->mod->init_layout.base + ++ ko->mod->init_layout.size)) { ++ rcu_read_unlock(); ++ if (current_ttbr1 == ko->ko_ttbr1 || *ptr == ko->ko_ttbr1) { ++ // ko_mm = ko->ko_mm; ++ // koi_check_and_switch_context(ko_mm); ++ // asid = ASID(ko_mm); ++ // ttbr1 = ko->ko_ttbr1; ++ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, ++ // asid); ++ return ko->ko_ttbr1; ++ } ++ return 0; ++ } ++ } ++ if (addr >= (unsigned long)ko->mod->core_layout.base && ++ addr < (unsigned long)ko->mod->core_layout.base + ++ ko->mod->core_layout.size) { ++ rcu_read_unlock(); ++ if (current_ttbr1 == ko->ko_ttbr1 || *ptr == ko->ko_ttbr1) { ++ // ko_mm = ko->ko_mm; ++ // koi_check_and_switch_context(ko_mm); ++ // asid = ASID(ko_mm); ++ // ttbr1 = ko->ko_ttbr1; ++ // ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); ++ return ko->ko_ttbr1; ++ } ++ return 0; ++ } ++ } ++ rcu_read_unlock(); ++ return 0; ++} ++ ++__koi_code noinline int koi_do_switch_to_kernel_pgtbl(void) ++{ ++ unsigned long curr_ttbr1; ++ // if (!cpu_online(smp_processor_id())) ++ // return 0; ++ asm volatile(" mrs %0, ttbr1_el1\n" : "=r"(curr_ttbr1)); ++ if ((curr_ttbr1 & KOI_SWAPPER_MASK) == ++ (koi_swapper_ttbr1 & KOI_SWAPPER_MASK)) { ++ return 0; ++ } ++ if (((curr_ttbr1 & TTBR_ASID_MASK) >> 48) <= 1) { ++ return 0; ++ } ++ iee_rwx_gate_entry(IEE_SWITCH_TO_KERNEL); ++ return 1; ++} ++#endif ++/** ++* koi_save_ttbr - save ttbr of each driver module ++* @mod: driver module ++* @pgdp:pointer to driver module top page table,pgd ++*/ ++static void koi_save_ttbr(struct module *mod, pgd_t *pgdp, ++ struct koi_mem_hash_node *node) ++{ ++ phys_addr_t ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); ++ if (system_supports_cnp()) ++ ttbr1 |= TTBR_CNP_BIT; ++ node->ko_ttbr1 = ttbr1; ++} ++/** ++*kio_normal_page - to obtain the pointer of the corresponding struct page structure ++*from a given page table entry(pte) ++*/ ++struct page *koi_normal_page(pte_t pte) ++{ ++ unsigned long pfn = pte_pfn(pte); ++ ++ if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) { ++ if (likely(!pte_special(pte))) ++ goto check_pfn; ++ if (is_zero_pfn(pfn)) { ++ printk(KERN_ERR "zero pfn found! pte=0x%16lx\n", pte); ++ return NULL; ++ } ++ if (pte_devmap(pte)) { ++ printk(KERN_ERR "pte for dev found! pte=0x%16lx\n", ++ pte); ++ return NULL; ++ } ++ return NULL; ++ } ++ ++check_pfn: ++ return pfn_to_page(pfn); ++} ++ ++/** ++ * Copy one pte. Returns 0 if succeeded, or -EAGAIN if one preallocated page ++ * is required to copy this pte. ++*/ ++static inline int koi_copy_present_pte(pte_t *dst_pte, pte_t *src_pte, ++ unsigned long addr, ++ struct page **prealloc) ++{ ++ pte_t pte = *src_pte; ++ struct page *page; ++ ++ page = koi_normal_page(pte); ++ if (!page) { ++ printk(KERN_ERR "pte_page unavailable. Impossible.....\n"); ++ return -1; ++ } ++ ++ set_pte(dst_pte, pte); ++ return 0; ++} ++/** ++* copy huge pmd from kernel space to driver space. ++*/ ++static int koi_copy_huge_pmd(struct mm_struct *ko_mm, pmd_t *dst_pmd, ++ pmd_t *src_pmd, unsigned long addr) ++{ ++ spinlock_t *src_ptl; ++ pmd_t pmd; ++ int ret = -ENOMEM; ++ ++ src_ptl = pmd_lockptr(&init_mm, src_pmd); ++ spin_lock_bh(src_ptl); ++ ++ ret = -EAGAIN; ++ pmd = *src_pmd; ++ ++ set_pte((pte_t *)dst_pmd, pmd_pte(pmd)); ++ ret = 0; ++ spin_unlock_bh(src_ptl); ++ return ret; ++} ++ ++int __koi_pte_alloc(struct mm_struct *mm, pmd_t *pmd) ++{ ++ spinlock_t *ptl; ++ pgtable_t new = pte_alloc_one(mm); ++ if (!new) ++ return -ENOMEM; ++ ++ /* ++ * Ensure all pte setup (eg. pte page lock and page clearing) are ++ * visible before the pte is made visible to other CPUs by being ++ * put into page tables. ++ * ++ * The other side of the story is the pointer chasing in the page ++ * table walking code (when walking the page table without locking; ++ * ie. most of the time). Fortunately, these data accesses consist ++ * of a chain of data-dependent loads, meaning most CPUs (alpha ++ * being the notable exception) will already guarantee loads are ++ * seen in-order. See the alpha page table accessors for the ++ * smp_rmb() barriers in page table walking code. ++ */ ++ smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */ ++ ++ ptl = pmd_lockptr(mm, pmd); ++ spin_lock_bh(ptl); ++ if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ ++ #ifdef CONFIG_PTP ++ pte_t *pte = (pte_t *)page_address(new); ++ unsigned long iee_addr = __phys_to_iee(__pa(pte)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)pte); ++ #endif ++ mm_inc_nr_ptes(mm); ++ pmd_populate(mm, pmd, new); ++ new = NULL; ++ } ++ spin_unlock_bh(ptl); ++ if (new) ++ pte_free(mm, new); ++ return 0; ++} ++ ++#define koi_pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __koi_pte_alloc(mm, pmd)) ++ ++#define koi_pte_offset_map_lock(mm, pmd, address, ptlp) \ ++({ \ ++ spinlock_t *__ptl = pte_lockptr(mm, pmd); \ ++ pte_t *__pte = pte_offset_map(pmd, address); \ ++ *(ptlp) = __ptl; \ ++ spin_lock_bh(__ptl); \ ++ __pte; \ ++}) ++ ++#define koi_pte_alloc_map_lock(mm, pmd, address, ptlp) \ ++ (koi_pte_alloc(mm, pmd) ? \ ++ NULL : koi_pte_offset_map_lock(mm, pmd, address, ptlp)) ++ ++/** ++*koi_copy_pte_range - copy pte from kernel space to driver space ++*/ ++static int koi_copy_pte_range(struct mm_struct *ko_mm, pmd_t *dst_pmd, ++ pmd_t *src_pmd, unsigned long addr, ++ unsigned long end) ++{ ++ pte_t *src_pte, *dst_pte; ++ spinlock_t *src_ptl, *dst_ptl; ++ int ret = 0; ++ struct page *prealloc = NULL; ++again: ++ dst_pte = koi_pte_alloc_map_lock(ko_mm, dst_pmd, addr, &dst_ptl); ++ if (!dst_pte) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ src_pte = pte_offset_map(src_pmd, addr); ++ src_ptl = pte_lockptr(&init_mm, src_pmd); ++ spin_lock_bh(src_ptl); ++ arch_enter_lazy_mmu_mode(); ++ ++ do { ++ if (pte_none(*src_pte)) ++ continue; ++ if (unlikely(!pte_present(*src_pte))) { ++ continue; ++ } ++ /* koi_copy_present_pte() will clear `*prealloc` if consumed */ ++ ret = koi_copy_present_pte(dst_pte, src_pte, addr, &prealloc); ++ if (unlikely(ret == -EAGAIN)) ++ break; ++ if (unlikely(prealloc)) { ++ put_page(prealloc); ++ prealloc = NULL; ++ } ++ } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); ++ arch_leave_lazy_mmu_mode(); ++ spin_unlock_bh(src_ptl); ++ spin_unlock_bh(dst_ptl); ++ ++ if (ret) { ++ WARN_ON_ONCE(ret != -EAGAIN); ++ ret = 0; ++ } ++ if (addr != end) ++ goto again; ++out: ++ if (unlikely(prealloc)) ++ put_page(prealloc); ++ return ret; ++} ++ ++int __koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) ++{ ++ spinlock_t *ptl; ++ pmd_t *new = pmd_alloc_one(mm, address); ++ if (!new) ++ return -ENOMEM; ++ ++ smp_wmb(); /* See comment in __pte_alloc */ ++ ++ ptl = pud_lockptr(mm, pud); ++ spin_lock_bh(ptl); ++ if (!pud_present(*pud)) { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(new)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)new); ++ #endif ++ mm_inc_nr_pmds(mm); ++ pud_populate(mm, pud, new); ++ } else /* Another has populated it */ ++ pmd_free(mm, new); ++ spin_unlock_bh(ptl); ++ return 0; ++} ++ ++static inline pmd_t *koi_pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) ++{ ++ return (unlikely(pud_none(*pud)) && __koi_pmd_alloc(mm, pud, address))? ++ NULL: pmd_offset(pud, address); ++} ++ ++/** ++*kio_copy_pmd_range - copy pmd from kernel to driver space ++*/ ++static inline int koi_copy_pmd_range(struct mm_struct *ko_mm, pud_t *dst_pud, ++ pud_t *src_pud, unsigned long addr, ++ unsigned long end) ++{ ++ pmd_t *src_pmd, *dst_pmd; ++ unsigned long next; ++ int err; ++ ++ dst_pmd = koi_pmd_alloc(ko_mm, dst_pud, addr); ++ if (!dst_pmd) { ++ return -ENOMEM; ++ } ++ src_pmd = pmd_offset(src_pud, addr); ++ do { ++ next = pmd_addr_end(addr, end); ++ // CONFIG_TRANSPARENT_HUGEPAGE is enabled, so we must add copy_huge_pmd ++ if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd) || ++ (pmd_devmap(*src_pmd))) { ++ err = koi_copy_huge_pmd(ko_mm, dst_pmd, src_pmd, addr); ++ if (err == -ENOMEM) ++ return -ENOMEM; ++ if (!err) ++ continue; ++ } ++ if (pmd_none_or_clear_bad(src_pmd)) { ++ continue; ++ } ++ if (koi_copy_pte_range(ko_mm, dst_pmd, src_pmd, addr, next)) ++ return -ENOMEM; ++ } while (dst_pmd++, src_pmd++, addr = next, addr != end); ++ return 0; ++} ++ ++int __koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) ++{ ++ pud_t *new = pud_alloc_one(mm, address); ++ if (!new) ++ return -ENOMEM; ++ ++ smp_wmb(); /* See comment in __pte_alloc */ ++ ++ spin_lock_bh(&mm->page_table_lock); ++ if (!p4d_present(*p4d)) { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(new)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)new); ++ #endif ++ mm_inc_nr_puds(mm); ++ p4d_populate(mm, p4d, new); ++ } else /* Another has populated it */ ++ pud_free(mm, new); ++ spin_unlock_bh(&mm->page_table_lock); ++ return 0; ++} ++ ++static inline pud_t *koi_pud_alloc(struct mm_struct *mm, p4d_t *p4d, ++ unsigned long address) ++{ ++ return (unlikely(p4d_none(*p4d)) && __koi_pud_alloc(mm, p4d, address)) ? ++ NULL : pud_offset(p4d, address); ++} ++ ++/** ++*koi_copy_pud_range - copy pud from kernel to driver ++*/ ++static inline int koi_copy_pud_range(struct mm_struct *ko_mm, p4d_t *dst_p4d, ++ p4d_t *src_p4d, unsigned long addr, ++ unsigned long end) ++{ ++ pud_t *src_pud, *dst_pud; ++ unsigned long next; ++ dst_pud = koi_pud_alloc(ko_mm, dst_p4d, addr); ++ if (!dst_pud) ++ return -ENOMEM; ++ src_pud = pud_offset(src_p4d, addr); ++ do { ++ next = pud_addr_end(addr, end); ++ if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) { ++ continue; ++ /* fall through */ ++ } ++ if (pud_none_or_clear_bad(src_pud)) ++ continue; ++ if (koi_copy_pmd_range(ko_mm, dst_pud, src_pud, addr, next)) ++ return -ENOMEM; ++ } while (dst_pud++, src_pud++, addr = next, addr != end); ++ return 0; ++} ++ ++/** ++* koi_copy_p4d_range - map the kernel pagetable to the driver space level by level ++* @ko_mm: the mm_struct of driver module ++* @dst_pgd: destination pgd ++* @src_pgd: source pgd ++* @addr: the start of address ++* @end: the end of address ++*/ ++static inline int koi_copy_p4d_range(struct mm_struct *ko_mm, pgd_t *dst_pgd, ++ pgd_t *src_pgd, unsigned long addr, ++ unsigned long end) ++{ ++ p4d_t *src_p4d, *dst_p4d; ++ unsigned long next; ++ dst_p4d = p4d_alloc(ko_mm, dst_pgd, addr); ++ if (!dst_p4d) ++ return -ENOMEM; ++ src_p4d = p4d_offset(src_pgd, addr); ++ do { ++ next = p4d_addr_end(addr, end); ++ if (p4d_none_or_clear_bad(src_p4d)) ++ continue; ++ if (koi_copy_pud_range(ko_mm, dst_p4d, src_p4d, addr, next)) { ++ return -ENOMEM; ++ } ++ } while (dst_p4d++, src_p4d++, addr = next, addr != end); ++ return 0; ++} ++ ++/** ++*int koi_copy_pagetable - map the address range from "addr" to "end" to the driver pagetable ++*@ko_mm: the mm_struct of the driver module ++*@koi_pg_dir: koi_pg_dir, related to the driver module, the entry for driver pagetable ++*@addr: the starting address of mapping zone ++*@end: the end address of mapping zone ++*/ ++int koi_copy_pagetable(struct mm_struct *ko_mm, pgd_t *koi_pg_dir, ++ unsigned long addr, unsigned long end) ++{ ++ int ret = 0; ++ unsigned long next; ++ ++ pgd_t *src_pgd, *dst_pgd; ++ ++ src_pgd = pgd_offset_pgd(swapper_pg_dir, addr); ++ dst_pgd = pgd_offset_pgd(koi_pg_dir, addr); ++ do { ++ next = pgd_addr_end(addr, end); ++ if (pgd_none_or_clear_bad(src_pgd)) ++ continue; ++ if (unlikely(koi_copy_p4d_range(ko_mm, dst_pgd, src_pgd, addr, ++ next))) { ++ ret = -ENOMEM; ++ break; ++ } ++ } while (dst_pgd++, src_pgd++, addr = next, addr != end); ++ ++ return ret; ++} ++ ++void koi_set_rdonly(unsigned long addr, pgd_t *pgdir) ++{ ++ p4d_t *p4dp; ++ pud_t *pudp; ++ pmd_t *pmdp; ++ pte_t *ptep; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ if (pgd_none(*pgdp) || pgd_bad(*pgdp)) { ++ return; ++ } ++ ++ p4dp = p4d_offset(pgdp, addr); ++ if (p4d_none(*p4dp) || p4d_bad(*p4dp)) { ++ return; ++ } ++ ++ pudp = pud_offset(p4dp, addr); ++ if (pud_none(*pudp) || pud_bad(*pudp)) { ++ return; ++ } ++ pmdp = pmd_offset(pudp, addr); ++ if (pmd_none(*pmdp) || pmd_bad(*pmdp)) { ++ return; ++ } ++ ++ ptep = pte_offset_kernel(pmdp, addr); ++ if (pte_none(*ptep)) { ++ printk(KERN_ERR "ptep 0x%llx not available\n", ptep); ++ return; ++ } ++ set_pte(ptep, __pte(pte_val(*ptep) | PTE_RDONLY)); ++ printk(KERN_ERR "set_readonly successfully\n"); ++ return; ++} ++ ++/** ++* koi_create_pagetable - create pagetable for driver ++* @mod: driver module ++* 1.create a new koi_mem_hash_node new_node ++* 2.create page table return the pgd address, init the new_node->pgdp ++* 3.create and init the new_node->ko_mm ++* 4.map swapper_ttbr1 to the newly created pagetable ++* 5.map the interrupt vector table to the newly created pagetable ++* 6.map the init_layout of the module ++* 7.map the core_layout of the module ++* 8.map switch_to_kernel_pgtable into driver view ++* 9.map share memory ++*/ ++void koi_create_pagetable(struct module *mod) ++{ ++ int ret = 0, cpu; ++ unsigned long vbar, addr, ttbr1; ++ pgd_t *pgdp; ++ unsigned long *ptr; ++ struct koi_mem_list *new_mem_node; ++ struct koi_mem_hash_node *new_node = ++ kzalloc(sizeof(struct koi_mem_hash_node), GFP_KERNEL); ++ if (!new_node) { ++ printk(KERN_ERR "NULL new_node\n"); ++ return; ++ }; ++ if (koi_swapper_ttbr1 == 0) { ++ pgdp = lm_alias(swapper_pg_dir); ++ ttbr1 = phys_to_ttbr(virt_to_phys(pgdp)); ++ if (system_supports_cnp() && ++ !WARN_ON(pgdp != lm_alias(swapper_pg_dir))) ++ ttbr1 |= TTBR_CNP_BIT; ++#ifdef CONFIG_IEE ++ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, 1); ++#endif ++ koi_swapper_ttbr1 = ttbr1; ++ // __WRITE_ONCE(koi_swapper_ttbr1, ttbr1); ++ // koi_set_rdonly(&koi_swapper_ttbr1, swapper_pg_dir); ++ } ++ new_node->pgdp = koi_pgd_alloc(); ++ new_node->ko_mm = ++ kzalloc(sizeof(struct mm_struct) + ++ sizeof(unsigned long) * BITS_TO_LONGS(NR_CPUS), ++ GFP_KERNEL); ++ init_ko_mm(new_node->ko_mm, new_node->pgdp); ++ new_node->mod = mod; ++ koi_save_ttbr(mod, new_node->pgdp, new_node); ++ printk(KERN_ERR "copying koi_data, start=0x%16llx, end=0x%16llx\n", ++ (unsigned long)__koi_data_start, (unsigned long)__koi_data_end); ++ // copy koi_swapper_ttbr1, which records page dir base for kernel view ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)__koi_data_start, ++ (unsigned long)__koi_data_end); ++ asm volatile("mrs %0, VBAR_EL1\n" : "=r"(vbar) :); ++ ++ // copy interrupt vectors ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, vbar & PAGE_MASK, ++ (vbar + PAGE_SIZE) & PAGE_MASK); ++ ++ // copy module init_layout, which contains init data and text in driver ++ ret = koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)mod->init_layout.base, ++ (unsigned long)mod->init_layout.base + ++ mod->init_layout.size); ++ if (ret != 0) ++ printk(KERN_ERR ++ "\033[33mError occur when copying init_layout, Eno:%d\033[0m\n", ++ ret); ++ ++ // copy module core_layout, which contains non-init data and text in driver ++ ret = koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)mod->core_layout.base, ++ (unsigned long)mod->core_layout.base + ++ mod->core_layout.size); ++ if (ret != 0) ++ printk(KERN_ERR ++ "\033[33mError occur when copying core_layout, Eno: %d\033[0m\n", ++ ret); ++ ++ // mapping switch_to_kernel_pgtable into driver view, which is used to switch to kernel view when entering INT ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)__koi_code_start, ++ (unsigned long)__koi_code_end); ++ ++ for_each_possible_cpu (cpu) { ++ ptr = per_cpu(irq_stack_ptr, cpu); ++ printk(KERN_ERR ++ "\033[33mirq_stack_ptr on cpu %d addr=0x%16llx, end=0x%16llx\033[0m\n", ++ cpu, (unsigned long)ptr, ++ (unsigned long)ptr + IRQ_STACK_SIZE); ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)ptr, ++ (unsigned long)ptr + IRQ_STACK_SIZE); ++ } ++ ++ for_each_possible_cpu (cpu) { ++ ptr = per_cpu(koi_irq_current_ttbr1, cpu); ++ printk(KERN_ERR ++ "\033[33mirq_current_ptr on cpu %d addr=0x%16llx, end=0x%16llx\033[0m\n", ++ cpu, (unsigned long)ptr, (unsigned long)ptr + PAGE_SIZE); ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)ptr, ++ (unsigned long)ptr + PAGE_SIZE); ++ } ++ ++#ifdef CONFIG_IEE ++ // mapping iee_rwx_gate_entry and iee_si_base to ko's pagetable ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, ++ (unsigned long)__iee_si_base_start, ++ (unsigned long)__iee_exec_entry_end); ++#endif ++ ++ // alloc 16KB memory for new ko, and add it into hashtable ++ addr = (unsigned long)kmalloc(THREAD_SIZE, GFP_KERNEL); ++ if ((void *)addr == NULL) { ++ printk(KERN_ERR "alloc buffer error\n"); ++ } ++ koi_copy_pagetable(new_node->ko_mm, new_node->pgdp, addr, ++ addr + THREAD_SIZE); ++ ++ new_mem_node = kmalloc(sizeof(struct koi_mem_list), GFP_KERNEL); ++ if (new_mem_node == NULL) { ++ printk(KERN_ERR "alloc new_mem_node error\n"); ++ } ++ new_mem_node->addr = addr; ++ new_mem_node->size = THREAD_SIZE; ++ ++ new_node->mem_list_head = ++ (struct list_head)LIST_HEAD_INIT(new_node->mem_list_head); ++ hash_init(new_node->addr_htbl); ++ spin_lock_init(&new_node->addr_htbl_spin_lock); ++ spin_lock_init(&new_node->spin_lock); ++ ++ spin_lock(&new_node->spin_lock); ++ list_add_rcu(&new_mem_node->list, &new_node->mem_list_head); ++ spin_unlock(&new_node->spin_lock); ++ ++ spin_lock(&koi_mem_htbl_spin_lock); ++ hash_add_rcu(koi_mem_htbl, &new_node->node, ++ (unsigned long)new_node->mod); ++ spin_unlock(&koi_mem_htbl_spin_lock); ++} ++/** ++* koi_mem_alloc ++*@mod: driver module ++*@orig_addr: the starting address of the parameter in kernel ++*@size: the size of the parameter ++*/ ++unsigned long koi_mem_alloc(struct module *mod, unsigned long orig_addr, ++ unsigned long size) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ struct koi_mem_list *mem_node; ++ struct koi_addr_map *new_addr_node; ++ unsigned long addr = 0, flags; ++ struct koi_mem_list *new_mem_node; ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk(KERN_ERR "mem node for module: %s not found\n", ++ mod->name); ++ return 0; ++ } ++ spin_lock_irqsave(&target->spin_lock, flags); ++ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { ++ if (mem_node->size >= size) { ++ addr = mem_node->addr; ++ mem_node->size -= size; ++ if (mem_node->size == 0) { ++ list_del_rcu(&mem_node->list); ++ } else { ++ new_mem_node = ++ kmalloc(sizeof(struct koi_mem_list), ++ GFP_ATOMIC); ++ new_mem_node->addr = addr + size; ++ new_mem_node->size = mem_node->size; ++ list_replace_rcu(&mem_node->list, ++ &new_mem_node->list); ++ } ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } ++ } ++ spin_unlock_irqrestore(&target->spin_lock, flags); ++ if (!addr) { ++ addr = (unsigned long)kmalloc(THREAD_SIZE, GFP_KERNEL); ++ if ((void *)addr == NULL) { ++ return 0; ++ } ++ koi_copy_pagetable(target->ko_mm, target->pgdp, addr, ++ addr + THREAD_SIZE); ++ mem_node = kmalloc(sizeof(struct koi_mem_list), GFP_KERNEL); ++ if (!mem_node) { ++ printk(KERN_ERR "NULL mem_node\n"); ++ } ++ if (size > THREAD_SIZE) { ++ return 0; ++ } ++ mem_node->addr = addr + size; ++ mem_node->size = THREAD_SIZE - size; ++ spin_lock_irqsave(&target->spin_lock, flags); ++ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); ++ spin_unlock_irqrestore(&target->spin_lock, flags); ++ } ++ ++ new_addr_node = kzalloc(sizeof(struct koi_addr_map), GFP_KERNEL); ++ new_addr_node->buffer_addr = addr; ++ new_addr_node->orig_addr = orig_addr; ++ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); ++ hash_add_rcu(target->addr_htbl, &new_addr_node->node, ++ new_addr_node->buffer_addr); ++ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); ++ return addr; ++} ++EXPORT_SYMBOL(koi_mem_alloc); ++// find the parameter pointer corresponding to the copy ++noinline void *koi_mem_lookup(struct module *mod, unsigned long addr) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ struct koi_addr_map *addr_map_node; ++ unsigned long orig_addr = addr; ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk(KERN_ERR "mem node for module: %s not found\n", ++ mod->name); ++ return NULL; ++ } ++ ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, ++ orig_addr) { ++ if (addr_map_node->buffer_addr == addr) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (addr_map_node) { ++ return (void *)(addr_map_node->orig_addr); ++ } else { ++ return NULL; ++ } ++} ++EXPORT_SYMBOL(koi_mem_lookup); ++/** ++* kio_mem_free - recycle a copy of the copied parameters and synchronize the parameters ++* @mod: driver module ++* @addr: the starting addr of parameter ++* @size: the size of the parameter ++* @is_const: const pointers or not ++* @count: contry the number of parameters ++*/ ++noinline void koi_mem_free(struct module *mod, unsigned long addr, ++ unsigned long size, bool is_const, int count, ...) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ struct koi_mem_list *mem_node; ++ struct list_head *pos = NULL; ++ struct koi_addr_map *addr_map_node; ++ unsigned long orig_size = size; ++ unsigned long orig_addr = addr; ++ va_list valist; ++ int i; ++ unsigned int offset; ++ unsigned long flags; ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk(KERN_ERR "mem node for module: %s not found\n", ++ mod->name); ++ return; ++ } ++ ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, ++ orig_addr) { ++ if (addr_map_node->buffer_addr == orig_addr) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ va_start(valist, count); ++ for (i = 0; i < count; i++) { ++ offset = va_arg(valist, int); ++ *(unsigned long *)(addr_map_node->buffer_addr + offset) = ++ *(unsigned long *)(addr_map_node->orig_addr + offset); ++ } ++ va_end(valist); ++ memcpy((void *)addr_map_node->orig_addr, ++ (void *)addr_map_node->buffer_addr, orig_size); ++ ++ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); ++ hlist_del_init_rcu(&addr_map_node->node); ++ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); ++ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); ++ ++ spin_lock_irqsave(&target->spin_lock, flags); ++ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { ++ if (mem_node->addr + mem_node->size == addr) { ++ pos = mem_node->list.prev; ++ addr = mem_node->addr; ++ size += mem_node->size; ++ list_del_rcu(&mem_node->list); ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } else if (addr + size == mem_node->addr) { ++ if (!pos) ++ pos = mem_node->list.prev; ++ size += mem_node->size; ++ list_del_rcu(&mem_node->list); ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } else if (addr + size < mem_node->addr) { ++ if (!pos) ++ pos = mem_node->list.prev; ++ break; ++ } ++ } ++ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC); ++ mem_node->addr = addr; ++ mem_node->size = size; ++ if (pos) ++ list_add_rcu(&mem_node->list, pos); ++ else ++ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); ++ spin_unlock_irqrestore(&target->spin_lock, flags); ++} ++EXPORT_SYMBOL(koi_mem_free); ++/** ++* koi_mem_free_callback - used to recycle the copy of parameter. ++*@addr: the address of the parameter ++*@(*func)(void*): callback func, used to release the copy of the parameter pointer ++*/ ++noinline void koi_mem_free_callback(struct module *mod, unsigned long addr, ++ unsigned long size, void (*func)(void *)) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ struct koi_mem_list *mem_node; ++ struct list_head *pos = NULL; ++ struct koi_addr_map *addr_map_node; ++ unsigned long flags; ++ unsigned long orig_size = size; ++ unsigned long orig_addr = addr; ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk("mem node for module: %s not found\n", mod->name); ++ return; ++ } ++ ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, ++ orig_addr) { ++ if (addr_map_node->buffer_addr == orig_addr) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (addr_map_node != NULL) { ++ memcpy((void *)addr_map_node->orig_addr, ++ (void *)addr_map_node->buffer_addr, orig_size); ++ func((void *)addr_map_node->orig_addr); ++ } else { ++ printk("Cannot find addr_map_node in addr_htbl, maybe addr is in kernel space!!\n"); ++ func((void *)orig_addr); ++ } ++ ++ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); ++ if (addr_map_node != NULL) { ++ hlist_del_init_rcu(&addr_map_node->node); ++ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); ++ } ++ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); ++ spin_lock_irqsave(&target->spin_lock, flags); ++ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { ++ if (mem_node->addr + mem_node->size == addr) { ++ pos = mem_node->list.prev; ++ addr = mem_node->addr; ++ size += mem_node->size; ++ list_del_rcu(&mem_node->list); ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } else if (addr + size == mem_node->addr) { ++ if (!pos) ++ pos = mem_node->list.prev; ++ size += mem_node->size; ++ list_del_rcu(&mem_node->list); ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } else if (addr + size < mem_node->addr) { ++ if (!pos) ++ pos = mem_node->list.prev; ++ break; ++ } ++ } ++ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC); ++ mem_node->addr = addr; ++ mem_node->size = size; ++ if (pos) ++ list_add_rcu(&mem_node->list, pos); ++ else ++ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); ++ spin_unlock_irqrestore(&target->spin_lock, flags); ++} ++EXPORT_SYMBOL(koi_mem_free_callback); ++ ++void koi_map_mem(struct module *mod, unsigned long addr, unsigned long size) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) ++ break; ++ } ++ rcu_read_unlock(); ++ ++ if (target == NULL) { ++ printk(KERN_ERR "mem node for module: %s not found\n", ++ mod->name); ++ return; ++ } ++ koi_copy_pagetable(target->ko_mm, target->pgdp, addr & PAGE_MASK, ++ (addr + size + PAGE_SIZE) & PAGE_MASK); ++} ++EXPORT_SYMBOL(koi_map_mem); ++/** ++* koi_mem_free_to_user - function 'copy_to_user' in driver space ++*/ ++void koi_mem_free_to_user(struct module *mod, unsigned long addr, ++ unsigned long size) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ struct koi_mem_list *mem_node; ++ struct list_head *pos = NULL; ++ struct koi_addr_map *addr_map_node; ++ unsigned long flags; ++ unsigned long orig_size = size; ++ unsigned long orig_addr = addr; ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk(KERN_ERR "mem node for module: %s not found\n", ++ mod->name); ++ return; ++ } ++ ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (target->addr_htbl, addr_map_node, node, ++ orig_addr) { ++ if (addr_map_node->buffer_addr == orig_addr) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (copy_to_user((void *)addr_map_node->orig_addr, ++ (void *)addr_map_node->buffer_addr, orig_size)) { ++ return; ++ } ++ ++ spin_lock_irqsave(&target->addr_htbl_spin_lock, flags); ++ hlist_del_init_rcu(&addr_map_node->node); ++ call_rcu(&addr_map_node->rcu, koi_addr_map_node_free); ++ spin_unlock_irqrestore(&target->addr_htbl_spin_lock, flags); ++ spin_lock_irqsave(&target->spin_lock, flags); ++ list_for_each_entry_rcu (mem_node, &target->mem_list_head, list) { ++ if (mem_node->addr + mem_node->size == addr) { ++ pos = mem_node->list.prev; ++ addr = mem_node->addr; ++ size += mem_node->size; ++ list_del_rcu(&mem_node->list); ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } else if (addr + size == mem_node->addr) { ++ if (!pos) ++ pos = mem_node->list.prev; ++ size += mem_node->size; ++ list_del_rcu(&mem_node->list); ++ call_rcu(&mem_node->rcu, koi_mem_node_free); ++ } else if (addr + size < mem_node->addr) { ++ if (!pos) ++ pos = mem_node->list.prev; ++ break; ++ } ++ } ++ mem_node = kzalloc(sizeof(struct koi_mem_list), GFP_ATOMIC); ++ mem_node->addr = addr; ++ mem_node->size = size; ++ if (pos) ++ list_add_rcu(&mem_node->list, pos); ++ else ++ list_add_tail_rcu(&mem_node->list, &target->mem_list_head); ++ spin_unlock_irqrestore(&target->spin_lock, flags); ++} ++EXPORT_SYMBOL(koi_mem_free_to_user); ++// map the driver stack to kernel ++void koi_map_kostack(struct module *mod) ++{ ++ struct koi_mem_hash_node *target = NULL; ++ void *koi_stack; ++ unsigned long cur_sp; ++ asm volatile("mov %0, sp\n" : "=r"(cur_sp) :); ++ if (on_irq_stack(cur_sp, NULL)) { ++ return; ++ } ++#ifndef CONFIG_IEE ++ unsigned long res, alloc_token; ++ struct task_token *token_addr = ++ (struct task_token *)((unsigned long)current + ++ (unsigned long)koi_offset); ++ if (token_addr->koi_stack_base != NULL) ++ return; ++#else ++ koi_stack = iee_rw_gate(IEE_READ_KOI_STACK, current); ++ if (koi_stack != NULL) ++ return; ++#endif ++ koi_stack = ++ (void *)__get_free_pages(THREADINFO_GFP & ~__GFP_ACCOUNT, 3); ++ free_pages(koi_stack + 4 * PAGE_SIZE, 2); ++ printk(KERN_ERR "alloc dstack start=0x%16llx, end=0x%16llx\n", ++ koi_stack, koi_stack + THREAD_SIZE); ++#ifndef CONFIG_IEE ++ token_addr->koi_stack = ++ (struct pt_regs *)(THREAD_SIZE + (unsigned long)koi_stack) - 1; ++ token_addr->koi_stack_base = koi_stack; ++#else ++ iee_rw_gate( ++ IEE_WRITE_KOI_STACK, current, ++ (unsigned long)((struct pt_regs *)(THREAD_SIZE + ++ (unsigned long)koi_stack) - ++ 1)); ++ iee_rw_gate(IEE_WRITE_KOI_STACK_BASE, current, ++ (unsigned long)koi_stack); ++#endif ++ rcu_read_lock(); ++ hash_for_each_possible_rcu (koi_mem_htbl, target, node, ++ (unsigned long)mod) { ++ if (target->mod == mod) { ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ if (target == NULL) { ++ printk(KERN_ERR "mem node for module: %s not found\n", ++ mod->name); ++ return; ++ } ++ koi_copy_pagetable(target->ko_mm, target->pgdp, ++ (unsigned long)koi_stack, ++ (unsigned long)koi_stack + THREAD_SIZE); ++ printk(KERN_ERR "create ko stack: 0x%16llx\n", ++ (unsigned long)koi_stack); ++} ++EXPORT_SYMBOL(koi_map_kostack); +\ No newline at end of file +diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c +index cea96ee75d22..cbddc8e464e4 100644 +--- a/arch/arm64/kernel/mte.c ++++ b/arch/arm64/kernel/mte.c +@@ -79,8 +79,13 @@ int memcmp_pages(struct page *page1, struct page *page2) + static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) + { + /* Enable MTE Sync Mode for EL1. */ ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_TCF_MASK, ++ SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf)); ++#else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK, + SYS_FIELD_PREP(SCTLR_EL1, TCF, tcf)); ++#endif + isb(); + + pr_info_once("MTE: enabled in %s mode at EL1\n", mode); +diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c +index 068e5bb2661b..c98bc1a24fc7 100644 +--- a/arch/arm64/kernel/process.c ++++ b/arch/arm64/kernel/process.c +@@ -467,11 +467,24 @@ static void ssbs_thread_switch(struct task_struct *next) + * This is *only* for exception entry from EL0, and is not valid until we + * __switch_to() a user task. + */ ++#ifdef CONFIG_IEE ++// Put __entry_task in a isolated page to protect it. ++__attribute__((aligned(PAGE_SIZE))) DEFINE_PER_CPU(struct task_struct *[PAGE_SIZE/sizeof(struct task_struct *)], __entry_task); ++extern void iee_write_entry_task(struct task_struct *tsk); ++#else + DEFINE_PER_CPU(struct task_struct *, __entry_task); ++#endif + + static void entry_task_switch(struct task_struct *next) + { ++ #ifdef CONFIG_IEE ++ if(next == &init_task) ++ iee_write_entry_task((struct task_struct *)__va(__pa_symbol(next))); ++ else ++ iee_write_entry_task(next); ++ #else + __this_cpu_write(__entry_task, next); ++ #endif + } + + /* +@@ -506,11 +519,15 @@ static void erratum_1418040_new_exec(void) + */ + void update_sctlr_el1(u64 sctlr) + { +- /* ++ /* + * EnIA must not be cleared while in the kernel as this is necessary for + * in-kernel PAC. It will be cleared on kernel exit if needed. + */ ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); ++ #else + sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr); ++ #endif + + /* ISB required for the kernel uaccess routines when setting TCF0. */ + isb(); +diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c +index 58a97861bfc5..c7839247327d 100644 +--- a/arch/arm64/kernel/proton-pack.c ++++ b/arch/arm64/kernel/proton-pack.c +@@ -551,7 +551,11 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void) + return state; + + if (spectre_v4_mitigations_off()) { ++#ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, 0, SCTLR_ELx_DSSBS); ++#else + sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS); ++#endif + set_pstate_ssbs(1); + return SPECTRE_VULNERABLE; + } +@@ -975,7 +979,11 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) + if (arm64_kernel_unmapped_at_el0()) + return; + ++#ifdef CONFIG_IEE ++ iee_rwx_gate_entry(IEE_WRITE_vbar_el1, v); ++#else + write_sysreg(v, vbar_el1); ++#endif + isb(); + } + +diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c +index 95cb22c083c8..df73a583a733 100644 +--- a/arch/arm64/kernel/setup.c ++++ b/arch/arm64/kernel/setup.c +@@ -33,6 +33,11 @@ + #include <linux/scs.h> + #include <linux/mm.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#include <asm/iee-si.h> ++#endif ++ + #include <asm/acpi.h> + #include <asm/fixmap.h> + #include <asm/cpu.h> +@@ -335,10 +340,30 @@ u64 cpu_logical_map(unsigned int cpu) + return __cpu_logical_map[cpu]; + } + ++#ifdef CONFIG_IEE ++/* used for secure modification of vbar*/ ++extern char __bp_harden_el1_vectors[]; ++/* prepare iee rwx gate for senario of ttbr1=init_pg_dir */ ++static void __init iee_si_init_early(void) ++{ ++ /* prepare data used for iee rwx gate. */ ++ iee_base_idmap_pg_dir = phys_to_ttbr(__pa_symbol(idmap_pg_dir)); ++ iee_base_reserved_pg_dir = phys_to_ttbr(__pa_symbol(reserved_pg_dir)) ++ | FIELD_PREP(TTBR_ASID_MASK, 1); ++ iee_base__bp_harden_el1_vectors = (unsigned long)__bp_harden_el1_vectors; ++ iee_si_tcr = 0; ++} ++#endif ++ + void __init __no_sanitize_address setup_arch(char **cmdline_p) + { + setup_initial_init_mm(_stext, _etext, _edata, _end); + ++ #ifdef CONFIG_IEE ++ init_new_context(&init_task, &init_mm); ++ atomic64_set(&init_mm.context.id, (1UL << get_cpu_asid_bits()) | INIT_ASID); ++ #endif ++ + *cmdline_p = boot_command_line; + + kaslr_init(); +@@ -371,6 +396,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) + */ + local_daif_restore(DAIF_PROCCTX_NOIRQ); + ++#ifdef CONFIG_IEE ++ /* ++ * Map iee si codes to init_pg_dir to run the following ++ * cpu_uninstall_idmap() which writes ttbr0. ++ */ ++ iee_si_init_early(); ++#endif ++ + /* + * TTBR0 is only used for the identity mapping at this stage. Make it + * point to zero page to avoid speculatively fetching new entries. +diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c +index b7b7afb4a8c7..168a9390d6e9 100644 +--- a/arch/arm64/kernel/traps.c ++++ b/arch/arm64/kernel/traps.c +@@ -902,6 +902,32 @@ const char *esr_get_class_string(unsigned long esr) + return esr_class_str[ESR_ELx_EC(esr)]; + } + ++#ifdef CONFIG_IEE ++extern void arm64_enter_nmi(struct pt_regs *regs); ++static const char *handler[]= { ++ "SP_EL0", ++ "ELR_EL1", ++ "TCR_EL1", ++ "TTBR0 ASID" ++ "IEE_SI" ++}; ++ ++asmlinkage void notrace iee_bad_mode(struct pt_regs *regs, int reason, unsigned int esr) ++{ ++ arm64_enter_nmi(regs); ++ ++ console_verbose(); ++ ++ pr_crit("IEE : Bad mode in %s check detected on CPU%d, code 0x%08x -- %s\n", ++ handler[reason], smp_processor_id(), esr, ++ esr_get_class_string(esr)); ++ ++ __show_regs(regs); ++ local_daif_mask(); ++ panic("bad mode"); ++} ++#endif ++ + /* + * bad_el0_sync handles unexpected, but potentially recoverable synchronous + * exceptions taken from EL0. +diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S +index 3cd7e76cc562..287eaba7a15b 100644 +--- a/arch/arm64/kernel/vmlinux.lds.S ++++ b/arch/arm64/kernel/vmlinux.lds.S +@@ -134,6 +134,52 @@ jiffies = jiffies_64; + #define UNWIND_DATA_SECTIONS + #endif + ++#ifdef CONFIG_IEE ++#define IEE_TEXT \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_code_start = .; \ ++ *(.iee.text.header) \ ++ *(.iee.text) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_code_end = .; ++#else ++#define IEE_TEXT ++#endif ++ ++#ifdef CONFIG_IEE ++#define IEE_SI_TEXT \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_data_start = .; \ ++ *(.iee.si_data) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_exec_entry_start = .; \ ++ __iee_si_no_irq = . + (16); \ ++ *(.iee.exec_entry) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __iee_si_start = .; \ ++ *(.iee.si_text) \ ++ . = ALIGN(PAGE_SIZE); \ ++ . += PAGE_SIZE - (24); \ ++ __iee_si_end = . + (24); \ ++ __iee_exec_exit = .; \ ++ *(.iee.exec_exit) \ ++ . = ALIGN(PAGE_SIZE); ++ ++#else ++#define IEE_SI_TEXT ++#endif ++ ++#ifdef CONFIG_KOI ++#define KOI_TEXT \ ++ . = ALIGN(PAGE_SIZE); \ ++ __koi_code_start = .; \ ++ *(.koi.text) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __koi_code_end = .; ++#else ++#define KOI_TEXT ++#endif ++ + /* + * The size of the PE/COFF section that covers the kernel image, which + * runs from _stext to _edata, must be a round multiple of the PE/COFF +@@ -176,10 +222,13 @@ SECTIONS + SOFTIRQENTRY_TEXT + ENTRY_TEXT + TEXT_TEXT ++ IEE_TEXT + SCHED_TEXT + LOCK_TEXT + KPROBES_TEXT + HYPERVISOR_TEXT ++ IEE_SI_TEXT ++ KOI_TEXT + *(.gnu.warning) + } + +@@ -318,6 +367,18 @@ SECTIONS + . += INIT_DIR_SIZE; + init_pg_end = .; + ++ #ifdef CONFIG_IEE ++ . = ALIGN(PAGE_SIZE*8); ++ init_iee_stack_begin = .; ++ . += PAGE_SIZE*4; ++ init_iee_stack_end = .; ++ ++ . = ALIGN(PAGE_SIZE); ++ init_iee_si_stack_begin = .; ++ . += PAGE_SIZE*4; ++ init_iee_si_stack_end = .; ++ #endif ++ + . = ALIGN(SEGMENT_ALIGN); + __pecoff_data_size = ABSOLUTE(. - __initdata_begin); + _end = .; +diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c +index 188197590fc9..97e1f86046cf 100644 +--- a/arch/arm64/mm/context.c ++++ b/arch/arm64/mm/context.c +@@ -17,6 +17,10 @@ + #include <asm/smp.h> + #include <asm/tlbflush.h> + ++#ifdef CONFIG_IEE ++#include <asm/iee-si.h> ++#endif ++ + static u32 asid_bits; + static DEFINE_RAW_SPINLOCK(cpu_asid_lock); + +@@ -39,7 +43,11 @@ static unsigned long *pinned_asid_map; + #define asid2ctxid(asid, genid) ((asid) | (genid)) + + /* Get the ASIDBits supported by the current CPU */ ++#ifdef CONFIG_IEE ++u32 get_cpu_asid_bits(void) ++#else + static u32 get_cpu_asid_bits(void) ++#endif + { + u32 asid; + int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1), +@@ -212,6 +220,38 @@ static u64 new_context(struct mm_struct *mm) + return asid2ctxid(asid, generation); + } + ++#ifdef CONFIG_KOI ++/* ++ * This function is used to check and allocate ASID for ko's pgd ++ * The mm MUST point to the isolated kos' mm_struct, other behaviours are undefined. ++ */ ++void koi_check_and_switch_context(struct mm_struct *mm) { ++ u64 asid = atomic64_read(&mm->context.id); ++ u64 old_active_asid; ++ unsigned long flags; ++ unsigned int cpu; ++ ++ old_active_asid = atomic64_read(this_cpu_ptr(&active_asids)); ++ if (old_active_asid && asid_gen_match(asid) && atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_asids), old_active_asid, asid)) { ++ return; ++ } ++ ++ raw_spin_lock_irqsave(&cpu_asid_lock, flags); ++ asid = atomic64_read(&mm->context.id); ++ if (!asid_gen_match(asid)) { ++ asid = new_context(mm); ++ atomic64_set(&mm->context.id, asid); ++ } ++ ++ cpu = smp_processor_id(); ++ if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) ++ local_flush_tlb_all(); ++ ++ atomic64_set(this_cpu_ptr(&active_asids), asid); ++ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); ++} ++#endif ++ + void check_and_switch_context(struct mm_struct *mm) + { + unsigned long flags; +@@ -348,7 +388,9 @@ asmlinkage void post_ttbr_update_workaround(void) + + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) + { ++ #ifndef CONFIG_IEE + unsigned long ttbr1 = read_sysreg(ttbr1_el1); ++ #endif + unsigned long asid = ASID(mm); + unsigned long ttbr0 = phys_to_ttbr(pgd_phys); + +@@ -360,14 +402,28 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) + if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN)) + ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); + +- /* Set ASID in TTBR1 since TCR.A1 is set */ ++ #ifdef CONFIG_IEE ++ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid+1); ++ iee_rwx_gate_entry(IEE_CONTEXT_SWITCH, ttbr0); ++ // TODO : if defined CONFIG_IEE and defined CONFIG_KOI ++ #else ++ /* Set ASID in TTBR0 since TCR.A1 is set 0*/ ++ ++ #ifdef CONFIG_KOI ++ ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid+1); ++ ttbr1 &= ~TTBR_ASID_MASK; ++ ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); ++ #else + ttbr1 &= ~TTBR_ASID_MASK; + ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); +- ++ ++ #endif + cpu_set_reserved_ttbr0_nosync(); + write_sysreg(ttbr1, ttbr1_el1); + write_sysreg(ttbr0, ttbr0_el1); + isb(); ++ #endif ++ + post_ttbr_update_workaround(); + } + +@@ -375,11 +431,21 @@ static int asids_update_limit(void) + { + unsigned long num_available_asids = NUM_USER_ASIDS; + +- if (arm64_kernel_unmapped_at_el0()) { +- num_available_asids /= 2; +- if (pinned_asid_map) +- set_kpti_asid_bits(pinned_asid_map); +- } ++ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ num_available_asids /= 2; ++ if (pinned_asid_map) { ++ unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); ++ memset(pinned_asid_map, 0xaa, len); ++ __set_bit(INIT_ASID, pinned_asid_map); ++ } ++ #else ++ if (arm64_kernel_unmapped_at_el0()) { ++ num_available_asids /= 2; ++ if (pinned_asid_map) ++ set_kpti_asid_bits(pinned_asid_map); ++ } ++ #endif ++ + /* + * Expect allocation after rollover to fail if we don't have at least + * one more ASID than CPUs. ASID #0 is reserved for init_mm. +@@ -400,6 +466,10 @@ arch_initcall(asids_update_limit); + + static int asids_init(void) + { ++ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ unsigned int len; ++ #endif ++ + asid_bits = get_cpu_asid_bits(); + atomic64_set(&asid_generation, ASID_FIRST_VERSION); + asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); +@@ -410,6 +480,11 @@ static int asids_init(void) + pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); + nr_pinned_asids = 0; + ++ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); ++ memset(asid_map, 0xaa, len); ++ __set_bit(INIT_ASID, asid_map); ++ #else + /* + * We cannot call set_reserved_asid_bits() here because CPU + * caps are not finalized yet, so it is safer to assume KPTI +@@ -417,6 +492,8 @@ static int asids_init(void) + */ + if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) + set_kpti_asid_bits(asid_map); ++ #endif ++ + return 0; + } + early_initcall(asids_init); +diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c +index 4ea07caba71c..de1d57c2b30f 100644 +--- a/arch/arm64/mm/fault.c ++++ b/arch/arm64/mm/fault.c +@@ -261,7 +261,11 @@ int __ptep_set_access_flags(struct vm_area_struct *vma, + pteval ^= PTE_RDONLY; + pteval |= pte_val(entry); + pteval ^= PTE_RDONLY; ++ #ifdef CONFIG_PTP ++ pteval = iee_set_cmpxchg_relaxed(ptep, old_pteval, pteval); ++ #else + pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval); ++ #endif + } while (pteval != old_pteval); + + /* Invalidate a stale read-only entry */ +@@ -376,8 +380,13 @@ static void do_tag_recovery(unsigned long addr, unsigned long esr, + * It will be done lazily on the other CPUs when they will hit a + * tag fault. + */ ++ #ifdef CONFIG_IEE ++ sysreg_clear_set_iee_si(sctlr_el1, SCTLR_EL1_TCF_MASK, ++ SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE)); ++ #else + sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF_MASK, + SYS_FIELD_PREP_ENUM(SCTLR_EL1, TCF, NONE)); ++ #endif + isb(); + } + +diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c +index bfc02568805a..580ecb596d2d 100644 +--- a/arch/arm64/mm/fixmap.c ++++ b/arch/arm64/mm/fixmap.c +@@ -32,6 +32,22 @@ static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __page_aligned_bss; + static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; + static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused; + ++#ifdef CONFIG_IEE ++void *bm_pte_addr = (void *)bm_pte; ++void *bm_pmd_addr = (void *)bm_pmd; ++void *bm_pud_addr = (void *)bm_pud; ++#endif ++ ++#ifdef CONFIG_PTP ++extern void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot); ++extern void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot); ++extern void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, ++ pmdval_t prot); ++ ++extern void iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d); ++#define set_pgd_init(pgdptr, pgdval) iee_set_p4d_pre_init((p4d_t *)(pgdptr), (p4d_t) { pgdval }) ++#endif ++ + static inline pte_t *fixmap_pte(unsigned long addr) + { + return &bm_pte[BM_PTE_TABLE_IDX(addr)][pte_index(addr)]; +@@ -44,7 +60,11 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr) + + if (pmd_none(pmd)) { + ptep = bm_pte[BM_PTE_TABLE_IDX(addr)]; ++ #ifdef CONFIG_PTP ++ __iee_pmd_populate_pre_init(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE); ++ #else + __pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE); ++ #endif + } + } + +@@ -55,8 +75,13 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr, + pud_t pud = READ_ONCE(*pudp); + pmd_t *pmdp; + +- if (pud_none(pud)) ++ if (pud_none(pud)) { ++ #ifdef CONFIG_PTP ++ __iee_pud_populate_pre_init(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); ++ #else + __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE); ++ #endif ++ } + + pmdp = pmd_offset_kimg(pudp, addr); + do { +@@ -82,8 +107,13 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, + BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); + } + +- if (p4d_none(p4d)) ++ if (p4d_none(p4d)) { ++ #ifdef CONFIG_PTP ++ __iee_p4d_populate_pre_init(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); ++ #else + __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE); ++ #endif ++ } + + pudp = pud_offset_kimg(p4dp, addr); + early_fixmap_init_pmd(pudp, addr, end); +@@ -106,6 +136,27 @@ void __init early_fixmap_init(void) + early_fixmap_init_pud(p4dp, addr, end); + } + ++#ifdef CONFIG_PTP ++extern void iee_set_pte_pre_init(pte_t *ptep, pte_t pte); ++void __iee_set_fixmap_pre_init(enum fixed_addresses idx, ++ phys_addr_t phys, pgprot_t flags) ++{ ++ unsigned long addr = __fix_to_virt(idx); ++ pte_t *ptep; ++ ++ BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); ++ ++ ptep = fixmap_pte(addr); ++ ++ if (pgprot_val(flags)) { ++ iee_set_pte_pre_init(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); ++ } else { ++ iee_set_pte_pre_init(ptep, __pte(0)); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ } ++} ++#endif ++ + /* + * Unusually, this is also called in IRQ context (ghes_iounmap_irq) so if we + * ever need to use IPIs for TLB broadcasting, then we're in trouble here. +@@ -121,9 +172,17 @@ void __set_fixmap(enum fixed_addresses idx, + ptep = fixmap_pte(addr); + + if (pgprot_val(flags)) { ++ #ifdef CONFIG_PTP ++ iee_set_bm_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); ++ #else + __set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); ++ #endif + } else { ++ #ifdef CONFIG_PTP ++ iee_set_bm_pte(ptep, __pte(0)); ++ #else + __pte_clear(&init_mm, addr, ptep); ++ #endif + flush_tlb_kernel_range(addr, addr+PAGE_SIZE); + } + } +@@ -179,8 +238,13 @@ void __init fixmap_copy(pgd_t *pgdir) + * live in the carveout for the swapper_pg_dir. We can simply + * re-use the existing dir for the fixmap. + */ ++ #ifdef CONFIG_PTP ++ set_pgd_init(pgd_offset_pgd(pgdir, FIXADDR_TOT_START), ++ READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START))); ++ #else + set_pgd(pgd_offset_pgd(pgdir, FIXADDR_TOT_START), + READ_ONCE(*pgd_offset_k(FIXADDR_TOT_START))); ++ #endif + } else if (CONFIG_PGTABLE_LEVELS > 3) { + pgd_t *bm_pgdp; + p4d_t *bm_p4dp; +@@ -194,9 +258,15 @@ void __init fixmap_copy(pgd_t *pgdir) + BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); + bm_pgdp = pgd_offset_pgd(pgdir, FIXADDR_TOT_START); + bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_TOT_START); ++ #ifdef CONFIG_PTP ++ bm_pudp = pud_set_fixmap_offset_init(bm_p4dp, FIXADDR_TOT_START); ++ __iee_pud_populate_pre_init(bm_pudp, __pa(lm_alias(bm_pmd)), PMD_TYPE_TABLE); ++ pud_clear_fixmap_init(); ++ #else + bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_TOT_START); + pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd)); + pud_clear_fixmap(); ++ #endif + } else { + BUG(); + } +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index 8c8d7653ba84..4190f5c10d68 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -55,8 +55,19 @@ + * that cannot be mistaken for a real physical address. + */ + s64 memstart_addr __ro_after_init = -1; ++#if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++s64 memstart_addr_init __ro_after_init = -1; ++#endif ++#ifdef CONFIG_KOI ++s64 koi_offset __ro_after_init = -1; ++EXPORT_SYMBOL(koi_offset); ++#endif ++#ifdef CONFIG_IEE ++s64 iee_offset __ro_after_init = -1; ++#endif + EXPORT_SYMBOL(memstart_addr); + ++ + /* + * If the corresponding config options are enabled, we create both ZONE_DMA + * and ZONE_DMA32. By default ZONE_DMA covers the 32-bit addressable memory +@@ -421,7 +432,11 @@ early_param("memmap", parse_memmap_opt); + + void __init arm64_memblock_init(void) + { ++ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ s64 linear_region_size = BIT(vabits_actual - 2); ++ #else + s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual); ++ #endif + + /* + * Corner case: 52-bit VA capable systems running KVM in nVHE mode may +@@ -438,13 +453,24 @@ void __init arm64_memblock_init(void) + } + + /* Remove memory above our supported physical address size */ ++ #ifdef CONFIG_IEE ++ // If config iee, phys size can not be above 0x400000000000 ++ if(__pa_symbol(_end) > BIT_ULL(vabits_actual - 2)) ++ panic("Image on too high phys mem.\n"); ++ else ++ memblock_remove(BIT_ULL(vabits_actual - 2), ULLONG_MAX); ++ #else + memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX); ++ #endif + + /* + * Select a suitable value for the base of physical memory. + */ + memstart_addr = round_down(memblock_start_of_DRAM(), + ARM64_MEMSTART_ALIGN); ++ #if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++ memstart_addr_init = memstart_addr; ++ #endif + + if ((memblock_end_of_DRAM() - memstart_addr) > linear_region_size) + pr_warn("Memory doesn't fit in the linear mapping, VA_BITS too small\n"); +@@ -531,6 +557,14 @@ void __init arm64_memblock_init(void) + ((range * memstart_offset_seed) >> 16); + } + } ++ ++ #ifdef CONFIG_KOI ++ koi_offset = memstart_addr - memstart_addr_init + KOI_OFFSET; ++ #endif ++ #ifdef CONFIG_IEE ++ iee_offset = memstart_addr - memstart_addr_init + ((unsigned long)BIT(vabits_actual - 2)); ++ #endif ++ //printk(KERN_ERR "koi_offset: 0x%16llx\n", koi_offset); + + /* + * Register the kernel text, kernel data, initrd, and initial +diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c +index 4142a75a414e..094f3798441d 100644 +--- a/arch/arm64/mm/mmu.c ++++ b/arch/arm64/mm/mmu.c +@@ -6,6 +6,7 @@ + * Copyright (C) 2012 ARM Ltd. + */ + ++#include "asm/pgtable.h" + #include <linux/cache.h> + #include <linux/export.h> + #include <linux/kernel.h> +@@ -40,6 +41,11 @@ + #include <asm/tlbflush.h> + #include <asm/pgalloc.h> + #include <asm/kfence.h> ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#include <asm/iee.h> ++#include <asm/iee-si.h> ++#endif + + #define NO_BLOCK_MAPPINGS BIT(0) + #define NO_CONT_MAPPINGS BIT(1) +@@ -76,8 +82,282 @@ EXPORT_SYMBOL(empty_zero_page); + static DEFINE_SPINLOCK(swapper_pgdir_lock); + static DEFINE_MUTEX(fixmap_lock); + ++#ifdef CONFIG_IEE ++extern struct cred init_cred; ++ ++extern unsigned long init_iee_stack_begin[]; ++extern unsigned long init_iee_stack_end[]; ++extern unsigned long __iee_si_data_start[]; ++extern unsigned long __iee_exec_entry_start[]; ++extern unsigned long __iee_si_start[]; ++extern unsigned long __iee_si_end[]; ++ ++extern void *bm_pte_addr; ++extern void *bm_pmd_addr; ++extern void *bm_pud_addr; ++ ++#ifdef CONFIG_PTP ++ ++/* Funcs to set pgtable before iee initialized. */ ++static void iee_set_swapper_pgd_pre_init(pgd_t *pgdp, pgd_t pgd) ++{ ++ pgd_t *fixmap_pgdp; ++ ++ spin_lock(&swapper_pgdir_lock); ++ fixmap_pgdp = pgd_set_fixmap_init(__pa_symbol(pgdp)); ++ WRITE_ONCE(*fixmap_pgdp, pgd); ++ /* ++ * We need dsb(ishst) here to ensure the page-table-walker sees ++ * our new entry before set_p?d() returns. The fixmap's ++ * flush_tlb_kernel_range() via clear_fixmap() does this for us. ++ */ ++ pgd_clear_fixmap_init(); ++ spin_unlock(&swapper_pgdir_lock); ++} ++ ++void iee_set_p4d_pre_init(p4d_t *p4dp, p4d_t p4d) ++{ ++ if (in_swapper_pgdir(p4dp)) { ++ iee_set_swapper_pgd_pre_init((pgd_t *)p4dp, __pgd(p4d_val(p4d))); ++ return; ++ } ++ ++ WRITE_ONCE(*p4dp, p4d); ++ dsb(ishst); ++ isb(); ++} ++ ++static inline void iee_set_pud_pre_init(pud_t *pudp, pud_t pud) ++{ ++#ifdef __PAGETABLE_PUD_FOLDED ++ if (in_swapper_pgdir(pudp)) { ++ iee_set_swapper_pgd_pre_init((pgd_t *)pudp, __pgd(pud_val(pud))); ++ return; ++ } ++#endif /* __PAGETABLE_PUD_FOLDED */ ++#ifdef CONFIG_KOI ++ pudval_t val = pud_val(pud); ++ if (pud_valid(pud) && !(val & PUD_TABLE_BIT)) { ++ // There is no PUD_SEC_NG, so we use PMD_SECT_NG instead. ++ pud = __pud(val | PMD_SECT_NG); ++ } ++#endif ++ WRITE_ONCE(*pudp, pud); ++ ++ if (pud_valid(pud)) { ++ dsb(ishst); ++ isb(); ++ } ++} ++ ++static inline void iee_set_pmd_pre_init(pmd_t *pmdp, pmd_t pmd) ++{ ++#ifdef __PAGETABLE_PMD_FOLDED ++ if (in_swapper_pgdir(pmdp)) { ++ iee_set_swapper_pgd_pre_init((pgd_t *)pmdp, __pgd(pmd_val(pmd))); ++ return; ++ } ++#endif /* __PAGETABLE_PMD_FOLDED */ ++#ifdef CONFIG_KOI ++ pmdval_t val = pmd_val(pmd); ++ if (pmd_valid(pmd) && !(val & PMD_TABLE_BIT)) { ++ pmd = __pmd(val | PMD_SECT_NG); ++ } ++#endif ++ WRITE_ONCE(*pmdp, pmd); ++ ++ if (pmd_valid(pmd)) { ++ dsb(ishst); ++ isb(); ++ } ++} ++ ++ ++void __iee_p4d_populate_pre_init(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) ++{ ++ iee_set_p4d_pre_init(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot)); ++} ++ ++void __iee_pud_populate_pre_init(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) ++{ ++ iee_set_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); ++} ++ ++void __iee_pmd_populate_pre_init(pmd_t *pmdp, phys_addr_t ptep, ++ pmdval_t prot) ++{ ++ iee_set_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); ++} ++ ++/* Funcs to set fixmap before iee initialized. */ ++bool pgattr_change_is_safe(u64 old, u64 new); ++static int iee_pud_set_huge_fixmap(pud_t *pudp, phys_addr_t phys, pgprot_t prot) ++{ ++ pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); ++ ++ /* Only allow permission changes for now */ ++ if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), ++ pud_val(new_pud))) ++ return 0; ++ ++ VM_BUG_ON(phys & ~PUD_MASK); ++ iee_set_fixmap_pud_pre_init(pudp, new_pud); ++ return 1; ++} ++ ++static int iee_pmd_set_huge_fixmap(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) ++{ ++ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); ++ ++ /* Only allow permission changes for now */ ++ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), ++ pmd_val(new_pmd))) ++ return 0; ++ ++ VM_BUG_ON(phys & ~PMD_MASK); ++ iee_set_fixmap_pmd_pre_init(pmdp, new_pmd); ++ return 1; ++} ++ ++static inline void __iee_pmd_populate_fixmap(pmd_t *pmdp, phys_addr_t ptep, ++ pmdval_t prot) ++{ ++ iee_set_fixmap_pmd_pre_init(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot)); ++} ++ ++static inline void __iee_pud_populate_fixmap(pud_t *pudp, phys_addr_t pmdp, pudval_t prot) ++{ ++ iee_set_fixmap_pud_pre_init(pudp, __pud(__phys_to_pud_val(pmdp) | prot)); ++} ++#endif /* END CONFIG_PTP*/ ++ ++void iee_set_pte_pre_init(pte_t *ptep, pte_t pte) ++{ ++#ifdef CONFIG_KOI ++ if (!pte_none(pte)) { ++ pte = __pte(pte_val(pte) | PTE_NG); ++ } ++#endif ++ WRITE_ONCE(*ptep, pte); ++ ++ /* ++ * Only if the new pte is valid and kernel, otherwise TLB maintenance ++ * or update_mmu_cache() have the necessary barriers. ++ */ ++ if (pte_valid_not_user(pte)) { ++ dsb(ishst); ++ isb(); ++ } ++} ++ ++static void __init iee_set_token_page_valid_pre_init(void *token, void *new) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, (unsigned long)token); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, (unsigned long)token); ++ ++ pud_t *pudp = pud_offset(p4dp, (unsigned long)token); ++ ++ pmd_t *pmdp = pmd_offset(pudp, (unsigned long)token); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, (unsigned long)token); ++ pte_t pte = READ_ONCE(*ptep); ++ pte = __pte(((pte_val(pte) | 0x1) & ~PTE_ADDR_MASK) | __phys_to_pte_val(__pa(new))); ++ iee_set_pte_pre_init(ptep, pte); ++ flush_tlb_kernel_range((unsigned long)token, (unsigned long)(token+PAGE_SIZE)); ++ isb(); ++} ++#endif /* END CONFIG_IEE*/ ++ ++#if defined(CONFIG_KOI) && !defined(CONFIG_IEE) ++int koi_add_page_mapping(unsigned long dst, unsigned long src) ++{ ++ pgd_t *src_pgdp, *dst_pgdp; ++ p4d_t *src_p4dp, *dst_p4dp; ++ pud_t *src_pudp, *dst_pudp; ++ pmd_t *src_pmdp, *dst_pmdp; ++ pte_t *src_ptep, *dst_ptep; ++ ++ src_pgdp = pgd_offset_pgd(swapper_pg_dir, src); ++ dst_pgdp = pgd_offset_pgd(swapper_pg_dir, dst); ++ ++ src_p4dp = p4d_offset(src_pgdp, src); ++ dst_p4dp = p4d_alloc(&init_mm, dst_pgdp, dst); ++ if (!dst_p4dp) { ++ return -ENOMEM; ++ } ++ src_pudp = pud_offset(src_p4dp, src); ++ dst_pudp = pud_alloc(&init_mm, dst_p4dp, dst); ++ if (!dst_pudp) { ++ return -ENOMEM; ++ } ++ if (pud_val(*src_pudp) & PMD_TABLE_BIT) { ++ src_pmdp = pmd_offset(src_pudp, src); ++ dst_pmdp = pmd_alloc(&init_mm, dst_pudp, dst); ++ if (!dst_pmdp) { ++ return -ENOMEM; ++ } ++ if (pmd_val(*src_pmdp) & PMD_TABLE_BIT) { ++ src_ptep = pte_offset_kernel(src_pmdp, src); ++ dst_ptep = pte_alloc_map(&init_mm, dst_pmdp, dst); ++ set_pte(dst_ptep, *src_ptep); ++ } else { ++ set_pte((pte_t *)dst_pmdp, pmd_pte(*src_pmdp)); ++ } ++ } else { ++ set_pte((pte_t *)dst_pudp, pud_pte(*src_pudp)); ++ } ++ ++ ++ flush_tlb_kernel_range(dst, dst+PAGE_SIZE); ++ isb(); ++ return 0; ++} ++ ++void koi_remove_page_mapping(unsigned long addr) { ++ pgd_t *src_pgdp; ++ p4d_t *src_p4dp; ++ pud_t *src_pudp; ++ pmd_t *src_pmdp; ++ pte_t *src_ptep; ++ ++ src_pgdp = pgd_offset_pgd(swapper_pg_dir, addr); ++ if (pgd_none(*src_pgdp) || pgd_bad(*src_pgdp)) ++ return; ++ src_p4dp = p4d_offset(src_pgdp, addr); ++ if (p4d_none(*src_p4dp) || p4d_bad(*src_p4dp)) ++ return; ++ src_pudp = pud_offset(src_p4dp, addr); ++ if (pud_none(*src_pudp)) ++ return; ++ if (pud_val(*src_pudp) & PMD_TABLE_BIT) { ++ src_pmdp = pmd_offset(src_pudp, addr); ++ if (pmd_none(*src_pmdp)) ++ return; ++ if (pmd_val(*src_pmdp) & PMD_TABLE_BIT) { ++ src_ptep = pte_offset_kernel(src_pmdp, addr); ++ if(!pte_none(*src_ptep)) ++ pte_clear(&init_mm, addr, src_ptep); ++ } else { ++ pmd_clear(src_pmdp); ++ } ++ } else { ++ pud_clear(src_pudp); ++ } ++ ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ isb(); ++} ++#endif ++ + void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) + { ++ #ifdef CONFIG_PTP ++ spin_lock(&swapper_pgdir_lock); ++ iee_rw_gate(IEE_OP_SET_SWAPPER_PGD, pgdp, pgd); ++ spin_unlock(&swapper_pgdir_lock); ++ #else + pgd_t *fixmap_pgdp; + + spin_lock(&swapper_pgdir_lock); +@@ -90,6 +370,7 @@ void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) + */ + pgd_clear_fixmap(); + spin_unlock(&swapper_pgdir_lock); ++ #endif + } + + pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, +@@ -118,7 +399,11 @@ static phys_addr_t __init early_pgtable_alloc(int shift) + * slot will be free, so we can (ab)use the FIX_PTE slot to initialise + * any level of table. + */ ++ #ifdef CONFIG_PTP ++ ptr = pte_set_fixmap_init(phys); ++ #else + ptr = pte_set_fixmap(phys); ++ #endif + + memset(ptr, 0, PAGE_SIZE); + +@@ -126,11 +411,16 @@ static phys_addr_t __init early_pgtable_alloc(int shift) + * Implicit barriers also ensure the zeroed page is visible to the page + * table walker + */ ++ #ifdef CONFIG_PTP ++ pte_clear_fixmap_init(); ++ #else + pte_clear_fixmap(); ++ #endif + + return phys; + } + ++ + bool pgattr_change_is_safe(u64 old, u64 new) + { + /* +@@ -178,7 +468,11 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end, + do { + pte_t old_pte = __ptep_get(ptep); + ++ #ifdef CONFIG_PTP ++ iee_set_fixmap_pte_pre_init(ptep, pfn_pte(__phys_to_pfn(phys), prot)); ++ #else + __set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot)); ++ #endif + + /* + * After the PTE entry has been populated once, we +@@ -211,7 +505,11 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr, + pmdval |= PMD_TABLE_PXN; + BUG_ON(!pgtable_alloc); + pte_phys = pgtable_alloc(PAGE_SHIFT); ++ #ifdef CONFIG_PTP ++ __iee_pmd_populate_fixmap(pmdp, pte_phys, pmdval); ++ #else + __pmd_populate(pmdp, pte_phys, pmdval); ++ #endif + pmd = READ_ONCE(*pmdp); + } + BUG_ON(pmd_bad(pmd)); +@@ -248,7 +546,11 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, + /* try section mapping first */ + if (((addr | next | phys) & ~PMD_MASK) == 0 && + (flags & NO_BLOCK_MAPPINGS) == 0) { ++ #ifdef CONFIG_PTP ++ iee_pmd_set_huge_fixmap(pmdp, phys, prot); ++ #else + pmd_set_huge(pmdp, phys, prot); ++ #endif + + /* + * After the PMD entry has been populated once, we +@@ -289,7 +591,11 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, + pudval |= PUD_TABLE_PXN; + BUG_ON(!pgtable_alloc); + pmd_phys = pgtable_alloc(PMD_SHIFT); +- __pud_populate(pudp, pmd_phys, pudval); ++ #ifdef CONFIG_PTP ++ __iee_pud_populate_fixmap(pudp, pmd_phys, PUD_TYPE_TABLE); ++ #else ++ __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); ++ #endif + pud = READ_ONCE(*pudp); + } + BUG_ON(pud_bad(pud)); +@@ -345,7 +651,11 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, + if (pud_sect_supported() && + ((addr | next | phys) & ~PUD_MASK) == 0 && + (flags & NO_BLOCK_MAPPINGS) == 0) { ++ #ifdef CONFIG_PTP ++ iee_pud_set_huge_fixmap(pudp, phys, prot); ++ #else + pud_set_huge(pudp, phys, prot); ++ #endif + + /* + * After the PUD entry has been populated once, we +@@ -374,6 +684,10 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, + { + unsigned long addr, end, next; + pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); ++ #ifdef CONFIG_IEE ++ p4d_t *p4dp; ++ p4d_t p4d; ++ #endif + + /* + * If the virtual and physical address don't have the same offset +@@ -390,10 +704,14 @@ static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, + next = pgd_addr_end(addr, end); + alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, + flags); ++ #ifdef CONFIG_IEE ++ p4dp = p4d_offset(pgdp, addr); ++ p4d = READ_ONCE(*p4dp); ++ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APT | PUD_TYPE_TABLE)); ++ #endif + phys += next - addr; + } while (pgdp++, addr = next, addr != end); + } +- + static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, +@@ -413,162 +731,844 @@ void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, + phys_addr_t (*pgtable_alloc)(int), int flags); + #endif + +-static phys_addr_t __pgd_pgtable_alloc(int shift) ++#ifdef CONFIG_PTP ++static int __init iee_pud_set_huge_pre_init(pud_t *pudp, phys_addr_t phys, pgprot_t prot) + { +- void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); +- BUG_ON(!ptr); ++ pud_t new_pud = pfn_pud(__phys_to_pfn(phys), mk_pud_sect_prot(prot)); + +- /* Ensure the zeroed page is visible to the page table walker */ +- dsb(ishst); +- return __pa(ptr); ++ /* Only allow permission changes for now */ ++ if (!pgattr_change_is_safe(READ_ONCE(pud_val(*pudp)), ++ pud_val(new_pud))) ++ return 0; ++ ++ VM_BUG_ON(phys & ~PUD_MASK); ++ iee_set_pud_pre_init(pudp, new_pud); ++ return 1; + } + +-static phys_addr_t pgd_pgtable_alloc(int shift) ++static int __init iee_pmd_set_huge_pre_init(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot) + { +- phys_addr_t pa = __pgd_pgtable_alloc(shift); +- struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); +- +- /* +- * Call proper page table ctor in case later we need to +- * call core mm functions like apply_to_page_range() on +- * this pre-allocated page table. +- * +- * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is +- * folded, and if so pagetable_pte_ctor() becomes nop. +- */ +- if (shift == PAGE_SHIFT) +- BUG_ON(!pagetable_pte_ctor(ptdesc)); +- else if (shift == PMD_SHIFT) +- BUG_ON(!pagetable_pmd_ctor(ptdesc)); ++ pmd_t new_pmd = pfn_pmd(__phys_to_pfn(phys), mk_pmd_sect_prot(prot)); + +- return pa; +-} ++ /* Only allow permission changes for now */ ++ if (!pgattr_change_is_safe(READ_ONCE(pmd_val(*pmdp)), ++ pmd_val(new_pmd))) ++ return 0; + +-/* +- * This function can only be used to modify existing table entries, +- * without allocating new levels of table. Note that this permits the +- * creation of new section or page entries. +- */ +-void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, +- phys_addr_t size, pgprot_t prot) +-{ +- if (virt < PAGE_OFFSET) { +- pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", +- &phys, virt); +- return; +- } +- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, +- NO_CONT_MAPPINGS); ++ VM_BUG_ON(phys & ~PMD_MASK); ++ iee_set_pmd_pre_init(pmdp, new_pmd); ++ return 1; + } + +-void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, +- unsigned long virt, phys_addr_t size, +- pgprot_t prot, bool page_mappings_only) ++static __init void iee_init_pte_pre_init(pmd_t *pmdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot) + { +- int flags = 0; +- +- BUG_ON(mm == &init_mm); ++ pte_t *ptep; + +- if (page_mappings_only) +- flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; ++ ptep = pte_set_fixmap_offset_init(pmdp, addr); ++ do { ++ pte_t old_pte = READ_ONCE(*ptep); + +- __create_pgd_mapping(mm->pgd, phys, virt, size, prot, +- pgd_pgtable_alloc, flags); +-} ++ iee_set_pte_pre_init(ptep, pfn_pte(__phys_to_pfn(phys), prot)); + +-static void update_mapping_prot(phys_addr_t phys, unsigned long virt, +- phys_addr_t size, pgprot_t prot) +-{ +- if (virt < PAGE_OFFSET) { +- pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", +- &phys, virt); +- return; +- } ++ /* ++ * After the PTE entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), ++ READ_ONCE(pte_val(*ptep)))); + +- __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, +- NO_CONT_MAPPINGS); ++ phys += PAGE_SIZE; ++ } while (ptep++, addr += PAGE_SIZE, addr != end); + +- /* flush the TLBs after updating live kernel mappings */ +- flush_tlb_kernel_range(virt, virt + size); ++ pte_clear_fixmap_init(); + } + +-static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, +- phys_addr_t end, pgprot_t prot, int flags) ++static __init void iee_alloc_init_cont_pte_pre_init(pmd_t *pmdp, unsigned long addr, ++ unsigned long end, phys_addr_t phys, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) + { +- __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, +- prot, early_pgtable_alloc, flags); +-} ++ unsigned long next; ++ pmd_t pmd = READ_ONCE(*pmdp); + +-void __init mark_linear_text_alias_ro(void) +-{ +- /* +- * Remove the write permissions from the linear alias of .text/.rodata +- */ +- update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), +- (unsigned long)__init_begin - (unsigned long)_stext, +- PAGE_KERNEL_RO); +-} ++ BUG_ON(pmd_sect(pmd)); ++ if (pmd_none(pmd)) { ++ pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN; ++ phys_addr_t pte_phys; + +-#ifdef CONFIG_KFENCE ++ if (flags & NO_EXEC_MAPPINGS) ++ pmdval |= PMD_TABLE_PXN; ++ BUG_ON(!pgtable_alloc); ++ pte_phys = pgtable_alloc(PAGE_SHIFT); ++ __iee_pmd_populate_pre_init(pmdp, pte_phys, pmdval); ++ pmd = READ_ONCE(*pmdp); ++ } ++ BUG_ON(pmd_bad(pmd)); + +-bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL; ++ do { ++ pgprot_t __prot = prot; + +-/* early_param() will be parsed before map_mem() below. */ +-static int __init parse_kfence_early_init(char *arg) +-{ +- int val; ++ next = pte_cont_addr_end(addr, end); + +- if (get_option(&arg, &val)) +- kfence_early_init = !!val; ++ /* use a contiguous mapping if the range is suitably aligned */ ++ if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) && ++ (flags & NO_CONT_MAPPINGS) == 0) ++ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); + +-#if IS_ENABLED(CONFIG_KFENCE_MUST_EARLY_INIT) +- kfence_must_early_init = (val == -1) ? true : false; +-#endif ++ iee_init_pte_pre_init(pmdp, addr, next, phys, __prot); + +- return 0; ++ phys += next - addr; ++ } while (addr = next, addr != end); + } +-early_param("kfence.sample_interval", parse_kfence_early_init); + +-static phys_addr_t __init arm64_kfence_alloc_pool(void) ++static __init void iee_init_pmd_pre_init(pud_t *pudp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags) + { +- phys_addr_t kfence_pool; ++ unsigned long next; ++ pmd_t *pmdp; + +- if (!kfence_early_init) +- return 0; ++ pmdp = pmd_set_fixmap_offset_init(pudp, addr); ++ do { ++ pmd_t old_pmd = READ_ONCE(*pmdp); + +- kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); +- if (!kfence_pool) { +- pr_err("failed to allocate kfence pool\n"); +- kfence_early_init = false; +- return 0; +- } ++ next = pmd_addr_end(addr, end); + +- /* Temporarily mark as NOMAP. */ +- memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); ++ /* try section mapping first */ ++ if (((addr | next | phys) & ~PMD_MASK) == 0 && ++ (flags & NO_BLOCK_MAPPINGS) == 0) { ++ iee_pmd_set_huge_pre_init(pmdp, phys, prot); ++ ++ /* ++ * After the PMD entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd), ++ READ_ONCE(pmd_val(*pmdp)))); ++ } else { ++ iee_alloc_init_cont_pte_pre_init(pmdp, addr, next, phys, prot, ++ pgtable_alloc, flags); ++ ++ BUG_ON(pmd_val(old_pmd) != 0 && ++ pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp))); ++ } ++ phys += next - addr; ++ } while (pmdp++, addr = next, addr != end); ++ ++ pmd_clear_fixmap_init(); ++} ++ ++static __init void iee_alloc_init_cont_pmd_pre_init(pud_t *pudp, unsigned long addr, ++ unsigned long end, phys_addr_t phys, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), int flags) ++{ ++ unsigned long next; ++ pud_t pud = READ_ONCE(*pudp); ++ ++ /* ++ * Check for initial section mappings in the pgd/pud. ++ */ ++ BUG_ON(pud_sect(pud)); ++ if (pud_none(pud)) { ++ pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN; ++ phys_addr_t pmd_phys; ++ ++ if (flags & NO_EXEC_MAPPINGS) ++ pudval |= PUD_TABLE_PXN; ++ BUG_ON(!pgtable_alloc); ++ pmd_phys = pgtable_alloc(PMD_SHIFT); ++ __iee_pud_populate_pre_init(pudp, pmd_phys, pudval); ++ pud = READ_ONCE(*pudp); ++ } ++ BUG_ON(pud_bad(pud)); ++ ++ do { ++ pgprot_t __prot = prot; ++ ++ next = pmd_cont_addr_end(addr, end); ++ ++ /* use a contiguous mapping if the range is suitably aligned */ ++ if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) && ++ (flags & NO_CONT_MAPPINGS) == 0) ++ __prot = __pgprot(pgprot_val(prot) | PTE_CONT); ++ ++ iee_init_pmd_pre_init(pudp, addr, next, phys, __prot, pgtable_alloc, flags); ++ ++ phys += next - addr; ++ } while (addr = next, addr != end); ++} ++ ++static __init void iee_alloc_init_pud_pre_init(pgd_t *pgdp, unsigned long addr, unsigned long end, ++ phys_addr_t phys, pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long next; ++ pud_t *pudp; ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); ++ ++ if (p4d_none(p4d)) { ++ p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN; ++ phys_addr_t pud_phys; ++ ++ if (flags & NO_EXEC_MAPPINGS) ++ p4dval |= P4D_TABLE_PXN; ++ BUG_ON(!pgtable_alloc); ++ pud_phys = pgtable_alloc(PUD_SHIFT); ++ __iee_p4d_populate_pre_init(p4dp, pud_phys, p4dval); ++ p4d = READ_ONCE(*p4dp); ++ } ++ BUG_ON(p4d_bad(p4d)); ++ ++ pudp = pud_set_fixmap_offset_init(p4dp, addr); ++ do { ++ pud_t old_pud = READ_ONCE(*pudp); ++ ++ next = pud_addr_end(addr, end); ++ ++ /* ++ * For 4K granule only, attempt to put down a 1GB block ++ */ ++ if (pud_sect_supported() && ++ ((addr | next | phys) & ~PUD_MASK) == 0 && ++ (flags & NO_BLOCK_MAPPINGS) == 0) { ++ iee_pud_set_huge_pre_init(pudp, phys, prot); ++ ++ /* ++ * After the PUD entry has been populated once, we ++ * only allow updates to the permission attributes. ++ */ ++ BUG_ON(!pgattr_change_is_safe(pud_val(old_pud), ++ READ_ONCE(pud_val(*pudp)))); ++ } else { ++ iee_alloc_init_cont_pmd_pre_init(pudp, addr, next, phys, prot, ++ pgtable_alloc, flags); ++ ++ BUG_ON(pud_val(old_pud) != 0 && ++ pud_val(old_pud) != READ_ONCE(pud_val(*pudp))); ++ } ++ phys += next - addr; ++ } while (pudp++, addr = next, addr != end); ++ ++ pud_clear_fixmap_init(); ++} ++ ++static __init void __iee_create_pgd_mapping_locked_pre_init(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); ++ p4d_t *p4dp; ++ p4d_t p4d; ++ ++ /* ++ * If the virtual and physical address don't have the same offset ++ * within a page, we cannot map the region as the caller expects. ++ */ ++ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) ++ return; ++ ++ phys &= PAGE_MASK; ++ addr = virt & PAGE_MASK; ++ end = PAGE_ALIGN(virt + size); ++ ++ do { ++ next = pgd_addr_end(addr, end); ++ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ p4dp = p4d_offset(pgdp, addr); ++ p4d = READ_ONCE(*p4dp); ++ __iee_p4d_populate_pre_init(p4dp, __p4d_to_phys(p4d), (PGD_APT | PUD_TYPE_TABLE)); ++ phys += next - addr; ++ } while (pgdp++, addr = next, addr != end); ++} ++ ++static __init void __iee_create_pgd_mapping_pre_init(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ mutex_lock(&fixmap_lock); ++ __iee_create_pgd_mapping_locked_pre_init(pgdir, phys, virt, size, prot, ++ pgtable_alloc, flags); ++ mutex_unlock(&fixmap_lock); ++} ++#endif ++ ++static phys_addr_t __pgd_pgtable_alloc(int shift) ++{ ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif ++ void *ptr = (void *)__get_free_page(GFP_PGTABLE_KERNEL); ++ BUG_ON(!ptr); ++ ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(ptr)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)ptr); ++ #endif ++ ++ /* Ensure the zeroed page is visible to the page table walker */ ++ dsb(ishst); ++ return __pa(ptr); ++} ++ ++static phys_addr_t pgd_pgtable_alloc(int shift) ++{ ++ phys_addr_t pa = __pgd_pgtable_alloc(shift); ++ struct ptdesc *ptdesc = page_ptdesc(phys_to_page(pa)); ++ ++ /* ++ * Call proper page table ctor in case later we need to ++ * call core mm functions like apply_to_page_range() on ++ * this pre-allocated page table. ++ * ++ * We don't select ARCH_ENABLE_SPLIT_PMD_PTLOCK if pmd is ++ * folded, and if so pagetable_pte_ctor() becomes nop. ++ */ ++ if (shift == PAGE_SHIFT) ++ BUG_ON(!pagetable_pte_ctor(ptdesc)); ++ else if (shift == PMD_SHIFT) ++ BUG_ON(!pagetable_pmd_ctor(ptdesc)); ++ ++ return pa; ++} ++ ++/* ++ * This function can only be used to modify existing table entries, ++ * without allocating new levels of table. Note that this permits the ++ * creation of new section or page entries. ++ */ ++void __init create_mapping_noalloc(phys_addr_t phys, unsigned long virt, ++ phys_addr_t size, pgprot_t prot) ++{ ++ if (virt < PAGE_OFFSET) { ++ pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", ++ &phys, virt); ++ return; ++ } ++ ++ #ifdef CONFIG_PTP ++ __iee_create_pgd_mapping_pre_init(init_mm.pgd, phys, virt, size, prot, NULL, ++ NO_CONT_MAPPINGS); ++ #else ++ __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, ++ NO_CONT_MAPPINGS); ++ #endif ++} ++ ++void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, bool page_mappings_only) ++{ ++ int flags = 0; ++ ++ BUG_ON(mm == &init_mm); ++ ++ if (page_mappings_only) ++ flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; ++ ++ __create_pgd_mapping(mm->pgd, phys, virt, size, prot, ++ pgd_pgtable_alloc, flags); ++} ++ ++static void update_mapping_prot(phys_addr_t phys, unsigned long virt, ++ phys_addr_t size, pgprot_t prot) ++{ ++ if (virt < PAGE_OFFSET) { ++ pr_warn("BUG: not updating mapping for %pa at 0x%016lx - outside kernel range\n", ++ &phys, virt); ++ return; ++ } ++ ++ __create_pgd_mapping(init_mm.pgd, phys, virt, size, prot, NULL, ++ NO_CONT_MAPPINGS); ++ ++ /* flush the TLBs after updating live kernel mappings */ ++ flush_tlb_kernel_range(virt, virt + size); ++} ++ ++static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start, ++ phys_addr_t end, pgprot_t prot, int flags) ++{ ++ #ifdef CONFIG_PTP ++ __iee_create_pgd_mapping_pre_init(pgdp, start, __phys_to_virt(start), end - start, ++ prot, early_pgtable_alloc, flags); ++ #else ++ __create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start, ++ prot, early_pgtable_alloc, flags); ++ #endif ++} ++ ++void __init mark_linear_text_alias_ro(void) ++{ ++ /* ++ * Remove the write permissions from the linear alias of .text/.rodata ++ */ ++ update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext), ++ (unsigned long)__init_begin - (unsigned long)_stext, ++ PAGE_KERNEL_RO); ++} ++ ++#ifdef CONFIG_KFENCE ++ ++bool __ro_after_init kfence_early_init = !!CONFIG_KFENCE_SAMPLE_INTERVAL; ++ ++/* early_param() will be parsed before map_mem() below. */ ++static int __init parse_kfence_early_init(char *arg) ++{ ++ int val; ++ ++ if (get_option(&arg, &val)) ++ kfence_early_init = !!val; ++ ++#if IS_ENABLED(CONFIG_KFENCE_MUST_EARLY_INIT) ++ kfence_must_early_init = (val == -1) ? true : false; ++#endif ++ ++ return 0; ++} ++early_param("kfence.sample_interval", parse_kfence_early_init); ++ ++static phys_addr_t __init arm64_kfence_alloc_pool(void) ++{ ++ phys_addr_t kfence_pool; ++ ++ if (!kfence_early_init) ++ return 0; ++ ++ kfence_pool = memblock_phys_alloc(KFENCE_POOL_SIZE, PAGE_SIZE); ++ if (!kfence_pool) { ++ pr_err("failed to allocate kfence pool\n"); ++ kfence_early_init = false; ++ return 0; ++ } ++ ++ /* Temporarily mark as NOMAP. */ ++ memblock_mark_nomap(kfence_pool, KFENCE_POOL_SIZE); ++ ++ return kfence_pool; ++} ++ ++static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) ++{ ++ if (!kfence_pool) ++ return; ++ ++ /* KFENCE pool needs page-level mapping. */ ++ __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE, ++ pgprot_tagged(PAGE_KERNEL), ++ NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); ++ memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); ++ __kfence_pool = phys_to_virt(kfence_pool); ++} ++#else /* CONFIG_KFENCE */ ++ ++static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } ++static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { } ++ ++#endif /* CONFIG_KFENCE */ ++ ++static void __init map_mem(pgd_t *pgdp) ++{ ++ static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); ++ phys_addr_t kernel_start = __pa_symbol(_stext); ++ phys_addr_t kernel_end = __pa_symbol(__init_begin); ++ phys_addr_t start, end; ++ phys_addr_t early_kfence_pool; ++ int flags = NO_EXEC_MAPPINGS; ++ u64 i; ++ ++ /* ++ * Setting hierarchical PXNTable attributes on table entries covering ++ * the linear region is only possible if it is guaranteed that no table ++ * entries at any level are being shared between the linear region and ++ * the vmalloc region. Check whether this is true for the PGD level, in ++ * which case it is guaranteed to be true for all other levels as well. ++ */ ++ BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end)); ++ ++ early_kfence_pool = arm64_kfence_alloc_pool(); ++ ++ if (can_set_direct_map()) ++ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; ++ ++ #ifdef CONFIG_IEE ++ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; ++ #endif ++ /* ++ * Take care not to create a writable alias for the ++ * read-only text and rodata sections of the kernel image. ++ * So temporarily mark them as NOMAP to skip mappings in ++ * the following for-loop ++ */ ++ memblock_mark_nomap(kernel_start, kernel_end - kernel_start); ++ ++ /* map all the memory banks */ ++ for_each_mem_range(i, &start, &end) { ++ if (start >= end) ++ break; ++ /* ++ * The linear map must allow allocation tags reading/writing ++ * if MTE is present. Otherwise, it has the same attributes as ++ * PAGE_KERNEL. ++ */ ++ __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), ++ flags); ++ } + +- return kfence_pool; ++ /* ++ * Map the linear alias of the [_stext, __init_begin) interval ++ * as non-executable now, and remove the write permission in ++ * mark_linear_text_alias_ro() below (which will be called after ++ * alternative patching has completed). This makes the contents ++ * of the region accessible to subsystems such as hibernate, ++ * but protects it from inadvertent modification or execution. ++ * Note that contiguous mappings cannot be remapped in this way, ++ * so we should avoid them here. ++ */ ++ #ifdef CONFIG_IEE ++ __map_memblock(pgdp, kernel_start, kernel_end, ++ PAGE_KERNEL, flags); ++ #else ++ __map_memblock(pgdp, kernel_start, kernel_end, ++ PAGE_KERNEL, NO_CONT_MAPPINGS); ++ #endif ++ memblock_clear_nomap(kernel_start, kernel_end - kernel_start); ++ arm64_kfence_map_pool(early_kfence_pool, pgdp); ++} ++ ++void mark_rodata_ro(void) ++{ ++ unsigned long section_size; ++ ++ /* ++ * mark .rodata as read only. Use __init_begin rather than __end_rodata ++ * to cover NOTES and EXCEPTION_TABLE. ++ */ ++ section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; ++ update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, ++ section_size, PAGE_KERNEL_RO); ++ ++ debug_checkwx(); ++} ++ ++static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, ++ pgprot_t prot, struct vm_struct *vma, ++ int flags, unsigned long vm_flags) ++{ ++ phys_addr_t pa_start = __pa_symbol(va_start); ++ unsigned long size = va_end - va_start; ++ ++ BUG_ON(!PAGE_ALIGNED(pa_start)); ++ BUG_ON(!PAGE_ALIGNED(size)); ++ ++ #ifdef CONFIG_PTP ++ __iee_create_pgd_mapping_pre_init(pgdp, pa_start, (unsigned long)va_start, size, prot, ++ early_pgtable_alloc, flags); ++ #else ++ __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, ++ early_pgtable_alloc, flags); ++ #endif ++ ++ if (!(vm_flags & VM_NO_GUARD)) ++ size += PAGE_SIZE; ++ ++ vma->addr = va_start; ++ vma->phys_addr = pa_start; ++ vma->size = size; ++ vma->flags = VM_MAP | vm_flags; ++ vma->caller = __builtin_return_address(0); ++ ++ vm_area_add_early(vma); ++} ++ ++static pgprot_t kernel_exec_prot(void) ++{ ++ return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; ++} ++ ++#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 ++static int __init map_entry_trampoline(void) ++{ ++ int i; ++ ++ pgprot_t prot = kernel_exec_prot(); ++ phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); ++ ++ /* The trampoline is always mapped and can therefore be global */ ++ pgprot_val(prot) &= ~PTE_NG; ++ ++ /* Map only the text into the trampoline page table */ ++ memset(tramp_pg_dir, 0, PGD_SIZE); ++ #ifdef CONFIG_PTP ++ iee_set_logical_mem_ro((unsigned long)tramp_pg_dir); ++ #endif ++ __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, ++ entry_tramp_text_size(), prot, ++ __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); ++ ++ /* Map both the text and data into the kernel page table */ ++ for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) ++ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, ++ pa_start + i * PAGE_SIZE, prot); ++ ++ if (IS_ENABLED(CONFIG_RELOCATABLE)) ++ __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, ++ pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO); ++ ++ return 0; ++} ++core_initcall(map_entry_trampoline); ++#endif ++ ++/* ++ * Open coded check for BTI, only for use to determine configuration ++ * for early mappings for before the cpufeature code has run. ++ */ ++static bool arm64_early_this_cpu_has_bti(void) ++{ ++ u64 pfr1; ++ ++ if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) ++ return false; ++ ++ pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); ++ return cpuid_feature_extract_unsigned_field(pfr1, ++ ID_AA64PFR1_EL1_BT_SHIFT); ++} ++ ++#ifdef CONFIG_IEE ++/* Set PMD APTable of iee si codes as (1,1) to revert it to ROX P pages when HPD1=0. */ ++static void __init iee_si_set_pmd_APtable(unsigned long addr, pgd_t *pgdir) ++{ ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); ++ ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ ++ #ifdef CONFIG_PTP ++ pud_t *pudp = pud_set_fixmap_offset_init(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_set_fixmap_offset_init(pudp, addr); ++ ++ pmd_t pmd = READ_ONCE(*pmdp); ++ ++ __iee_pmd_populate_pre_init(pmdp, __pmd_to_phys(pmd), PGD_APT_RO | PGD_APT | PMD_TYPE_TABLE); ++ ++ pud_clear_fixmap_init(); ++ pmd_clear_fixmap_init(); ++ #else ++ pud_t *pudp = pud_set_fixmap_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_set_fixmap_offset(pudp, addr); ++ ++ pmd_t pmd = READ_ONCE(*pmdp); ++ ++ __pmd_populate(pmdp, __pmd_to_phys(pmd), PGD_APT_RO | PGD_APT | PMD_TYPE_TABLE); ++ ++ pud_clear_fixmap(); ++ pmd_clear_fixmap(); ++ #endif ++} ++/* Set PMD APTable of iee si codes as (1,1) to revert it to ROX P pages when HPD1=0. */ ++static void __init mark_iee_si_pmd_APtable(pgd_t *pgdir) ++{ ++ unsigned long addr = (unsigned long)__iee_si_start; ++ iee_si_set_pmd_APtable(addr, pgdir); ++ // iee rwx gate exit may be mapped by another pmd. ++ iee_si_set_pmd_APtable(addr + PAGE_SIZE, pgdir); ++} ++#endif ++ ++/* ++ * Create fine-grained mappings for the kernel. ++ */ ++static void __init map_kernel(pgd_t *pgdp) ++{ ++ static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, ++ vmlinux_initdata, vmlinux_data; ++ ++ #ifdef CONFIG_IEE ++ static struct vm_struct vmlinux_iee_code, vmlinux_iee_data, vmlinux_iee_gate, vmlinux_text_end; ++ #endif ++ ++ /* ++ * External debuggers may need to write directly to the text ++ * mapping to install SW breakpoints. Allow this (only) when ++ * explicitly requested with rodata=off. ++ */ ++ pgprot_t text_prot = kernel_exec_prot(); ++ ++ /* ++ * If we have a CPU that supports BTI and a kernel built for ++ * BTI then mark the kernel executable text as guarded pages ++ * now so we don't have to rewrite the page tables later. ++ */ ++ if (arm64_early_this_cpu_has_bti()) ++ text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); ++ ++ /* ++ * Only rodata will be remapped with different permissions later on, ++ * all other segments are allowed to use contiguous mappings. ++ */ ++ #ifdef CONFIG_IEE ++ map_kernel_segment(pgdp, _stext, __iee_si_data_start, text_prot, &vmlinux_text, ++ 0, VM_NO_GUARD); ++ /* Set iee si data RW. */ ++ map_kernel_segment(pgdp, __iee_si_data_start, __iee_exec_entry_start, SET_NG(PAGE_KERNEL), ++ &vmlinux_iee_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); ++ /* Set iee entry codes NG. */ ++ map_kernel_segment(pgdp, __iee_exec_entry_start, __iee_si_start, SET_NG(text_prot), &vmlinux_iee_gate, ++ NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); ++ /* Map __iee_si_start - __iee_si_end as U RWX pages and set PMD APTABLE = (1,1). */ ++ map_kernel_segment(pgdp, __iee_si_start, __iee_si_end, SET_NG((PAGE_KERNEL_EXEC)), ++ &vmlinux_iee_code, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); ++ mark_iee_si_pmd_APtable(pgdp); ++ ++ map_kernel_segment(pgdp, __iee_si_end, _etext, text_prot, &vmlinux_text_end, 0, ++ VM_NO_GUARD); ++ ++ map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, ++ &vmlinux_rodata, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, VM_NO_GUARD); ++ map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, ++ &vmlinux_inittext, 0, VM_NO_GUARD); ++ map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, ++ &vmlinux_initdata, 0, VM_NO_GUARD); ++ map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, NO_CONT_MAPPINGS | NO_BLOCK_MAPPINGS, 0); ++ #else ++ map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, ++ VM_NO_GUARD); ++ map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, ++ &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); ++ map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, ++ &vmlinux_inittext, 0, VM_NO_GUARD); ++ map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, ++ &vmlinux_initdata, 0, VM_NO_GUARD); ++ map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); ++ #endif ++ ++ ++ fixmap_copy(pgdp); ++ kasan_copy_shadow(pgdp); ++} ++ ++static void __init create_idmap(void) ++{ ++ u64 start = __pa_symbol(__idmap_text_start); ++ u64 size = __pa_symbol(__idmap_text_end) - start; ++ pgd_t *pgd = idmap_pg_dir; ++ u64 pgd_phys; ++ ++ /* check if we need an additional level of translation */ ++ if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) { ++ pgd_phys = early_pgtable_alloc(PAGE_SHIFT); ++ set_pgd(&idmap_pg_dir[start >> VA_BITS], ++ __pgd(pgd_phys | P4D_TYPE_TABLE)); ++ pgd = __va(pgd_phys); ++ } ++ #ifdef CONFIG_PTP ++ __iee_create_pgd_mapping_pre_init(pgd, start, start, size, PAGE_KERNEL_ROX, ++ early_pgtable_alloc, 0); ++ #else ++ __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX, ++ early_pgtable_alloc, 0); ++ #endif ++ ++ if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { ++ extern u32 __idmap_kpti_flag; ++ u64 pa = __pa_symbol(&__idmap_kpti_flag); ++ ++ /* ++ * The KPTI G-to-nG conversion code needs a read-write mapping ++ * of its synchronization flag in the ID map. ++ */ ++ #ifdef CONFIG_PTP ++ __iee_create_pgd_mapping_pre_init(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, ++ early_pgtable_alloc, 0); ++ #else ++ __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, ++ early_pgtable_alloc, 0); ++ #endif ++ } ++} ++ ++#ifdef CONFIG_IEE ++static void __create_pgd_mapping_for_iee_locked(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); ++ p4d_t *p4dp; ++ p4d_t p4d; ++ ++ /* ++ * If the virtual and physical address don't have the same offset ++ * within a page, we cannot map the region as the caller expects. ++ */ ++ if (WARN_ON((phys ^ virt) & ~PAGE_MASK)) ++ return; ++ ++ phys &= PAGE_MASK; ++ addr = virt & PAGE_MASK; ++ end = PAGE_ALIGN(virt + size); ++ ++ do { ++ next = pgd_addr_end(addr, end); ++ #ifdef CONFIG_PTP ++ iee_alloc_init_pud_pre_init(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ #else ++ alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc, ++ flags); ++ #endif ++ p4dp = p4d_offset(pgdp, addr); ++ p4d = READ_ONCE(*p4dp); ++ #ifdef CONFIG_PTP ++ __iee_p4d_populate_pre_init(p4dp, __p4d_to_phys(p4d), (PGD_APT | PGD_PXN | PGD_UXN | PUD_TYPE_TABLE)); ++ #else ++ __p4d_populate(p4dp, __p4d_to_phys(p4d), (PGD_APT | PGD_PXN | PGD_UXN | PUD_TYPE_TABLE)); ++ #endif ++ phys += next - addr; ++ } while (pgdp++, addr = next, addr != end); ++} ++ ++static void __create_pgd_mapping_for_iee(pgd_t *pgdir, phys_addr_t phys, ++ unsigned long virt, phys_addr_t size, ++ pgprot_t prot, ++ phys_addr_t (*pgtable_alloc)(int), ++ int flags) ++{ ++ mutex_lock(&fixmap_lock); ++ __create_pgd_mapping_for_iee_locked(pgdir, phys, virt, size, prot, ++ pgtable_alloc, flags); ++ mutex_unlock(&fixmap_lock); + } + +-static void __init arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) ++static void __init __map_memblock_for_iee(pgd_t *pgdp, phys_addr_t start, ++ phys_addr_t end, pgprot_t prot, int flags) + { +- if (!kfence_pool) +- return; +- +- /* KFENCE pool needs page-level mapping. */ +- __map_memblock(pgdp, kfence_pool, kfence_pool + KFENCE_POOL_SIZE, +- pgprot_tagged(PAGE_KERNEL), +- NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS); +- memblock_clear_nomap(kfence_pool, KFENCE_POOL_SIZE); +- __kfence_pool = phys_to_virt(kfence_pool); ++ #ifdef CONFIG_PTP ++ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, ++ prot, early_pgtable_alloc, flags); ++ #else ++ __create_pgd_mapping_for_iee(pgdp, start, __phys_to_iee(start), end - start, ++ prot, early_pgtable_alloc, flags); ++ #endif + } +-#else /* CONFIG_KFENCE */ + +-static inline phys_addr_t arm64_kfence_alloc_pool(void) { return 0; } +-static inline void arm64_kfence_map_pool(phys_addr_t kfence_pool, pgd_t *pgdp) { } +- +-#endif /* CONFIG_KFENCE */ +- +-static void __init map_mem(pgd_t *pgdp) ++static void __init map_iee(pgd_t *pgdp) + { + static const u64 direct_map_end = _PAGE_END(VA_BITS_MIN); + phys_addr_t kernel_start = __pa_symbol(_stext); +@@ -578,6 +1578,8 @@ static void __init map_mem(pgd_t *pgdp) + int flags = NO_EXEC_MAPPINGS; + u64 i; + ++ flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; ++ + /* + * Setting hierarchical PXNTable attributes on table entries covering + * the linear region is only possible if it is guaranteed that no table +@@ -589,9 +1591,6 @@ static void __init map_mem(pgd_t *pgdp) + + early_kfence_pool = arm64_kfence_alloc_pool(); + +- if (can_set_direct_map()) +- flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; +- + /* + * Take care not to create a writable alias for the + * read-only text and rodata sections of the kernel image. +@@ -609,12 +1608,11 @@ static void __init map_mem(pgd_t *pgdp) + * if MTE is present. Otherwise, it has the same attributes as + * PAGE_KERNEL. + */ +- __map_memblock(pgdp, start, end, pgprot_tagged(PAGE_KERNEL), +- flags); ++ __map_memblock_for_iee(pgdp, start, end, SET_NG(SET_INVALID(SET_UPAGE(PAGE_KERNEL))), flags); + } + + /* +- * Map the linear alias of the [_stext, __init_begin) interval ++ * Map the linear alias of the [_text, __init_begin) interval + * as non-executable now, and remove the write permission in + * mark_linear_text_alias_ro() below (which will be called after + * alternative patching has completed). This makes the contents +@@ -623,178 +1621,384 @@ static void __init map_mem(pgd_t *pgdp) + * Note that contiguous mappings cannot be remapped in this way, + * so we should avoid them here. + */ +- __map_memblock(pgdp, kernel_start, kernel_end, +- PAGE_KERNEL, NO_CONT_MAPPINGS); ++ __map_memblock_for_iee(pgdp, kernel_start, kernel_end, ++ SET_NG(SET_INVALID(SET_UPAGE(PAGE_KERNEL))), flags); + memblock_clear_nomap(kernel_start, kernel_end - kernel_start); + arm64_kfence_map_pool(early_kfence_pool, pgdp); + } + +-void mark_rodata_ro(void) ++/* ++ * Change page access permission, whereas not handling huge pages. ++ * Only used on IEE init functions. ++ */ ++static void __init iee_si_set_page_attr(unsigned long addr, pteval_t attr) + { +- unsigned long section_size; ++ unsigned long flag; ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + +- /* +- * mark .rodata as read only. Use __init_begin rather than __end_rodata +- * to cover NOTES and EXCEPTION_TABLE. +- */ +- section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata; +- update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata, +- section_size, PAGE_KERNEL_RO); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); + +- debug_checkwx(); ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if(attr & PTE_RDONLY) ++ pte = __pte((pte_val(pte) | PTE_RDONLY) & ~PTE_DBM); ++ pte = __pte(pte_val(pte) | attr); ++ #ifdef CONFIG_PTP ++ // Write pgtable in IEE directly. ++ flag = local_daif_save(); ++ asm volatile ("msr pan, #0"); ++ WRITE_ONCE(*((pte_t *)(__phys_to_iee(__pa(ptep)))), pte); ++ asm volatile ("msr pan, #1"); ++ local_daif_restore(flag); ++ #else ++ WRITE_ONCE(*ptep, pte); ++ #endif ++} ++ ++/* Prepare data used for iee rwx gates. These data are setted only once. */ ++void __init iee_si_prepare_data(void) ++{ ++ unsigned long va; ++ // Record current TCR val after system init. ++ iee_si_tcr = read_sysreg(tcr_el1) & ~(SYS_TCR_IEE_SI); ++ // Mark iee data as RO and move it to iee after setting up. ++ va = (unsigned long)__iee_si_data_start; ++ iee_si_set_page_attr(va, PTE_RDONLY); ++ iee_si_set_page_attr(lm_alias(va)+iee_offset, 0x1 | PTE_RDONLY); ++ // Set iee sensitive inst code page U RWX here to hide it from kernel. ++ va = (unsigned long)__iee_si_start; ++ iee_si_set_page_attr(va, PTE_USER); ++ va = (unsigned long)__iee_si_start + PAGE_SIZE; ++ iee_si_set_page_attr(va, PTE_USER); ++ flush_tlb_all(); + } + +-static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, +- pgprot_t prot, struct vm_struct *vma, +- int flags, unsigned long vm_flags) ++#endif ++ ++#ifdef CONFIG_PTP ++// Attention : Using set_xxx without adding offset. ++static void __init set_iee_valid_pre_init(unsigned long addr) + { +- phys_addr_t pa_start = __pa_symbol(va_start); +- unsigned long size = va_end - va_start; ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + +- BUG_ON(!PAGE_ALIGNED(pa_start)); +- BUG_ON(!PAGE_ALIGNED(size)); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); + +- __create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot, +- early_pgtable_alloc, flags); ++ pud_t *pudp = pud_offset(p4dp, addr); + +- if (!(vm_flags & VM_NO_GUARD)) +- size += PAGE_SIZE; ++ pmd_t *pmdp = pmd_offset(pudp, addr); + +- vma->addr = va_start; +- vma->phys_addr = pa_start; +- vma->size = size; +- vma->flags = VM_MAP | vm_flags; +- vma->caller = __builtin_return_address(0); ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ pte_t pte = READ_ONCE(*ptep); + +- vm_area_add_early(vma); ++ if((addr < (PAGE_OFFSET + IEE_OFFSET)) | (addr > (PAGE_OFFSET + BIT(vabits_actual - 1)))) ++ return; ++ ++ pte = __pte(pte_val(pte) | 0x1); ++ iee_set_pte_pre_init(ptep, pte); ++ flush_tlb_kernel_range(addr, addr+PAGE_SIZE); ++ isb(); + } + +-static pgprot_t kernel_exec_prot(void) ++static void __init move_pte_table_into_iee(pmd_t *pmdp, unsigned long addr, unsigned long end) + { +- return rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC; ++ pmd_t pmd = READ_ONCE(*pmdp); ++ unsigned long iee_addr = __phys_to_iee(__pmd_to_phys(pmd)); ++ set_iee_valid_pre_init(iee_addr); + } + +-#ifdef CONFIG_UNMAP_KERNEL_AT_EL0 +-static int __init map_entry_trampoline(void) ++static void __init move_pmd_table_into_iee(pud_t *pudp, unsigned long addr, unsigned long end) + { +- int i; ++ unsigned long next; ++ pud_t pud = READ_ONCE(*pudp); ++ pmd_t *pmdp; ++ pmd_t pmd; ++ unsigned long iee_addr = __phys_to_iee(__pud_to_phys(pud)); ++ set_iee_valid_pre_init(iee_addr); + +- pgprot_t prot = kernel_exec_prot(); +- phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start); ++ pmdp = pmd_offset(pudp, addr); ++ do { ++ next = pmd_addr_end(addr, end); ++ pmd = READ_ONCE(*pmdp); ++ if((pmd_val(pmd) & PMD_TABLE_BIT) == 0) ++ { ++ continue; ++ } ++ else ++ { ++ move_pte_table_into_iee(pmdp, addr, next); ++ } ++ } while (pmdp++, addr = next, addr != end); ++} + +- /* The trampoline is always mapped and can therefore be global */ +- pgprot_val(prot) &= ~PTE_NG; ++static void __init move_pud_table_into_iee(pgd_t *pgdp, unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); ++ pud_t *pudp; ++ pud_t pud; ++ unsigned long iee_addr = __phys_to_iee(__p4d_to_phys(p4d)); ++ set_iee_valid_pre_init(iee_addr); + +- /* Map only the text into the trampoline page table */ +- memset(tramp_pg_dir, 0, PGD_SIZE); +- __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, +- entry_tramp_text_size(), prot, +- __pgd_pgtable_alloc, NO_BLOCK_MAPPINGS); ++ pudp = pud_offset(p4dp, addr); ++ do { ++ next = pud_addr_end(addr, end); ++ pud = READ_ONCE(*pudp); ++ if ((pud_val(pud) & PUD_TABLE_BIT) == 0) ++ { ++ continue; ++ } ++ else ++ { ++ move_pmd_table_into_iee(pudp, addr, next); ++ } ++ } while (pudp++, addr = next, addr != end); ++} + +- /* Map both the text and data into the kernel page table */ +- for (i = 0; i < DIV_ROUND_UP(entry_tramp_text_size(), PAGE_SIZE); i++) +- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, +- pa_start + i * PAGE_SIZE, prot); ++static void __init init_iee_for_one_region(pgd_t *pgdir, unsigned long va_start, unsigned long va_end) ++{ ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); + +- if (IS_ENABLED(CONFIG_RELOCATABLE)) +- __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i, +- pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO); ++ addr = va_start & PAGE_MASK; ++ end = PAGE_ALIGN(va_end); + +- return 0; ++ do { ++ next = pgd_addr_end(addr, end); ++ move_pud_table_into_iee(pgdp, addr, next); ++ } while (pgdp++, addr = next, addr != end); + } +-core_initcall(map_entry_trampoline); +-#endif + +-/* +- * Open coded check for BTI, only for use to determine configuration +- * for early mappings for before the cpufeature code has run. +- */ +-static bool arm64_early_this_cpu_has_bti(void) ++static void __init init_iee(void) + { +- u64 pfr1; ++ unsigned long iee_addr; ++ phys_addr_t start, end; ++ u64 i; ++ pgd_t *pgdp; ++ ++ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 ++ // handling 1-level tramp page table tramp_pg_dir ++ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(tramp_pg_dir)); ++ set_iee_valid_pre_init(iee_addr); ++ #endif ++ // handling 1-level page table swapper_pg_dir ++ pgdp = swapper_pg_dir; ++ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(swapper_pg_dir)); ++ set_iee_valid_pre_init(iee_addr); ++ // handling 2/3/4-level page table for kernel ++ init_iee_for_one_region(pgdp, (unsigned long)_text, (unsigned long)_etext); ++ init_iee_for_one_region(pgdp, (unsigned long)__start_rodata, (unsigned long)__inittext_begin); ++ init_iee_for_one_region(pgdp, (unsigned long)__inittext_begin, (unsigned long)__inittext_end); ++ init_iee_for_one_region(pgdp, (unsigned long)__initdata_begin, (unsigned long)__initdata_end); ++ init_iee_for_one_region(pgdp, (unsigned long)_data, (unsigned long)_end); ++ // handling 2/3/4-level page table for fixmap i.e. remap bm_xxx ++ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pte_addr)); ++ set_iee_valid_pre_init(iee_addr); ++ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pmd_addr)); ++ set_iee_valid_pre_init(iee_addr); ++ iee_addr = (unsigned long)__phys_to_iee(__pa_symbol(bm_pud_addr)); ++ set_iee_valid_pre_init(iee_addr); ++ // handling 2/3/4-level page table for logical mem and iee ++ for_each_mem_range(i, &start, &end) { ++ if (start >= end) ++ break; ++ /* ++ * The linear map must allow allocation tags reading/writing ++ * if MTE is present. Otherwise, it has the same attributes as ++ * PAGE_KERNEL. ++ */ ++ init_iee_for_one_region(pgdp, (unsigned long)__va(start), (unsigned long)__va(end)); ++ init_iee_for_one_region(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); ++ } ++} + +- if (!IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) +- return false; ++static void iee_set_kernel_upage_pre_init(unsigned long addr) ++{ ++ pgd_t *pgdir = swapper_pg_dir; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, addr); + +- pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); +- return cpuid_feature_extract_unsigned_field(pfr1, +- ID_AA64PFR1_EL1_BT_SHIFT); ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); ++ ++ pud_t *pudp = pud_offset(p4dp, addr); ++ ++ pmd_t *pmdp = pmd_offset(pudp, addr); ++ ++ pte_t *ptep = pte_offset_kernel(pmdp, addr); ++ ++ int i; ++ for(i = 0; i < 4; i++) ++ { ++ pte_t pte = READ_ONCE(*ptep); ++ pte = __pte(pte_val(pte) | PTE_USER | PTE_NG); ++ iee_set_pte_pre_init(ptep, pte); ++ ptep++; ++ } ++ flush_tlb_kernel_range(addr, addr+4*PAGE_SIZE); ++ isb(); + } + +-/* +- * Create fine-grained mappings for the kernel. +- */ +-static void __init map_kernel(pgd_t *pgdp) ++static void __init iee_set_pte_table_ro(pmd_t *pmdp, unsigned long addr, unsigned long end) + { +- static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext, +- vmlinux_initdata, vmlinux_data; ++ pmd_t pmd = READ_ONCE(*pmdp); ++ unsigned long logical_addr = (unsigned long)__va(__pmd_to_phys(pmd)); ++ iee_set_logical_mem_ro(logical_addr); ++} + +- /* +- * External debuggers may need to write directly to the text +- * mapping to install SW breakpoints. Allow this (only) when +- * explicitly requested with rodata=off. +- */ +- pgprot_t text_prot = kernel_exec_prot(); ++static void __init iee_set_pmd_table_ro(pud_t *pudp, unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ pud_t pud = READ_ONCE(*pudp); ++ pmd_t *pmdp; ++ pmd_t pmd; ++ unsigned long logical_addr = (unsigned long)__va(__pud_to_phys(pud)); ++ iee_set_logical_mem_ro(logical_addr); + +- /* +- * If we have a CPU that supports BTI and a kernel built for +- * BTI then mark the kernel executable text as guarded pages +- * now so we don't have to rewrite the page tables later. +- */ +- if (arm64_early_this_cpu_has_bti()) +- text_prot = __pgprot_modify(text_prot, PTE_GP, PTE_GP); ++ pmdp = pmd_offset(pudp, addr); ++ do { ++ next = pmd_addr_end(addr, end); ++ pmd = READ_ONCE(*pmdp); ++ if((pmd_val(pmd) & PMD_TABLE_BIT) == 0) ++ { ++ continue; ++ } ++ else ++ { ++ iee_set_pte_table_ro(pmdp, addr, next); ++ } ++ } while (pmdp++, addr = next, addr != end); ++} + +- /* +- * Only rodata will be remapped with different permissions later on, +- * all other segments are allowed to use contiguous mappings. +- */ +- map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0, +- VM_NO_GUARD); +- map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL, +- &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD); +- map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot, +- &vmlinux_inittext, 0, VM_NO_GUARD); +- map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL, +- &vmlinux_initdata, 0, VM_NO_GUARD); +- map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); ++static void __init iee_set_pud_table_ro(pgd_t *pgdp, unsigned long addr, unsigned long end) ++{ ++ unsigned long next; ++ p4d_t *p4dp = p4d_offset(pgdp, addr); ++ p4d_t p4d = READ_ONCE(*p4dp); ++ pud_t *pudp; ++ pud_t pud; ++ unsigned long logical_addr = (unsigned long)__va(__p4d_to_phys(p4d)); ++ iee_set_logical_mem_ro(logical_addr); + +- fixmap_copy(pgdp); +- kasan_copy_shadow(pgdp); ++ pudp = pud_offset(p4dp, addr); ++ do { ++ next = pud_addr_end(addr, end); ++ pud = READ_ONCE(*pudp); ++ if ((pud_val(pud) & PUD_TABLE_BIT) == 0) ++ { ++ continue; ++ } ++ else ++ { ++ iee_set_pmd_table_ro(pudp, addr, next); ++ } ++ } while (pudp++, addr = next, addr != end); + } + +-static void __init create_idmap(void) ++static void __init iee_mark_pgtable_for_one_region_ro(pgd_t *pgdir, unsigned long va_start, unsigned long va_end) + { +- u64 start = __pa_symbol(__idmap_text_start); +- u64 size = __pa_symbol(__idmap_text_end) - start; +- pgd_t *pgd = idmap_pg_dir; +- u64 pgd_phys; ++ unsigned long addr, end, next; ++ pgd_t *pgdp = pgd_offset_pgd(pgdir, va_start); + +- /* check if we need an additional level of translation */ +- if (VA_BITS < 48 && idmap_t0sz < (64 - VA_BITS_MIN)) { +- pgd_phys = early_pgtable_alloc(PAGE_SHIFT); +- set_pgd(&idmap_pg_dir[start >> VA_BITS], +- __pgd(pgd_phys | P4D_TYPE_TABLE)); +- pgd = __va(pgd_phys); +- } +- __create_pgd_mapping(pgd, start, start, size, PAGE_KERNEL_ROX, +- early_pgtable_alloc, 0); ++ addr = va_start & PAGE_MASK; ++ end = PAGE_ALIGN(va_end); + +- if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) { +- extern u32 __idmap_kpti_flag; +- u64 pa = __pa_symbol(&__idmap_kpti_flag); ++ do { ++ next = pgd_addr_end(addr, end); ++ iee_set_pud_table_ro(pgdp, addr, next); ++ } while (pgdp++, addr = next, addr != end); ++} + ++static void __init iee_mark_all_lm_pgtable_ro(void) ++{ ++ unsigned long logical_addr; ++ phys_addr_t start, end; ++ u64 i; ++ pgd_t *pgdp; ++ ++ // handling static allocated page table ++ #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 ++ // handling 1-level tramp page table tramp_pg_dir ++ logical_addr = (unsigned long)__va(__pa_symbol(tramp_pg_dir)); ++ iee_set_logical_mem_ro(logical_addr); ++ #endif ++ // handling 1-level page table swapper_pg_dir ++ pgdp = swapper_pg_dir; ++ iee_set_logical_mem_ro((unsigned long)swapper_pg_dir); ++ logical_addr = (unsigned long)__va(__pa_symbol(swapper_pg_dir)); ++ iee_set_logical_mem_ro(logical_addr); ++ ++ // handling 2/3/4-level page table for kernel ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_text, (unsigned long)_etext); ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__start_rodata, (unsigned long)__inittext_begin); ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__inittext_begin, (unsigned long)__inittext_end); ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__initdata_begin, (unsigned long)__initdata_end); ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)_data, (unsigned long)_end); ++ ++ // handling 2/3/4-level page table for fixmap i.e. remap bm_xxx ++ logical_addr = (unsigned long)__va(__pa_symbol(bm_pte_addr)); ++ iee_set_logical_mem_ro(logical_addr); ++ ++ iee_set_logical_mem_ro((unsigned long)bm_pmd_addr); ++ logical_addr = (unsigned long)__va(__pa_symbol(bm_pmd_addr)); ++ iee_set_logical_mem_ro(logical_addr); ++ ++ iee_set_logical_mem_ro((unsigned long)bm_pud_addr); ++ logical_addr = (unsigned long)__va(__pa_symbol(bm_pud_addr)); ++ iee_set_logical_mem_ro(logical_addr); ++ ++ // handling 2/3/4-level page table for logical mem and iee ++ for_each_mem_range(i, &start, &end) { ++ if (start >= end) ++ break; + /* +- * The KPTI G-to-nG conversion code needs a read-write mapping +- * of its synchronization flag in the ID map. ++ * The linear map must allow allocation tags reading/writing ++ * if MTE is present. Otherwise, it has the same attributes as ++ * PAGE_KERNEL. + */ +- __create_pgd_mapping(pgd, pa, pa, sizeof(u32), PAGE_KERNEL, +- early_pgtable_alloc, 0); ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__va(start), (unsigned long)__va(end)); ++ iee_mark_pgtable_for_one_region_ro(pgdp, (unsigned long)__phys_to_iee(start), (unsigned long)__phys_to_iee(end)); + } + } ++#endif ++ ++#ifdef CONFIG_KOI ++extern s64 koi_offset; ++#endif + + void __init paging_init(void) + { ++ #ifdef CONFIG_IEE ++ unsigned long SP_EL0; ++ void *new; ++ void *init_token; ++ struct task_token *token; ++ unsigned long tcr; ++ ++ // Check if cpu has PAN and HPDS. ++ if(!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1), ++ ID_AA64MMFR1_EL1_PAN_SHIFT)) ++ panic("Architecture doesn't support PAN, please disable CONFIG_IEE.\n"); ++ ++ if(!cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR1_EL1), ++ ID_AA64MMFR1_EL1_HPDS_SHIFT)) ++ panic("Architecture doesn't support HPDS, please disable CONFIG_IEE.\n"); ++ #endif ++ ++ // Avoid using iee code to modify pgtable before iee initialized. ++ #ifdef CONFIG_PTP ++ pgd_t *pgdp = pgd_set_fixmap_init(__pa_symbol(swapper_pg_dir)); ++ #else + pgd_t *pgdp = pgd_set_fixmap(__pa_symbol(swapper_pg_dir)); ++ #endif ++ ++ + extern pgd_t init_idmap_pg_dir[]; + + idmap_t0sz = 63UL - __fls(__pa_symbol(_end) | GENMASK(VA_BITS_MIN - 1, 0)); +@@ -802,7 +2006,17 @@ void __init paging_init(void) + map_kernel(pgdp); + map_mem(pgdp); + ++ // Map the whole physical mem into IEE, but set invalid. ++ #ifdef CONFIG_IEE ++ map_iee(pgdp); ++ #endif ++ ++ // Avoid using iee code to modify pgtable before iee initialized. ++ #ifdef CONFIG_PTP ++ pgd_clear_fixmap_init(); ++ #else + pgd_clear_fixmap(); ++ #endif + + cpu_replace_ttbr1(lm_alias(swapper_pg_dir), init_idmap_pg_dir); + init_mm.pgd = swapper_pg_dir; +@@ -813,6 +2027,80 @@ void __init paging_init(void) + memblock_allow_resize(); + + create_idmap(); ++ ++ #ifdef CONFIG_IEE ++ // test iee_exec_entry ++ iee_rwx_gate_entry(IEE_SI_TEST); ++ // Initialize init iee stack. ++ #ifdef CONFIG_PTP ++ iee_set_kernel_upage_pre_init((unsigned long)init_iee_stack_begin); ++ iee_set_kernel_upage_pre_init((unsigned long)__va(__pa_symbol(init_iee_stack_begin))); ++ #else ++ iee_set_kernel_upage((unsigned long)init_iee_stack_begin); ++ iee_set_kernel_upage((unsigned long)__va(__pa_symbol(init_iee_stack_begin))); ++ #endif ++ #endif ++ ++ // Init token for init_task. ++ #ifdef CONFIG_IEE ++ // Change SP_EL0 from Image VA to Logical VA. ++ SP_EL0 = (unsigned long)__va(__pa_symbol(&init_task)); ++ write_sysreg(SP_EL0, sp_el0); ++ init_task.cpus_ptr = &(((struct task_struct *)(__va(__pa_symbol(&init_task))))->cpus_mask); ++ init_task.children.prev = (__va(__pa_symbol(init_task.children.prev))); ++ init_task.children.next = (__va(__pa_symbol(init_task.children.next))); ++ // Set init_task into __entry_task before per_cpu init. ++ *(struct task_struct **)__entry_task = __va(__pa_symbol(&init_task)); ++ // Alloc a page for init_token. ++ new = __va(early_pgtable_alloc(0)); ++ init_token = (void *)__phys_to_iee(__pa_symbol(&init_task)); ++ #ifdef CONFIG_PTP ++ iee_set_token_page_valid_pre_init(init_token, new); ++ #else ++ iee_set_token_page_valid(init_token, new); ++ #endif ++ // Use lm to write token before IEE initialized. ++ token = (struct task_token *)((unsigned long)new + (((unsigned long)&init_task) & ~PAGE_MASK)); ++ token->mm = &init_mm; ++ token->pgd = NULL; ++ token->iee_stack = (void *)init_iee_stack_end; ++ token->valid = true; ++ #endif ++ ++ #ifdef CONFIG_PTP ++ // Map the existing pgtable into IEE, set valid. ++ init_iee(); ++ #endif ++ ++ #ifdef CONFIG_IEE ++ sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SPAN); ++ #endif ++ ++ #ifdef CONFIG_PTP ++ // IEE ready. ++ // Pgtable writing before uses logical memory and after uses IEE memory. ++ ++ // Set the logical va of existing pgtable readonly. ++ iee_mark_all_lm_pgtable_ro(); ++ #endif ++ ++ // Set the init token readonly. ++ #ifdef CONFIG_IEE ++ set_iee_page_valid(__phys_to_iee(__pa(new))); ++ iee_set_logical_mem_ro((unsigned long)new); ++ ++ // Set HPD1 as 1. ++ tcr = read_sysreg(tcr_el1); ++ tcr |= ((unsigned long)0x1 << 42); ++ write_sysreg(tcr, tcr_el1); ++ isb(); ++ ++ // Flush tlb to enable IEE. ++ flush_tlb_all(); ++ ++ // mark that iee is prepared. ++ iee_init_done = true; ++ #endif + } + + #ifdef CONFIG_MEMORY_HOTPLUG +diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c +index 4a64089e5771..894bda11c389 100644 +--- a/arch/arm64/mm/pgd.c ++++ b/arch/arm64/mm/pgd.c +@@ -15,14 +15,44 @@ + #include <asm/page.h> + #include <asm/tlbflush.h> + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + static struct kmem_cache *pgd_cache __ro_after_init; + ++#ifdef CONFIG_KOI ++pgd_t *koi_pgd_alloc(void) ++{ ++ pgd_t *pgd; ++#ifdef CONFIG_PTP ++ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL); ++ unsigned long iee_addr = __phys_to_iee(__pa(pgd)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)pgd); ++#else ++ pgd = (pgd_t *)__get_free_page(GFP_PGTABLE_KERNEL); ++#endif ++ return pgd; ++} ++#endif ++ + pgd_t *pgd_alloc(struct mm_struct *mm) + { + gfp_t gfp = GFP_PGTABLE_USER; + + if (PGD_SIZE == PAGE_SIZE) ++#ifdef CONFIG_PTP ++ { ++ pgd_t* new = (pgd_t *)__get_free_page(gfp); ++ unsigned long iee_addr = __phys_to_iee(__pa(new)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)new); ++ return new; ++ } ++#else + return (pgd_t *)__get_free_page(gfp); ++#endif + else + return kmem_cache_alloc(pgd_cache, gfp); + } +@@ -30,7 +60,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm) + void pgd_free(struct mm_struct *mm, pgd_t *pgd) + { + if (PGD_SIZE == PAGE_SIZE) ++#ifdef CONFIG_PTP ++ { ++ unsigned long iee_addr = __phys_to_iee(__pa(pgd)); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)pgd); ++ free_page((unsigned long)pgd); ++ } ++#else + free_page((unsigned long)pgd); ++#endif + else + kmem_cache_free(pgd_cache, pgd); + } +diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S +index 14fdf645edc8..230b2b883a51 100644 +--- a/arch/arm64/mm/proc.S ++++ b/arch/arm64/mm/proc.S +@@ -22,6 +22,8 @@ + #include <asm/smp.h> + #include <asm/sysreg.h> + ++ ++ + #ifdef CONFIG_ARM64_64K_PAGES + #define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K + #elif defined(CONFIG_ARM64_16K_PAGES) +@@ -105,6 +107,19 @@ SYM_FUNC_START(cpu_do_suspend) + ret + SYM_FUNC_END(cpu_do_suspend) + ++ ++#ifdef CONFIG_IEE ++// SP_EL0 check failed. ++SYM_FUNC_START_LOCAL(sp_el0_check_failed) ++1: ++ nop ++ nop ++ nop ++ nop ++ b 1f ++SYM_FUNC_END(sp_el0_check_failed) ++#endif ++ + /** + * cpu_do_resume - restore CPU register context + * +@@ -148,6 +163,13 @@ SYM_FUNC_START(cpu_do_resume) + msr sctlr_el1, x12 + set_this_cpu_offset x13 + msr sp_el0, x14 ++#ifdef CONFIG_IEE ++ // tsk check. ++ ldr_this_cpu x2, __entry_task, x3 ++ mrs x3, sp_el0 ++ cmp x2, x3 ++ b.ne sp_el0_check_failed ++#endif + /* + * Restore oslsr_el1 by writing oslar_el1 + */ +@@ -190,6 +212,7 @@ SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) + __idmap_cpu_set_reserved_ttbr1 x1, x3 + + offset_ttbr1 x0, x3 ++ + msr ttbr1_el1, x0 + isb + +@@ -452,6 +475,11 @@ SYM_FUNC_START(__cpu_setup) + orr tcr, tcr, #TCR_HA // hardware Access flag update + 1: + #endif /* CONFIG_ARM64_HW_AFDBM */ ++ ++#ifdef CONFIG_IEE ++ orr tcr, tcr, #TCR_HPD1 // Hierarchical permission disables ++#endif ++ + msr mair_el1, mair + msr tcr_el1, tcr + +diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c +index 5139a28130c0..15d2a3faa048 100644 +--- a/arch/arm64/mm/trans_pgd.c ++++ b/arch/arm64/mm/trans_pgd.c +@@ -25,6 +25,9 @@ + #include <linux/mm.h> + #include <linux/mmzone.h> + #include <linux/kfence.h> ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif + + static void *trans_alloc(struct trans_pgd_info *info) + { +@@ -65,10 +68,18 @@ static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, + pte_t *src_ptep; + pte_t *dst_ptep; + unsigned long addr = start; ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + + dst_ptep = trans_alloc(info); + if (!dst_ptep) + return -ENOMEM; ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(dst_ptep)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)dst_ptep); ++ #endif + pmd_populate_kernel(NULL, dst_pmdp, dst_ptep); + dst_ptep = pte_offset_kernel(dst_pmdp, start); + +@@ -87,11 +98,19 @@ static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp, + pmd_t *dst_pmdp; + unsigned long next; + unsigned long addr = start; ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + + if (pud_none(READ_ONCE(*dst_pudp))) { + dst_pmdp = trans_alloc(info); + if (!dst_pmdp) + return -ENOMEM; ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(dst_pmdp)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)dst_pmdp); ++ #endif + pud_populate(NULL, dst_pudp, dst_pmdp); + } + dst_pmdp = pmd_offset(dst_pudp, start); +@@ -123,11 +142,19 @@ static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp, + pud_t *src_pudp; + unsigned long next; + unsigned long addr = start; ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + + if (p4d_none(READ_ONCE(*dst_p4dp))) { + dst_pudp = trans_alloc(info); + if (!dst_pudp) + return -ENOMEM; ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(dst_pudp)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)dst_pudp); ++ #endif + p4d_populate(NULL, dst_p4dp, dst_pudp); + } + dst_pudp = pud_offset(dst_p4dp, start); +@@ -212,6 +239,12 @@ int trans_pgd_create_copy(struct trans_pgd_info *info, pgd_t **dst_pgdp, + return -ENOMEM; + } + ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(trans_pgd)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)trans_pgd); ++ #endif ++ + rc = copy_page_tables(info, trans_pgd, start, end); + if (!rc) + *dst_pgdp = trans_pgd; +@@ -238,6 +271,9 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, + int bits_mapped = PAGE_SHIFT - 4; + unsigned long level_mask, prev_level_entry, *levels[4]; + int this_level, index, level_lsb, level_msb; ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + + dst_addr &= PAGE_MASK; + prev_level_entry = pte_val(pfn_pte(pfn, PAGE_KERNEL_ROX)); +@@ -247,12 +283,22 @@ int trans_pgd_idmap_page(struct trans_pgd_info *info, phys_addr_t *trans_ttbr0, + if (!levels[this_level]) + return -ENOMEM; + ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(levels[this_level])); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)levels[this_level]); ++ #endif ++ + level_lsb = ARM64_HW_PGTABLE_LEVEL_SHIFT(this_level); + level_msb = min(level_lsb + bits_mapped, max_msb); + level_mask = GENMASK_ULL(level_msb, level_lsb); + + index = (dst_addr & level_mask) >> level_lsb; ++ #ifdef CONFIG_PTP ++ set_pte((pte_t *)(levels[this_level] + index), __pte(prev_level_entry)); ++ #else + *(levels[this_level] + index) = prev_level_entry; ++ #endif + + pfn = virt_to_pfn(levels[this_level]); + prev_level_entry = pte_val(pfn_pte(pfn, +diff --git a/drivers/firmware/efi/arm-runtime.c b/drivers/firmware/efi/arm-runtime.c +index 83092d93f36a..fb12e7d0660a 100644 +--- a/drivers/firmware/efi/arm-runtime.c ++++ b/drivers/firmware/efi/arm-runtime.c +@@ -94,7 +94,11 @@ static int __init arm_enable_runtime_services(void) + return 0; + } + ++ #ifdef CONFIG_PTP ++ efi_memmap_unmap_after_init(); ++ #else + efi_memmap_unmap(); ++ #endif + + mapsize = efi.memmap.desc_size * efi.memmap.nr_map; + +diff --git a/drivers/firmware/efi/memmap.c b/drivers/firmware/efi/memmap.c +index a1180461a445..4c64b6f15717 100644 +--- a/drivers/firmware/efi/memmap.c ++++ b/drivers/firmware/efi/memmap.c +@@ -105,6 +105,26 @@ void __init efi_memmap_unmap(void) + clear_bit(EFI_MEMMAP, &efi.flags); + } + ++#ifdef CONFIG_PTP ++void __init efi_memmap_unmap_after_init(void) ++{ ++ if (!efi_enabled(EFI_MEMMAP)) ++ return; ++ ++ if (!(efi.memmap.flags & EFI_MEMMAP_LATE)) { ++ unsigned long size; ++ ++ size = efi.memmap.desc_size * efi.memmap.nr_map; ++ early_iounmap_after_init((__force void __iomem *)efi.memmap.map, size); ++ } else { ++ memunmap(efi.memmap.map); ++ } ++ ++ efi.memmap.map = NULL; ++ clear_bit(EFI_MEMMAP, &efi.flags); ++} ++#endif ++ + /** + * efi_memmap_init_late - Map efi.memmap with memremap() + * @phys_addr: Physical address of the new EFI memory map +diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c +index a5fbb6ed38ae..81428783b9da 100644 +--- a/drivers/tty/serial/earlycon.c ++++ b/drivers/tty/serial/earlycon.c +@@ -40,7 +40,11 @@ static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size) + { + void __iomem *base; + #ifdef CONFIG_FIX_EARLYCON_MEM ++ #ifdef CONFIG_PTP ++ __iee_set_fixmap_pre_init(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK, FIXMAP_PAGE_IO); ++ #else + set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK); ++ #endif + base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE); + base += paddr & ~PAGE_MASK; + #else +diff --git a/drivers/usb/early/ehci-dbgp.c b/drivers/usb/early/ehci-dbgp.c +index 45b42d8f6453..b71072d6957e 100644 +--- a/drivers/usb/early/ehci-dbgp.c ++++ b/drivers/usb/early/ehci-dbgp.c +@@ -879,7 +879,11 @@ int __init early_dbgp_init(char *s) + * FIXME I don't have the bar size so just guess PAGE_SIZE is more + * than enough. 1K is the biggest I have seen. + */ ++ #ifdef CONFIG_PTP ++ __iee_set_fixmap_pre_init(FIX_DBGP_BASE, bar_val & PAGE_MASK, FIXMAP_PAGE_NOCACHE); ++ #else + set_fixmap_nocache(FIX_DBGP_BASE, bar_val & PAGE_MASK); ++ #endif + ehci_bar = (void __iomem *)__fix_to_virt(FIX_DBGP_BASE); + ehci_bar += bar_val & ~PAGE_MASK; + dbgp_printk("ehci_bar: %p\n", ehci_bar); +diff --git a/fs/coredump.c b/fs/coredump.c +index 9d235fa14ab9..72be355903ca 100644 +--- a/fs/coredump.c ++++ b/fs/coredump.c +@@ -53,6 +53,10 @@ + + #include <trace/events/sched.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + static bool dump_vma_snapshot(struct coredump_params *cprm); + static void free_vma_snapshot(struct coredump_params *cprm); + +@@ -564,7 +568,11 @@ void do_coredump(const kernel_siginfo_t *siginfo) + */ + if (__get_dumpable(cprm.mm_flags) == SUID_DUMP_ROOT) { + /* Setuid core dump mode */ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(cred,GLOBAL_ROOT_UID); ++ #else + cred->fsuid = GLOBAL_ROOT_UID; /* Dump root private */ ++ #endif + need_suid_safe = true; + } + +diff --git a/fs/exec.c b/fs/exec.c +index 04fb89656cc3..3689c5f008ba 100644 +--- a/fs/exec.c ++++ b/fs/exec.c +@@ -76,6 +76,14 @@ + + #include <trace/events/sched.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ ++#ifdef CONFIG_IEE ++#include <asm/iee-token.h> ++#endif ++ + static int bprm_creds_from_file(struct linux_binprm *bprm); + + int suid_dumpable = 0; +@@ -1005,6 +1013,10 @@ static int exec_mmap(struct mm_struct *mm) + if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + activate_mm(active_mm, mm); ++ #ifdef CONFIG_IEE ++ iee_set_token_mm(tsk, mm); ++ iee_set_token_pgd(tsk, mm->pgd); ++ #endif + if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + lru_gen_add_mm(mm); +@@ -1618,12 +1630,20 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) + + if (mode & S_ISUID) { + bprm->per_clear |= PER_CLEAR_ON_SETID; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_euid(bprm->cred, vfsuid_into_kuid(vfsuid)); ++ #else + bprm->cred->euid = vfsuid_into_kuid(vfsuid); ++ #endif + } + + if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { + bprm->per_clear |= PER_CLEAR_ON_SETID; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_egid(bprm->cred, vfsgid_into_kgid(vfsgid)); ++ #else + bprm->cred->egid = vfsgid_into_kgid(vfsgid); ++ #endif + } + } + +diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c +index 3e724cb7ef01..e32e136e4271 100644 +--- a/fs/nfs/flexfilelayout/flexfilelayout.c ++++ b/fs/nfs/flexfilelayout/flexfilelayout.c +@@ -15,6 +15,10 @@ + + #include <linux/sunrpc/metrics.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + #include "flexfilelayout.h" + #include "../nfs4session.h" + #include "../nfs4idmap.h" +@@ -502,8 +506,13 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, + rc = -ENOMEM; + if (!kcred) + goto out_err_free; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(kcred,uid); ++ iee_set_cred_fsgid(kcred,gid); ++ #else + kcred->fsuid = uid; + kcred->fsgid = gid; ++ #endif + cred = RCU_INITIALIZER(kcred); + + if (lgr->range.iomode == IOMODE_READ) +diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c +index 25a7c771cfd8..b15ab8e33e0e 100644 +--- a/fs/nfs/nfs4idmap.c ++++ b/fs/nfs/nfs4idmap.c +@@ -48,6 +48,10 @@ + #include <linux/module.h> + #include <linux/user_namespace.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + #include "internal.h" + #include "netns.h" + #include "nfs4idmap.h" +@@ -226,8 +230,13 @@ int nfs_idmap_init(void) + goto failed_reg_legacy; + + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(cred,keyring); ++ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); ++ #else + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; ++ #endif + id_resolver_cache = cred; + return 0; + +diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c +index e6beaaf4f170..e89385fd81f1 100644 +--- a/fs/nfsd/auth.c ++++ b/fs/nfsd/auth.c +@@ -2,6 +2,9 @@ + /* Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> */ + + #include <linux/sched.h> ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif + #include "nfsd.h" + #include "auth.h" + +@@ -32,22 +35,40 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) + if (!new) + return -ENOMEM; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,rqstp->rq_cred.cr_uid); ++ iee_set_cred_fsgid(new,rqstp->rq_cred.cr_gid); ++ #else + new->fsuid = rqstp->rq_cred.cr_uid; + new->fsgid = rqstp->rq_cred.cr_gid; ++ #endif + + rqgi = rqstp->rq_cred.cr_group_info; + + if (flags & NFSEXP_ALLSQUASH) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,exp->ex_anon_uid); ++ iee_set_cred_fsgid(new,exp->ex_anon_gid); ++ #else + new->fsuid = exp->ex_anon_uid; + new->fsgid = exp->ex_anon_gid; ++ #endif + gi = groups_alloc(0); + if (!gi) + goto oom; + } else if (flags & NFSEXP_ROOTSQUASH) { + if (uid_eq(new->fsuid, GLOBAL_ROOT_UID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,exp->ex_anon_uid); ++ #else + new->fsuid = exp->ex_anon_uid; ++ #endif + if (gid_eq(new->fsgid, GLOBAL_ROOT_GID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsgid(new,exp->ex_anon_gid); ++ #else + new->fsgid = exp->ex_anon_gid; ++ #endif + + gi = groups_alloc(rqgi->ngroups); + if (!gi) +@@ -67,18 +88,35 @@ int nfsd_setuser(struct svc_rqst *rqstp, struct svc_export *exp) + } + + if (uid_eq(new->fsuid, INVALID_UID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,exp->ex_anon_uid); ++ #else + new->fsuid = exp->ex_anon_uid; ++ #endif + if (gid_eq(new->fsgid, INVALID_GID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsgid(new,exp->ex_anon_gid); ++ #else + new->fsgid = exp->ex_anon_gid; ++ #endif + + set_groups(new, gi); + put_group_info(gi); + + if (!uid_eq(new->fsuid, GLOBAL_ROOT_UID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,cap_drop_nfsd_set(new->cap_effective)); ++ #else + new->cap_effective = cap_drop_nfsd_set(new->cap_effective); ++ #endif + else ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,cap_raise_nfsd_set(new->cap_effective, ++ new->cap_permitted)); ++ #else + new->cap_effective = cap_raise_nfsd_set(new->cap_effective, + new->cap_permitted); ++ #endif + put_cred(override_creds(new)); + put_cred(new); + return 0; +diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c +index 4039ffcf90ba..6e0dfa01e01e 100644 +--- a/fs/nfsd/nfs4callback.c ++++ b/fs/nfsd/nfs4callback.c +@@ -41,6 +41,9 @@ + #include "trace.h" + #include "xdr4cb.h" + #include "xdr4.h" ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif + + #define NFSDDBG_FACILITY NFSDDBG_PROC + +@@ -946,8 +949,13 @@ static const struct cred *get_backchannel_cred(struct nfs4_client *clp, struct r + if (!kcred) + return NULL; + +- kcred->fsuid = ses->se_cb_sec.uid; +- kcred->fsgid = ses->se_cb_sec.gid; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_uid(kcred,ses->se_cb_sec.uid); ++ iee_set_cred_gid(kcred,ses->se_cb_sec.gid); ++ #else ++ kcred->uid = ses->se_cb_sec.uid; ++ kcred->gid = ses->se_cb_sec.gid; ++ #endif + return kcred; + } + } +diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c +index 3509e73abe1f..798fe0352841 100644 +--- a/fs/nfsd/nfs4recover.c ++++ b/fs/nfsd/nfs4recover.c +@@ -44,6 +44,10 @@ + #include <linux/sunrpc/clnt.h> + #include <linux/nfsd/cld.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + #include "nfsd.h" + #include "state.h" + #include "vfs.h" +@@ -78,8 +82,13 @@ nfs4_save_creds(const struct cred **original_creds) + if (!new) + return -ENOMEM; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,GLOBAL_ROOT_UID); ++ iee_set_cred_fsgid(new,GLOBAL_ROOT_GID); ++ #else + new->fsuid = GLOBAL_ROOT_UID; + new->fsgid = GLOBAL_ROOT_GID; ++ #endif + *original_creds = override_creds(new); + put_cred(new); + return 0; +diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c +index 355bf0db3235..6cb276dde4e9 100644 +--- a/fs/nfsd/nfsfh.c ++++ b/fs/nfsd/nfsfh.c +@@ -16,6 +16,10 @@ + #include "auth.h" + #include "trace.h" + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + #define NFSDDBG_FACILITY NFSDDBG_FH + + +@@ -223,9 +227,14 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp) + error = nfserrno(-ENOMEM); + goto out; + } ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,cap_raise_nfsd_set(new->cap_effective, ++ new->cap_permitted)); ++ #else + new->cap_effective = + cap_raise_nfsd_set(new->cap_effective, + new->cap_permitted); ++ #endif + put_cred(override_creds(new)); + put_cred(new); + } else { +diff --git a/fs/open.c b/fs/open.c +index 54723fceb776..d83901dc50ff 100644 +--- a/fs/open.c ++++ b/fs/open.c +@@ -35,6 +35,11 @@ + #include <linux/mnt_idmapping.h> + #include <linux/filelock.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ ++ + #include "internal.h" + + int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry, +@@ -414,17 +419,34 @@ static const struct cred *access_override_creds(void) + * routine. + */ + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(override_cred,override_cred->uid); ++ iee_set_cred_fsgid(override_cred,override_cred->gid); ++ #else + override_cred->fsuid = override_cred->uid; + override_cred->fsgid = override_cred->gid; ++ #endif + + if (!issecure(SECURE_NO_SETUID_FIXUP)) { + /* Clear the capabilities if we switch to a non-root user */ + kuid_t root_uid = make_kuid(override_cred->user_ns, 0); + if (!uid_eq(override_cred->uid, root_uid)) ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = override_cred->cap_effective; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_effective(override_cred, tmp_cap); ++ } while (0); ++ #else + cap_clear(override_cred->cap_effective); ++ #endif + else ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(override_cred,override_cred->cap_permitted); ++ #else + override_cred->cap_effective = + override_cred->cap_permitted; ++ #endif + } + + /* +@@ -444,7 +466,11 @@ static const struct cred *access_override_creds(void) + * expecting RCU freeing. But normal thread-synchronous + * cred accesses will keep things non-RCY. + */ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_non_rcu(override_cred,1); ++ #else + override_cred->non_rcu = 1; ++ #endif + + old_cred = override_creds(override_cred); + +diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c +index 033fc0458a3d..2afa31ead2b5 100644 +--- a/fs/overlayfs/dir.c ++++ b/fs/overlayfs/dir.c +@@ -16,6 +16,10 @@ + #include <linux/ratelimit.h> + #include "overlayfs.h" + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + static unsigned short ovl_redirect_max = 256; + module_param_named(redirect_max, ovl_redirect_max, ushort, 0644); + MODULE_PARM_DESC(redirect_max, +@@ -593,8 +597,13 @@ static int ovl_create_or_link(struct dentry *dentry, struct inode *inode, + * create a new inode, so just use the ovl mounter's + * fs{u,g}id. + */ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(override_cred,inode->i_uid); ++ iee_set_cred_fsgid(override_cred,inode->i_gid); ++ #else + override_cred->fsuid = inode->i_uid; + override_cred->fsgid = inode->i_gid; ++ #endif + err = security_dentry_create_files_as(dentry, + attr->mode, &dentry->d_name, old_cred, + override_cred); +diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c +index 2c056d737c27..9ede99ddb04b 100644 +--- a/fs/overlayfs/super.c ++++ b/fs/overlayfs/super.c +@@ -21,6 +21,10 @@ + #include "overlayfs.h" + #include "params.h" + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + MODULE_AUTHOR("Miklos Szeredi <miklos@szeredi.hu>"); + MODULE_DESCRIPTION("Overlay filesystem"); + MODULE_LICENSE("GPL"); +@@ -1485,7 +1489,15 @@ int ovl_fill_super(struct super_block *sb, struct fs_context *fc) + sb->s_export_op = &ovl_export_fid_operations; + + /* Never override disk quota limits or use reserved space */ ++ #ifdef CONFIG_CREDP ++ { ++ kernel_cap_t tmp = cred->cap_effective; ++ cap_lower(tmp, CAP_SYS_RESOURCE); ++ iee_set_cred_cap_effective(cred, tmp); ++ } ++ #else + cap_lower(cred->cap_effective, CAP_SYS_RESOURCE); ++ #endif + + sb->s_magic = OVERLAYFS_SUPER_MAGIC; + sb->s_xattr = ofs->config.userxattr ? ovl_user_xattr_handlers : +diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c +index af7849e5974f..4ac2f0e65955 100644 +--- a/fs/smb/client/cifs_spnego.c ++++ b/fs/smb/client/cifs_spnego.c +@@ -18,6 +18,10 @@ + #include "cifs_spnego.h" + #include "cifs_debug.h" + #include "cifsproto.h" ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + static const struct cred *spnego_cred; + + /* create a new cifs key */ +@@ -212,8 +216,13 @@ init_cifs_spnego(void) + * the results it looks up + */ + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(cred,keyring); ++ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); ++ #else + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; ++ #endif + spnego_cred = cred; + + cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring)); +diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c +index f5b6df82e857..9f0ff045836d 100644 +--- a/fs/smb/client/cifsacl.c ++++ b/fs/smb/client/cifsacl.c +@@ -26,6 +26,10 @@ + #include "cifs_fs_sb.h" + #include "cifs_unicode.h" + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + /* security id for everyone/world system group */ + static const struct cifs_sid sid_everyone = { + 1, 1, {0, 0, 0, 0, 0, 1}, {0} }; +@@ -491,8 +495,13 @@ init_cifs_idmap(void) + /* instruct request_key() to use this special keyring as a cache for + * the results it looks up */ + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(cred,keyring); ++ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); ++ #else + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; ++ #endif + root_cred = cred; + + cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring)); +diff --git a/include/asm-generic/early_ioremap.h b/include/asm-generic/early_ioremap.h +index 9d0479f50f97..f501e0f965f8 100644 +--- a/include/asm-generic/early_ioremap.h ++++ b/include/asm-generic/early_ioremap.h +@@ -17,6 +17,9 @@ extern void *early_memremap_ro(resource_size_t phys_addr, + extern void *early_memremap_prot(resource_size_t phys_addr, + unsigned long size, unsigned long prot_val); + extern void early_iounmap(void __iomem *addr, unsigned long size); ++#ifdef CONFIG_PTP ++extern void early_iounmap_after_init(void __iomem *addr, unsigned long size); ++#endif + extern void early_memunmap(void *addr, unsigned long size); + + #if defined(CONFIG_GENERIC_EARLY_IOREMAP) && defined(CONFIG_MMU) +diff --git a/include/asm-generic/fixmap.h b/include/asm-generic/fixmap.h +index 8cc7b09c1bc7..83158589a545 100644 +--- a/include/asm-generic/fixmap.h ++++ b/include/asm-generic/fixmap.h +@@ -70,6 +70,24 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) + __set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR) + #endif + ++#ifdef CONFIG_PTP ++#ifndef clear_fixmap_init ++#define clear_fixmap_init(idx) \ ++ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR) ++#endif ++ ++#define __iee_set_fixmap_offset_pre_init(idx, phys, flags) \ ++({ \ ++ unsigned long ________addr; \ ++ __iee_set_fixmap_pre_init(idx, phys, flags); \ ++ ________addr = fix_to_virt(idx) + ((phys) & (PAGE_SIZE - 1)); \ ++ ________addr; \ ++}) ++ ++#define iee_set_fixmap_offset_pre_init(idx, phys) \ ++ __iee_set_fixmap_offset_pre_init(idx, phys, FIXMAP_PAGE_NORMAL) ++#endif ++ + /* Return a pointer with offset calculated */ + #define __set_fixmap_offset(idx, phys, flags) \ + ({ \ +diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h +index c75d4a753849..506ff9662e02 100644 +--- a/include/asm-generic/pgalloc.h ++++ b/include/asm-generic/pgalloc.h +@@ -7,6 +7,10 @@ + #define GFP_PGTABLE_KERNEL (GFP_KERNEL | __GFP_ZERO) + #define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT) + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + /** + * __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table + * @mm: the mm_struct of the current context +@@ -23,6 +27,13 @@ static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm) + + if (!ptdesc) + return NULL; ++ ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(ptdesc_address(ptdesc))); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)ptdesc_address(ptdesc)); ++ #endif ++ + return ptdesc_address(ptdesc); + } + +@@ -46,6 +57,11 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) + */ + static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) + { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(pte)); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)pte); ++ #endif + pagetable_free(virt_to_ptdesc(pte)); + } + +@@ -73,6 +89,13 @@ static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp) + return NULL; + } + ++ #ifdef CONFIG_PTP ++ pte_t *pte = (pte_t *)page_address(ptdesc_page(ptdesc)); ++ unsigned long iee_addr = __phys_to_iee(__pa(pte)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)pte); ++ #endif ++ + return ptdesc_page(ptdesc); + } + +@@ -103,9 +126,20 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm) + */ + static inline void pte_free(struct mm_struct *mm, struct page *pte_page) + { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif ++ + struct ptdesc *ptdesc = page_ptdesc(pte_page); + + pagetable_pte_dtor(ptdesc); ++ ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(page_address(pte_page))); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)page_address(pte_page)); ++ #endif ++ + pagetable_free(ptdesc); + } + +@@ -145,10 +179,21 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) + #ifndef __HAVE_ARCH_PMD_FREE + static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) + { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif ++ + struct ptdesc *ptdesc = virt_to_ptdesc(pmd); + + BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); + pagetable_pmd_dtor(ptdesc); ++ ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(pmd)); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)pmd); ++ #endif ++ + pagetable_free(ptdesc); + } + #endif +@@ -190,7 +235,16 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) + + static inline void __pud_free(struct mm_struct *mm, pud_t *pud) + { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif ++ + BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(pud)); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)pud); ++ #endif + pagetable_free(virt_to_ptdesc(pud)); + } + +diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h +index 200853042fc7..9d733afced53 100644 +--- a/include/asm-generic/vmlinux.lds.h ++++ b/include/asm-generic/vmlinux.lds.h +@@ -346,6 +346,17 @@ + KEEP(*(.dtb.init.rodata)) \ + __dtb_end = .; + ++#ifdef CONFIG_KOI ++#define KOI_DATA() \ ++ . = ALIGN(PAGE_SIZE); \ ++ __koi_data_start = .; \ ++ *(.data..koi) \ ++ . = ALIGN(PAGE_SIZE); \ ++ __koi_data_end = .; ++#else ++#define KOI_DATA() ++#endif ++ + /* + * .data section + */ +@@ -370,8 +381,8 @@ + BRANCH_PROFILE() \ + TRACE_PRINTKS() \ + BPF_RAW_TP() \ +- TRACEPOINT_STR() +- ++ TRACEPOINT_STR() \ ++ KOI_DATA() + /* + * Data section helpers + */ +@@ -1093,6 +1104,14 @@ + * They will fit only a subset of the architectures + */ + ++#ifdef CONFIG_CREDP ++ #define CRED_DATA \ ++ . = ALIGN(PAGE_SIZE); \ ++ *(.iee.cred) \ ++ . = ALIGN(PAGE_SIZE); ++#else ++ #define CRED_DATA ++#endif + + /* + * Writeable data. +@@ -1110,6 +1129,7 @@ + . = ALIGN(PAGE_SIZE); \ + .data : AT(ADDR(.data) - LOAD_OFFSET) { \ + INIT_TASK_DATA(inittask) \ ++ CRED_DATA \ + NOSAVE_DATA \ + PAGE_ALIGNED_DATA(pagealigned) \ + CACHELINE_ALIGNED_DATA(cacheline) \ +diff --git a/include/linux/cred.h b/include/linux/cred.h +index e01c6d094a30..cceb4842b619 100644 +--- a/include/linux/cred.h ++++ b/include/linux/cred.h +@@ -18,6 +18,10 @@ + #include <linux/sched/user.h> + #include <linux/kabi.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-def.h> ++#endif ++ + struct cred; + struct inode; + +@@ -153,6 +157,22 @@ struct cred { + KABI_RESERVE(4) + } __randomize_layout; + ++#ifdef CONFIG_CREDP ++extern unsigned long long iee_rw_gate(int flag, ...); ++static void iee_set_cred_non_rcu(struct cred *cred, int non_rcu) ++{ ++ iee_rw_gate(IEE_OP_SET_CRED_NON_RCU,cred,non_rcu); ++ *(int *)(&(((struct rcu_head *)(cred->rcu.func))->next)) = non_rcu; ++} ++ ++static bool noinline iee_set_cred_atomic_op_usage(struct cred *cred, int flag, int nr) ++{ ++ bool ret; ++ ret = iee_rw_gate(IEE_OP_SET_CRED_ATOP_USAGE,cred,flag,nr); ++ return ret; ++} ++#endif ++ + extern void __put_cred(struct cred *); + extern void exit_creds(struct task_struct *); + extern int copy_creds(struct task_struct *, unsigned long); +@@ -189,7 +209,11 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred) + */ + static inline struct cred *get_new_cred_many(struct cred *cred, int nr) + { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_atomic_op_usage(cred, AT_ADD, nr); ++ #else + atomic_long_add(nr, &cred->usage); ++ #endif + return cred; + } + +@@ -202,7 +226,7 @@ static inline struct cred *get_new_cred_many(struct cred *cred, int nr) + */ + static inline struct cred *get_new_cred(struct cred *cred) + { +- return get_new_cred_many(cred, 1); ++ return get_new_cred_many(cred, 1); // XXXzgc atomic_inc -> get_new_cred_many + } + + /** +@@ -224,7 +248,11 @@ static inline const struct cred *get_cred_many(const struct cred *cred, int nr) + struct cred *nonconst_cred = (struct cred *) cred; + if (!cred) + return cred; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_non_rcu(nonconst_cred,0); ++ #else + nonconst_cred->non_rcu = 0; ++ #endif + return get_new_cred_many(nonconst_cred, nr); + } + +@@ -247,9 +275,19 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred) + struct cred *nonconst_cred = (struct cred *) cred; + if (!cred) + return NULL; ++ #ifdef CONFIG_CREDP ++ if (!iee_set_cred_atomic_op_usage(nonconst_cred,AT_INC_NOT_ZERO,0)) ++ return NULL; ++ #else + if (!atomic_long_inc_not_zero(&nonconst_cred->usage)) + return NULL; ++ #endif ++ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_non_rcu(nonconst_cred,0); ++ #else + nonconst_cred->non_rcu = 0; ++ #endif + return cred; + } + +@@ -270,8 +308,13 @@ static inline void put_cred_many(const struct cred *_cred, int nr) + struct cred *cred = (struct cred *) _cred; + + if (cred) { ++ #ifdef CONFIG_CREDP ++ if (iee_set_cred_atomic_op_usage(cred,AT_SUB_AND_TEST,nr)) ++ __put_cred(cred); ++ #else + if (atomic_long_sub_and_test(nr, &cred->usage)) + __put_cred(cred); ++ #endif + } + } + +diff --git a/include/linux/efi.h b/include/linux/efi.h +index 9ed79128458c..970cc4f7068b 100644 +--- a/include/linux/efi.h ++++ b/include/linux/efi.h +@@ -740,6 +740,15 @@ extern int __init __efi_memmap_init(struct efi_memory_map_data *data); + extern int __init efi_memmap_init_early(struct efi_memory_map_data *data); + extern int __init efi_memmap_init_late(phys_addr_t addr, unsigned long size); + extern void __init efi_memmap_unmap(void); ++#ifdef CONFIG_PTP ++extern void __init efi_memmap_unmap_after_init(void); ++#endif ++extern int __init efi_memmap_install(struct efi_memory_map_data *data); ++extern int __init efi_memmap_split_count(efi_memory_desc_t *md, ++ struct range *range); ++extern void __init efi_memmap_insert(struct efi_memory_map *old_memmap, ++ void *buf, struct efi_mem_range *mem); ++extern void __init efi_print_memmap(void); + + #ifdef CONFIG_EFI_ESRT + extern void __init efi_esrt_init(void); +diff --git a/include/linux/iee-func.h b/include/linux/iee-func.h +new file mode 100644 +index 000000000000..79171de67c2a +--- /dev/null ++++ b/include/linux/iee-func.h +@@ -0,0 +1,27 @@ ++#ifndef _LINUX_IEE_FUNC_H ++#define _LINUX_IEE_FUNC_H ++ ++#ifdef CONFIG_IEE ++// Declare the __entry_task. ++__attribute__((aligned(PAGE_SIZE))) DECLARE_PER_CPU(struct task_struct *[PAGE_SIZE/sizeof(struct task_struct *)], __entry_task); ++ ++extern unsigned long long iee_rw_gate(int flag, ...); ++extern u32 get_cpu_asid_bits(void); ++extern unsigned long arm64_mm_context_get(struct mm_struct *mm); ++extern void set_iee_page_valid(unsigned long addr); ++extern void set_iee_page_invalid(unsigned long addr); ++extern void iee_set_logical_mem_ro(unsigned long addr); ++extern void iee_set_logical_mem_rw(unsigned long addr); ++extern void iee_set_token_mm(struct task_struct *tsk, struct mm_struct *mm); ++extern void iee_set_token_pgd(struct task_struct *tsk, pgd_t *pgd); ++extern void iee_init_token(struct task_struct *tsk, void *kernel_stack, void *iee_stack); ++extern void iee_free_token(struct task_struct *tsk); ++extern unsigned long iee_read_token_stack(struct task_struct *tsk); ++extern void iee_set_token_page_valid(void *token, void *new); ++extern void iee_set_token_page_invalid(void *token); ++extern void iee_set_kernel_ppage(unsigned long addr); ++extern void iee_set_kernel_upage(unsigned long addr); ++extern void iee_write_in_byte(void *ptr, u64 data, int length); ++#endif ++ ++#endif +\ No newline at end of file +diff --git a/include/linux/module.h b/include/linux/module.h +index 4db2878d9e42..ef8d51994017 100644 +--- a/include/linux/module.h ++++ b/include/linux/module.h +@@ -606,6 +606,7 @@ struct module { + KABI_RESERVE(2) + KABI_RESERVE(3) + KABI_RESERVE(4) ++ + } ____cacheline_aligned __randomize_layout; + #ifndef MODULE_ARCH_INIT + #define MODULE_ARCH_INIT {} +diff --git a/include/linux/sched.h b/include/linux/sched.h +index f40411aa7b70..297becfbc8e3 100644 +--- a/include/linux/sched.h ++++ b/include/linux/sched.h +@@ -773,6 +773,24 @@ struct task_struct_resvd { + struct task_struct *task; + }; + ++#if defined(CONFIG_IEE) || defined(CONFIG_KOI) ++struct task_token { ++#ifdef CONFIG_IEE ++ struct mm_struct *mm; /* VA */ ++ pgd_t *pgd; /* Logical VA */ ++ void *iee_stack; /* VA */ ++ bool valid; ++ void *kernel_stack; /* VA */ ++#endif ++#ifdef CONFIG_KOI ++ void *koi_kernel_stack; /* VA */ ++ void *koi_stack; /* VA */ ++ void *koi_stack_base; /* VA */ ++ unsigned long current_ttbr1; ++#endif ++}; ++#endif ++ + struct task_struct { + #ifdef CONFIG_THREAD_INFO_IN_TASK + /* +@@ -795,6 +813,7 @@ struct task_struct { + randomized_struct_fields_start + + void *stack; ++ + refcount_t usage; + /* Per task flags (PF_*), defined further below: */ + unsigned int flags; +diff --git a/init/main.c b/init/main.c +index 803332dd3d90..0f8d6e2744c2 100644 +--- a/init/main.c ++++ b/init/main.c +@@ -102,6 +102,12 @@ + #include <linux/randomize_kstack.h> + #include <net/net_namespace.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#include <asm/iee-si.h> ++#include <linux/stop_machine.h> ++#endif ++ + #include <asm/io.h> + #include <asm/setup.h> + #include <asm/sections.h> +@@ -112,6 +118,10 @@ + + #include <kunit/test.h> + ++#ifdef CONFIG_PTP ++extern void *bm_pte_addr; ++#endif ++ + static int kernel_init(void *); + + /* +@@ -880,6 +890,9 @@ void start_kernel(void) + { + char *command_line; + char *after_dashes; ++ #ifdef CONFIG_IEE ++ unsigned int cpu; ++ #endif + + set_task_stack_end_magic(&init_task); + smp_setup_processor_id(); +@@ -904,6 +917,16 @@ void start_kernel(void) + setup_command_line(command_line); + setup_nr_cpu_ids(); + setup_per_cpu_areas(); ++ #ifdef CONFIG_IEE ++ for_each_possible_cpu(cpu) ++ { ++ // Map the __entry_task to IEE. ++ set_iee_page_valid((unsigned long)__phys_to_iee(__pa(SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu])))); ++ // Set the __entry_task of cpu 0 readonly in lm. ++ if(cpu == smp_processor_id()) ++ iee_set_logical_mem_ro((unsigned long)SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu])); ++ } ++ #endif + smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ + boot_cpu_hotplug_init(); + +@@ -1446,6 +1469,9 @@ static int __ref kernel_init(void *unused) + wait_for_completion(&kthreadd_done); + + kernel_init_freeable(); ++ #ifdef CONFIG_PTP ++ iee_set_logical_mem_ro((unsigned long)bm_pte_addr); ++ #endif + /* need to finish all async __init code before freeing the memory */ + async_synchronize_full(); + +@@ -1462,7 +1488,7 @@ static int __ref kernel_init(void *unused) + * to finalize PTI. + */ + pti_finalize(); +- ++ + system_state = SYSTEM_RUNNING; + numa_default_policy(); + +diff --git a/kernel/cred.c b/kernel/cred.c +index c033a201c808..2e44530976d5 100644 +--- a/kernel/cred.c ++++ b/kernel/cred.c +@@ -20,6 +20,11 @@ + #include <linux/cn_proc.h> + #include <linux/uidgid.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#include <linux/iee-func.h> ++#endif ++ + #if 0 + #define kdebug(FMT, ...) \ + printk("[%-5.5s%5u] " FMT "\n", \ +@@ -34,6 +39,9 @@ do { \ + #endif + + static struct kmem_cache *cred_jar; ++#ifdef CONFIG_CREDP ++static struct kmem_cache *rcu_jar; ++#endif + + /* init to 2 - one for init_task, one to ensure it is never freed */ + static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) }; +@@ -41,6 +49,32 @@ static struct group_info init_groups = { .usage = REFCOUNT_INIT(2) }; + /* + * The initial credentials for the initial task + */ ++#ifdef CONFIG_CREDP ++struct cred init_cred __section(".iee.cred") = { ++ .usage = ATOMIC_INIT(4), ++#ifdef CONFIG_DEBUG_CREDENTIALS ++ .subscribers = ATOMIC_INIT(2), ++ .magic = CRED_MAGIC, ++#endif ++ .uid = GLOBAL_ROOT_UID, ++ .gid = GLOBAL_ROOT_GID, ++ .suid = GLOBAL_ROOT_UID, ++ .sgid = GLOBAL_ROOT_GID, ++ .euid = GLOBAL_ROOT_UID, ++ .egid = GLOBAL_ROOT_GID, ++ .fsuid = GLOBAL_ROOT_UID, ++ .fsgid = GLOBAL_ROOT_GID, ++ .securebits = SECUREBITS_DEFAULT, ++ .cap_inheritable = CAP_EMPTY_SET, ++ .cap_permitted = CAP_FULL_SET, ++ .cap_effective = CAP_FULL_SET, ++ .cap_bset = CAP_FULL_SET, ++ .user = INIT_USER, ++ .user_ns = &init_user_ns, ++ .group_info = &init_groups, ++ .ucounts = &init_ucounts, ++}; ++#else + struct cred init_cred = { + .usage = ATOMIC_INIT(4), + .uid = GLOBAL_ROOT_UID, +@@ -61,13 +95,43 @@ struct cred init_cred = { + .group_info = &init_groups, + .ucounts = &init_ucounts, + }; ++#endif ++ ++static inline void set_cred_subscribers(struct cred *cred, int n) ++{ ++#ifdef CONFIG_DEBUG_CREDENTIALS ++ atomic_set(&cred->subscribers, n); ++#endif ++} ++ ++static inline int read_cred_subscribers(const struct cred *cred) ++{ ++#ifdef CONFIG_DEBUG_CREDENTIALS ++ return atomic_read(&cred->subscribers); ++#else ++ return 0; ++#endif ++} ++ ++static inline void alter_cred_subscribers(const struct cred *_cred, int n) ++{ ++#ifdef CONFIG_DEBUG_CREDENTIALS ++ struct cred *cred = (struct cred *) _cred; ++ ++ atomic_add(n, &cred->subscribers); ++#endif ++} + + /* + * The RCU callback to actually dispose of a set of credentials + */ + static void put_cred_rcu(struct rcu_head *rcu) + { ++ #ifdef CONFIG_CREDP ++ struct cred *cred = *(struct cred **)(rcu + 1); ++ #else + struct cred *cred = container_of(rcu, struct cred, rcu); ++ #endif + + kdebug("put_cred_rcu(%p)", cred); + +@@ -86,6 +150,9 @@ static void put_cred_rcu(struct rcu_head *rcu) + if (cred->ucounts) + put_ucounts(cred->ucounts); + put_user_ns(cred->user_ns); ++ #ifdef CONFIG_CREDP ++ kmem_cache_free(rcu_jar, (struct rcu_head *)(cred->rcu.func)); ++ #endif + kmem_cache_free(cred_jar, cred); + } + +@@ -104,10 +171,22 @@ void __put_cred(struct cred *cred) + BUG_ON(cred == current->cred); + BUG_ON(cred == current->real_cred); + ++ #ifdef CONFIG_CREDP ++ if (*(int *)(&(((struct rcu_head *)(cred->rcu.func))->next))) ++ #else + if (cred->non_rcu) ++ #endif ++ #ifdef CONFIG_CREDP ++ put_cred_rcu((struct rcu_head *)(cred->rcu.func)); ++ #else + put_cred_rcu(&cred->rcu); ++ #endif + else ++ #ifdef CONFIG_CREDP ++ call_rcu((struct rcu_head *)(cred->rcu.func), put_cred_rcu); ++ #else + call_rcu(&cred->rcu, put_cred_rcu); ++ #endif + } + EXPORT_SYMBOL(__put_cred); + +@@ -178,7 +257,18 @@ struct cred *cred_alloc_blank(void) + if (!new) + return NULL; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_rcu(new,kmem_cache_zalloc(rcu_jar, GFP_KERNEL)); ++ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; ++ iee_set_cred_atomic_set_usage(new,1); ++ #else + atomic_long_set(&new->usage, 1); ++ #endif ++ ++ #ifdef CONFIG_DEBUG_CREDENTIALS ++ new->magic = CRED_MAGIC; ++ #endif ++ + if (security_cred_alloc_blank(new, GFP_KERNEL_ACCOUNT) < 0) + goto error; + +@@ -213,13 +303,25 @@ struct cred *prepare_creds(void) + if (!new) + return NULL; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_rcu(new,kmem_cache_alloc(rcu_jar, GFP_KERNEL)); ++ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; ++ #endif ++ + kdebug("prepare_creds() alloc %p", new); + + old = task->cred; ++ #ifdef CONFIG_CREDP ++ iee_copy_cred(old,new); ++ ++ iee_set_cred_non_rcu(new,0); ++ iee_set_cred_atomic_set_usage(new,1); ++ #else + memcpy(new, old, sizeof(struct cred)); + + new->non_rcu = 0; + atomic_long_set(&new->usage, 1); ++ #endif + get_group_info(new->group_info); + get_uid(new->user); + get_user_ns(new->user_ns); +@@ -232,10 +334,18 @@ struct cred *prepare_creds(void) + #endif + + #ifdef CONFIG_SECURITY ++#ifdef CONFIG_CREDP ++ iee_set_cred_security(new,NULL); ++#else + new->security = NULL; ++#endif + #endif + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_ucounts(new, get_ucounts(new->ucounts)); ++ #else + new->ucounts = get_ucounts(new->ucounts); ++ #endif + if (!new->ucounts) + goto error; + +@@ -265,15 +375,30 @@ struct cred *prepare_exec_creds(void) + #ifdef CONFIG_KEYS + /* newly exec'd tasks don't get a thread keyring */ + key_put(new->thread_keyring); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(new,NULL); ++ #else + new->thread_keyring = NULL; ++ #endif + + /* inherit the session keyring; new process keyring */ + key_put(new->process_keyring); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_process_keyring(new,NULL); ++ #else + new->process_keyring = NULL; ++ #endif + #endif + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,new->euid); ++ iee_set_cred_suid(new,new->euid); ++ iee_set_cred_fsgid(new,new->egid); ++ iee_set_cred_sgid(new,new->egid); ++ #else + new->suid = new->fsuid = new->euid; + new->sgid = new->fsgid = new->egid; ++ #endif + + return new; + } +@@ -327,7 +452,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags) + * had one */ + if (new->thread_keyring) { + key_put(new->thread_keyring); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(new,NULL); ++ #else + new->thread_keyring = NULL; ++ #endif + if (clone_flags & CLONE_THREAD) + install_thread_keyring_to_cred(new); + } +@@ -337,7 +466,11 @@ int copy_creds(struct task_struct *p, unsigned long clone_flags) + */ + if (!(clone_flags & CLONE_THREAD)) { + key_put(new->process_keyring); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_process_keyring(new,NULL); ++ #else + new->process_keyring = NULL; ++ #endif + } + #endif + +@@ -594,7 +727,11 @@ int set_cred_ucounts(struct cred *new) + if (!(new_ucounts = alloc_ucounts(new->user_ns, new->uid))) + return -EAGAIN; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_ucounts(new, new_ucounts); ++ #else + new->ucounts = new_ucounts; ++ #endif + put_ucounts(old_ucounts); + + return 0; +@@ -606,8 +743,21 @@ int set_cred_ucounts(struct cred *new) + void __init cred_init(void) + { + /* allocate a slab in which we can store credentials */ ++ #ifdef CONFIG_CREDP + cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, ++ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT|SLAB_RED_ZONE, NULL); ++ rcu_jar = kmem_cache_create("rcu_jar", sizeof(struct rcu_head) + sizeof(struct cred *), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); ++ // Map init_cred ++ *((struct rcu_head **)(&(init_cred.rcu.func))) = (struct rcu_head *)kmem_cache_zalloc(rcu_jar, GFP_KERNEL); ++ *(struct cred **)(((struct rcu_head *)(init_cred.rcu.func)) + 1) = &init_cred; ++ set_iee_page_valid(__phys_to_iee(__pa_symbol(&init_cred))); ++ iee_set_logical_mem_ro((unsigned long)&init_cred); ++ iee_set_logical_mem_ro((unsigned long)__va(__pa_symbol(&init_cred))); ++ #else ++ cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred), 0, ++ SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); ++ #endif + } + + /** +@@ -638,29 +788,56 @@ struct cred *prepare_kernel_cred(struct task_struct *daemon) + if (!new) + return NULL; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_rcu(new,kmem_cache_alloc(rcu_jar, GFP_KERNEL)); ++ *(struct cred **)(((struct rcu_head *)(new->rcu.func)) + 1) = new; ++ #endif ++ + kdebug("prepare_kernel_cred() alloc %p", new); + + old = get_task_cred(daemon); + ++ #ifdef CONFIG_CREDP ++ iee_copy_cred(old,new); ++ iee_set_cred_non_rcu(new,0); ++ iee_set_cred_atomic_set_usage(new,1); ++ #else + *new = *old; + new->non_rcu = 0; + atomic_long_set(&new->usage, 1); ++ #endif + get_uid(new->user); + get_user_ns(new->user_ns); + get_group_info(new->group_info); + + #ifdef CONFIG_KEYS ++#ifdef CONFIG_CREDP ++ iee_set_cred_session_keyring(new,NULL); ++ iee_set_cred_process_keyring(new,NULL); ++ iee_set_cred_thread_keyring(new,NULL); ++ iee_set_cred_request_key_auth(new,NULL); ++ iee_set_cred_jit_keyring(new,KEY_REQKEY_DEFL_THREAD_KEYRING); ++#else + new->session_keyring = NULL; + new->process_keyring = NULL; + new->thread_keyring = NULL; + new->request_key_auth = NULL; + new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + #endif ++#endif + + #ifdef CONFIG_SECURITY ++#ifdef CONFIG_CREDP ++ iee_set_cred_security(new,NULL); ++#else + new->security = NULL; + #endif ++#endif ++ #ifdef CONFIG_CREDP ++ iee_set_cred_ucounts(new, get_ucounts(new->ucounts)); ++ #else + new->ucounts = get_ucounts(new->ucounts); ++ #endif + if (!new->ucounts) + goto error; + +@@ -727,8 +904,13 @@ int set_create_files_as(struct cred *new, struct inode *inode) + { + if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid)) + return -EINVAL; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,inode->i_uid); ++ iee_set_cred_fsgid(new,inode->i_gid); ++ #else + new->fsuid = inode->i_uid; + new->fsgid = inode->i_gid; ++ #endif + return security_kernel_create_files_as(new, inode); + } + EXPORT_SYMBOL(set_create_files_as); +diff --git a/kernel/exit.c b/kernel/exit.c +index 21a59a6e1f2e..d21a109f0497 100644 +--- a/kernel/exit.c ++++ b/kernel/exit.c +@@ -74,6 +74,10 @@ + #include <asm/unistd.h> + #include <asm/mmu_context.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#endif ++ + /* + * The default value should be high enough to not crash a system that randomly + * crashes its kernel from time to time, but low enough to at least not permit +@@ -558,6 +562,10 @@ static void exit_mm(void) + smp_mb__after_spinlock(); + local_irq_disable(); + current->mm = NULL; ++ #ifdef CONFIG_IEE ++ iee_set_token_mm(current, NULL); ++ iee_set_token_pgd(current, NULL); ++ #endif + membarrier_update_current_mm(NULL); + enter_lazy_tlb(mm, current); + local_irq_enable(); +diff --git a/kernel/fork.c b/kernel/fork.c +index e033388b11bd..c93e18a4f0b3 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -115,6 +115,10 @@ + #define CREATE_TRACE_POINTS + #include <trace/events/task.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#endif ++ + /* + * Minimum number of threads to boot the kernel + */ +@@ -128,14 +132,14 @@ + /* + * Protected counters by write_lock_irq(&tasklist_lock) + */ +-unsigned long total_forks; /* Handle normal Linux uptimes. */ +-int nr_threads; /* The idle threads do not count.. */ ++unsigned long total_forks; /* Handle normal Linux uptimes. */ ++int nr_threads; /* The idle threads do not count.. */ + +-static int max_threads; /* tunable limit on nr_threads */ ++static int max_threads; /* tunable limit on nr_threads */ + +-#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) ++#define NAMED_ARRAY_INDEX(x) [x] = __stringify(x) + +-static const char * const resident_page_types[] = { ++static const char *const resident_page_types[] = { + NAMED_ARRAY_INDEX(MM_FILEPAGES), + NAMED_ARRAY_INDEX(MM_ANONPAGES), + NAMED_ARRAY_INDEX(MM_SWAPENTS), +@@ -144,7 +148,7 @@ static const char * const resident_page_types[] = { + + DEFINE_PER_CPU(unsigned long, process_counts) = 0; + +-__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ ++__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ + + #ifdef CONFIG_PROVE_RCU + int lockdep_tasklist_lock_is_held(void) +@@ -159,7 +163,7 @@ int nr_processes(void) + int cpu; + int total = 0; + +- for_each_possible_cpu(cpu) ++ for_each_possible_cpu (cpu) + total += per_cpu(process_counts, cpu); + + return total; +@@ -190,7 +194,7 @@ static inline void free_task_struct(struct task_struct *tsk) + * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a + * kmemcache based allocator. + */ +-# if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) ++#if THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) + + # ifdef CONFIG_VMAP_STACK + /* +@@ -311,8 +315,8 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) + * so memcg accounting is performed manually on assigning/releasing + * stacks to tasks. Drop __GFP_ACCOUNT. + */ +- stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, +- VMALLOC_START, VMALLOC_END, ++ stack = __vmalloc_node_range(THREAD_SIZE, THREAD_ALIGN, VMALLOC_START, ++ VMALLOC_END, + THREADINFO_GFP & ~__GFP_ACCOUNT, + PAGE_KERNEL, + 0, node, __builtin_return_address(0)); +@@ -410,9 +414,10 @@ static void free_thread_stack(struct task_struct *tsk) + + void thread_stack_cache_init(void) + { +- thread_stack_cache = kmem_cache_create_usercopy("thread_stack", +- THREAD_SIZE, THREAD_SIZE, 0, 0, +- THREAD_SIZE, NULL); ++ thread_stack_cache = ++ kmem_cache_create_usercopy("thread_stack", THREAD_SIZE, ++ THREAD_SIZE, 0, 0, THREAD_SIZE, ++ NULL); + BUG_ON(thread_stack_cache == NULL); + } + +@@ -502,7 +507,8 @@ struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) + + struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) + { +- struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); ++ struct vm_area_struct *new = ++ kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); + + if (!new) + return NULL; +@@ -602,8 +608,15 @@ void put_task_stack(struct task_struct *tsk) + } + #endif + ++#ifdef CONFIG_KOI ++extern s64 koi_offset; ++#endif ++ + void free_task(struct task_struct *tsk) + { ++ #ifdef CONFIG_IEE ++ void *iee_stack; ++ #endif + #ifdef CONFIG_SECCOMP + WARN_ON_ONCE(tsk->seccomp.filter); + #endif +@@ -633,6 +646,45 @@ void free_task(struct task_struct *tsk) + if (dynamic_affinity_enabled()) + sched_prefer_cpus_free(tsk); + #endif ++#ifdef CONFIG_IEE ++ // Free iee stack. ++ iee_stack = (void *)iee_read_token_stack(tsk); ++ if (iee_stack) { ++ iee_set_kernel_ppage( ++ (unsigned long)(iee_stack - PAGE_SIZE * 4)); ++ free_pages((unsigned long)(iee_stack - PAGE_SIZE * 4), 3); ++ } ++ // Free task_token. ++ // Empty the token ++ iee_free_token(tsk); ++ ++#ifdef CONFIG_KOI ++ // Free koi stack. ++ unsigned long koi_stack = iee_rw_gate(IEE_READ_KOI_STACK_BASE, current); ++ if (koi_stack != 0) ++ free_pages(koi_stack, 2); ++#endif ++#else ++#ifdef CONFIG_KOI ++// free koi stack ++ struct task_token *token = (struct task_token *)((unsigned long)current + koi_offset); ++ unsigned long flags; ++ local_irq_save(flags); ++ asm volatile( ++ "at s1e1r, %0\n" ++ "isb\n" ++ : ++ :"r"(token)); ++ unsigned long res = read_sysreg(par_el1); ++ local_irq_restore(flags); ++ if (!(res & 0x1)) { ++ unsigned long koi_stack = token->koi_stack_base; ++ if (koi_stack != 0) ++ free_pages(koi_stack, 2); ++ } ++#endif ++#endif ++ + #ifdef CONFIG_QOS_SCHED_SMART_GRID + if (smart_grid_enabled()) + sched_grid_qos_free(tsk); +@@ -657,7 +709,7 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) + + #ifdef CONFIG_MMU + static __latent_entropy int dup_mmap(struct mm_struct *mm, +- struct mm_struct *oldmm) ++ struct mm_struct *oldmm) + { + struct vm_area_struct *mpnt, *tmp; + int retval; +@@ -773,7 +825,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, + flush_dcache_mmap_lock(mapping); + /* insert tmp into the share list, just after mpnt */ + vma_interval_tree_insert_after(tmp, mpnt, +- &mapping->i_mmap); ++ &mapping->i_mmap); + flush_dcache_mmap_unlock(mapping); + i_mmap_unlock_write(mapping); + } +@@ -842,7 +894,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) + mmap_write_unlock(oldmm); + return 0; + } +-#define mm_alloc_pgd(mm) (0) ++#define mm_alloc_pgd(mm) (0) + #define mm_free_pgd(mm) + #endif /* CONFIG_MMU */ + +@@ -850,20 +902,22 @@ static void check_mm(struct mm_struct *mm) + { + int i; + +- BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, +- "Please make sure 'struct resident_page_types[]' is updated as well"); ++ BUILD_BUG_ON_MSG( ++ ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS, ++ "Please make sure 'struct resident_page_types[]' is updated as well"); + + for (i = 0; i < NR_MM_COUNTERS; i++) { + long x = mm_counter_sum(mm, i); + + if (unlikely(x)) +- pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", +- mm, resident_page_types[i], x); ++ pr_alert( ++ "BUG: Bad rss-counter state mm:%p type:%s val:%ld\n", ++ mm, resident_page_types[i], x); + } + + if (mm_pgtables_bytes(mm)) + pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", +- mm_pgtables_bytes(mm)); ++ mm_pgtables_bytes(mm)); + + #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS + VM_BUG_ON_MM(mm->pmd_huge_pte, mm); +@@ -1014,14 +1068,6 @@ void __put_task_struct(struct task_struct *tsk) + } + EXPORT_SYMBOL_GPL(__put_task_struct); + +-void __put_task_struct_rcu_cb(struct rcu_head *rhp) +-{ +- struct task_struct *task = container_of(rhp, struct task_struct, rcu); +- +- __put_task_struct(task); +-} +-EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb); +- + void __init __weak arch_task_cache_init(void) { } + + /* +@@ -1039,8 +1085,8 @@ static void set_max_threads(unsigned int max_threads_suggested) + if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64) + threads = MAX_THREADS; + else +- threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE, +- (u64) THREAD_SIZE * 8UL); ++ threads = div64_u64((u64)nr_pages * (u64)PAGE_SIZE, ++ (u64)THREAD_SIZE * 8UL); + + if (threads > max_threads_suggested) + threads = max_threads_suggested; +@@ -1075,17 +1121,24 @@ void __init fork_init(void) + int i; + #ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR + #ifndef ARCH_MIN_TASKALIGN +-#define ARCH_MIN_TASKALIGN 0 ++#define ARCH_MIN_TASKALIGN 0 + #endif + int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN); + unsigned long useroffset, usersize; + + /* create a slab on which task_structs can be allocated */ + task_struct_whitelist(&useroffset, &usersize); ++ #ifdef CONFIG_IEE + task_struct_cachep = kmem_cache_create_usercopy("task_struct", + arch_task_struct_size, align, +- SLAB_PANIC|SLAB_ACCOUNT, ++ SLAB_PANIC|SLAB_ACCOUNT|SLAB_RED_ZONE, + useroffset, usersize, NULL); ++ #else ++ task_struct_cachep = ++ kmem_cache_create_usercopy("task_struct", arch_task_struct_size, ++ align, SLAB_PANIC | SLAB_ACCOUNT, ++ useroffset, usersize, NULL); ++ #endif + #endif + + /* do the arch specific task caches init */ +@@ -1093,8 +1146,8 @@ void __init fork_init(void) + + set_max_threads(MAX_THREADS); + +- init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2; +- init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2; ++ init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads / 2; ++ init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads / 2; + init_task.signal->rlim[RLIMIT_SIGPENDING] = + init_task.signal->rlim[RLIMIT_NPROC]; + +@@ -1107,8 +1160,8 @@ void __init fork_init(void) + set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY); + + #ifdef CONFIG_VMAP_STACK +- cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", +- NULL, free_vm_stack_cache); ++ cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", NULL, ++ free_vm_stack_cache); + #endif + + scs_init(); +@@ -1118,7 +1171,7 @@ void __init fork_init(void) + } + + int __weak arch_dup_task_struct(struct task_struct *dst, +- struct task_struct *src) ++ struct task_struct *src) + { + *dst = *src; + return 0; +@@ -1129,14 +1182,14 @@ void set_task_stack_end_magic(struct task_struct *tsk) + unsigned long *stackend; + + stackend = end_of_stack(tsk); +- *stackend = STACK_END_MAGIC; /* for overflow detection */ ++ *stackend = STACK_END_MAGIC; /* for overflow detection */ + } + + static bool dup_resvd_task_struct(struct task_struct *dst, + struct task_struct *orig, int node) + { +- dst->_resvd = kzalloc_node(sizeof(struct task_struct_resvd), +- GFP_KERNEL, node); ++ dst->_resvd = kzalloc_node(sizeof(struct task_struct_resvd), GFP_KERNEL, ++ node); + if (!dst->_resvd) + return false; + +@@ -1309,7 +1362,7 @@ static void mm_init_uprobes_state(struct mm_struct *mm) + } + + static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, +- struct user_namespace *user_ns) ++ struct user_namespace *user_ns) + { + mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); + mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); +@@ -1425,8 +1478,8 @@ EXPORT_SYMBOL_GPL(mmput); + #ifdef CONFIG_MMU + static void mmput_async_fn(struct work_struct *work) + { +- struct mm_struct *mm = container_of(work, struct mm_struct, +- async_put_work); ++ struct mm_struct *mm = ++ container_of(work, struct mm_struct, async_put_work); + + __mmput(mm); + } +@@ -1602,13 +1655,12 @@ struct mm_struct *mm_access(struct task_struct *task, unsigned int mode) + struct mm_struct *mm; + int err; + +- err = down_read_killable(&task->signal->exec_update_lock); ++ err = down_read_killable(&task->signal->exec_update_lock); + if (err) + return ERR_PTR(err); + + mm = get_task_mm(task); +- if (mm && mm != current->mm && +- !ptrace_may_access(task, mode)) { ++ if (mm && mm != current->mm && !ptrace_may_access(task, mode)) { + mmput(mm); + mm = ERR_PTR(-EACCES); + } +@@ -1631,7 +1683,7 @@ static void complete_vfork_done(struct task_struct *tsk) + } + + static int wait_for_vfork_done(struct task_struct *child, +- struct completion *vfork) ++ struct completion *vfork) + { + unsigned int state = TASK_UNINTERRUPTIBLE|TASK_KILLABLE|TASK_FREEZABLE; + int killed; +@@ -1682,8 +1734,8 @@ static void mm_release(struct task_struct *tsk, struct mm_struct *mm) + * not set up a proper pointer then tough luck. + */ + put_user(0, tsk->clear_child_tid); +- do_futex(tsk->clear_child_tid, FUTEX_WAKE, +- 1, NULL, NULL, 0, 0); ++ do_futex(tsk->clear_child_tid, FUTEX_WAKE, 1, NULL, ++ NULL, 0, 0); + } + tsk->clear_child_tid = NULL; + } +@@ -1767,6 +1819,10 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) + #endif + + tsk->mm = NULL; ++#ifdef CONFIG_IEE ++ iee_set_token_mm(tsk, NULL); ++ iee_set_token_pgd(tsk, NULL); ++#endif + tsk->active_mm = NULL; + + /* +@@ -1798,6 +1854,10 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) + } + + tsk->mm = mm; ++#ifdef CONFIG_IEE ++ iee_set_token_mm(tsk, mm); ++ iee_set_token_pgd(tsk, mm->pgd); ++#endif + tsk->active_mm = mm; + sched_mm_cid_fork(tsk); + return 0; +@@ -2015,8 +2075,8 @@ static inline void init_task_pid_links(struct task_struct *task) + INIT_HLIST_NODE(&task->pid_links[type]); + } + +-static inline void +-init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid) ++static inline void init_task_pid(struct task_struct *task, enum pid_type type, ++ struct pid *pid) + { + if (type == PIDTYPE_PID) + task->thread_pid = pid; +@@ -2277,6 +2337,12 @@ static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk) + mutex_unlock(&oom_adj_mutex); + } + ++#if defined(CONFIG_KOI) && !defined(CONFIG_IEE) ++extern s64 koi_offset; ++extern int koi_add_page_mapping(unsigned long dst, unsigned long src); ++#endif ++ ++ + #ifdef CONFIG_RV + static void rv_task_fork(struct task_struct *p) + { +@@ -2309,15 +2375,21 @@ __latent_entropy struct task_struct *copy_process( + struct file *pidfile = NULL; + const u64 clone_flags = args->flags; + struct nsproxy *nsp = current->nsproxy; ++ #ifdef CONFIG_IEE ++ gfp_t gfp; ++ void *pstack; ++ #endif + + /* + * Don't allow sharing the root directory with processes in a different + * namespace + */ +- if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS)) ++ if ((clone_flags & (CLONE_NEWNS | CLONE_FS)) == ++ (CLONE_NEWNS | CLONE_FS)) + return ERR_PTR(-EINVAL); + +- if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS)) ++ if ((clone_flags & (CLONE_NEWUSER | CLONE_FS)) == ++ (CLONE_NEWUSER | CLONE_FS)) + return ERR_PTR(-EINVAL); + + /* +@@ -2342,7 +2414,7 @@ __latent_entropy struct task_struct *copy_process( + * from creating siblings. + */ + if ((clone_flags & CLONE_PARENT) && +- current->signal->flags & SIGNAL_UNKILLABLE) ++ current->signal->flags & SIGNAL_UNKILLABLE) + return ERR_PTR(-EINVAL); + + /* +@@ -2387,6 +2459,15 @@ __latent_entropy struct task_struct *copy_process( + p = dup_task_struct(current, node); + if (!p) + goto fork_out; ++ #ifdef CONFIG_IEE ++ // Alloc iee stack. ++ gfp = GFP_KERNEL; ++ pstack = (void *)__get_free_pages(gfp, 3); ++ iee_set_kernel_upage((unsigned long)pstack); ++ // Init token. ++ iee_init_token(p, NULL, pstack + PAGE_SIZE * 4); ++ #endif ++ + p->flags &= ~PF_KTHREAD; + if (args->kthread) + p->flags |= PF_KTHREAD; +@@ -2408,7 +2489,8 @@ __latent_entropy struct task_struct *copy_process( + /* + * Clear TID on mm_release()? + */ +- p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; ++ p->clear_child_tid = ++ (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL; + + ftrace_graph_init_task(p); + +@@ -2519,10 +2601,10 @@ __latent_entropy struct task_struct *copy_process( + #endif + #ifdef CONFIG_TRACE_IRQFLAGS + memset(&p->irqtrace, 0, sizeof(p->irqtrace)); +- p->irqtrace.hardirq_disable_ip = _THIS_IP_; +- p->irqtrace.softirq_enable_ip = _THIS_IP_; +- p->softirqs_enabled = 1; +- p->softirq_context = 0; ++ p->irqtrace.hardirq_disable_ip = _THIS_IP_; ++ p->irqtrace.softirq_enable_ip = _THIS_IP_; ++ p->softirqs_enabled = 1; ++ p->softirq_context = 0; + #endif + + p->pagefault_disabled = 0; +@@ -2535,8 +2617,8 @@ __latent_entropy struct task_struct *copy_process( + p->blocked_on = NULL; /* not blocked yet */ + #endif + #ifdef CONFIG_BCACHE +- p->sequential_io = 0; +- p->sequential_io_avg = 0; ++ p->sequential_io = 0; ++ p->sequential_io_avg = 0; + #endif + #ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(p->bpf_storage, NULL); +@@ -2623,7 +2705,7 @@ __latent_entropy struct task_struct *copy_process( + /* + * sigaltstack should be cleared when sharing the same VM + */ +- if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM) ++ if ((clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM) + sas_ss_reset(p); + + /* +@@ -2702,7 +2784,7 @@ __latent_entropy struct task_struct *copy_process( + write_lock_irq(&tasklist_lock); + + /* CLONE_PARENT re-uses the old parent */ +- if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) { ++ if (clone_flags & (CLONE_PARENT | CLONE_THREAD)) { + p->real_parent = current->real_parent; + p->parent_exec_id = current->parent_exec_id; + if (clone_flags & CLONE_THREAD) +@@ -2766,8 +2848,9 @@ __latent_entropy struct task_struct *copy_process( + * tasklist_lock with adding child to the process tree + * for propagate_has_child_subreaper optimization. + */ +- p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper || +- p->real_parent->signal->is_child_subreaper; ++ p->signal->has_child_subreaper = ++ p->real_parent->signal->has_child_subreaper || ++ p->real_parent->signal->is_child_subreaper; + list_add_tail(&p->sibling, &p->real_parent->children); + list_add_tail_rcu(&p->tasks, &init_task.tasks); + attach_pid(p, PIDTYPE_TGID); +@@ -2918,8 +3001,8 @@ struct task_struct * __init fork_idle(int cpu) + */ + struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node) + { +- unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD| +- CLONE_IO; ++ unsigned long flags = CLONE_FS | CLONE_FILES | CLONE_SIGHAND | ++ CLONE_THREAD | CLONE_IO; + struct kernel_clone_args args = { + .flags = ((lower_32_bits(flags) | CLONE_VM | + CLONE_UNTRACED) & ~CSIGNAL), +@@ -3083,8 +3166,8 @@ SYSCALL_DEFINE0(fork) + SYSCALL_DEFINE0(vfork) + { + struct kernel_clone_args args = { +- .flags = CLONE_VFORK | CLONE_VM, +- .exit_signal = SIGCHLD, ++ .flags = CLONE_VFORK | CLONE_VM, ++ .exit_signal = SIGCHLD, + }; + + return kernel_clone(&args); +@@ -3094,35 +3177,30 @@ SYSCALL_DEFINE0(vfork) + #ifdef __ARCH_WANT_SYS_CLONE + #ifdef CONFIG_CLONE_BACKWARDS + SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, +- int __user *, parent_tidptr, +- unsigned long, tls, +- int __user *, child_tidptr) ++ int __user *, parent_tidptr, unsigned long, tls, int __user *, ++ child_tidptr) + #elif defined(CONFIG_CLONE_BACKWARDS2) + SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags, +- int __user *, parent_tidptr, +- int __user *, child_tidptr, +- unsigned long, tls) +-#elif defined(CONFIG_CLONE_BACKWARDS3) +-SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, +- int, stack_size, +- int __user *, parent_tidptr, +- int __user *, child_tidptr, ++ int __user *, parent_tidptr, int __user *, child_tidptr, + unsigned long, tls) ++#elif defined(CONFIG_CLONE_BACKWARDS3) ++SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp, int, ++ stack_size, int __user *, parent_tidptr, int __user *, ++ child_tidptr, unsigned long, tls) + #else + SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp, +- int __user *, parent_tidptr, +- int __user *, child_tidptr, +- unsigned long, tls) ++ int __user *, parent_tidptr, int __user *, child_tidptr, ++ unsigned long, tls) + #endif + { + struct kernel_clone_args args = { +- .flags = (lower_32_bits(clone_flags) & ~CSIGNAL), +- .pidfd = parent_tidptr, +- .child_tid = child_tidptr, +- .parent_tid = parent_tidptr, +- .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL), +- .stack = newsp, +- .tls = tls, ++ .flags = (lower_32_bits(clone_flags) & ~CSIGNAL), ++ .pidfd = parent_tidptr, ++ .child_tid = child_tidptr, ++ .parent_tid = parent_tidptr, ++ .exit_signal = (lower_32_bits(clone_flags) & CSIGNAL), ++ .stack = newsp, ++ .tls = tls, + }; + + return kernel_clone(&args); +@@ -3178,21 +3256,21 @@ noinline static int copy_clone_args_from_user(struct kernel_clone_args *kargs, + return -EINVAL; + + *kargs = (struct kernel_clone_args){ +- .flags = args.flags, +- .pidfd = u64_to_user_ptr(args.pidfd), +- .child_tid = u64_to_user_ptr(args.child_tid), +- .parent_tid = u64_to_user_ptr(args.parent_tid), +- .exit_signal = args.exit_signal, +- .stack = args.stack, +- .stack_size = args.stack_size, +- .tls = args.tls, +- .set_tid_size = args.set_tid_size, +- .cgroup = args.cgroup, ++ .flags = args.flags, ++ .pidfd = u64_to_user_ptr(args.pidfd), ++ .child_tid = u64_to_user_ptr(args.child_tid), ++ .parent_tid = u64_to_user_ptr(args.parent_tid), ++ .exit_signal = args.exit_signal, ++ .stack = args.stack, ++ .stack_size = args.stack_size, ++ .tls = args.tls, ++ .set_tid_size = args.set_tid_size, ++ .cgroup = args.cgroup, + }; + + if (args.set_tid && +- copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid), +- (kargs->set_tid_size * sizeof(pid_t)))) ++ copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid), ++ (kargs->set_tid_size * sizeof(pid_t)))) + return -EFAULT; + + kargs->set_tid = kset_tid; +@@ -3287,7 +3365,8 @@ SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size) + } + #endif + +-void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data) ++void walk_process_tree(struct task_struct *top, proc_visitor visitor, ++ void *data) + { + struct task_struct *leader, *parent, *child; + int res; +@@ -3295,8 +3374,8 @@ void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data + read_lock(&tasklist_lock); + leader = top = top->group_leader; + down: +- for_each_thread(leader, parent) { +- list_for_each_entry(child, &parent->children, sibling) { ++ for_each_thread (leader, parent) { ++ list_for_each_entry (child, &parent->children, sibling) { + res = visitor(child, data); + if (res) { + if (res < 0) +@@ -3304,8 +3383,7 @@ void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data + leader = child; + goto down; + } +-up: +- ; ++ up:; + } + } + +@@ -3382,11 +3460,11 @@ void __init proc_caches_init(void) + */ + static int check_unshare_flags(unsigned long unshare_flags) + { +- if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| +- CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| +- CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET| +- CLONE_NEWUSER|CLONE_NEWPID|CLONE_NEWCGROUP| +- CLONE_NEWTIME)) ++ if (unshare_flags & ++ ~(CLONE_THREAD | CLONE_FS | CLONE_NEWNS | CLONE_SIGHAND | CLONE_VM | ++ CLONE_FILES | CLONE_SYSVSEM | CLONE_NEWUTS | CLONE_NEWIPC | ++ CLONE_NEWNET | CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWCGROUP | ++ CLONE_NEWTIME)) + return -EINVAL; + /* + * Not implemented, but pretend it works if there is nothing +@@ -3497,7 +3575,7 @@ int ksys_unshare(unsigned long unshare_flags) + * to a new ipc namespace, the semaphore arrays from the old + * namespace are unreachable. + */ +- if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM)) ++ if (unshare_flags & (CLONE_NEWIPC | CLONE_SYSVSEM)) + do_sysvsem = 1; + err = unshare_fs(unshare_flags, &new_fs); + if (err) +@@ -3508,8 +3586,8 @@ int ksys_unshare(unsigned long unshare_flags) + err = unshare_userns(unshare_flags, &new_cred); + if (err) + goto bad_unshare_cleanup_fd; +- err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, +- new_cred, new_fs); ++ err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy, new_cred, ++ new_fs); + if (err) + goto bad_unshare_cleanup_cred; + +@@ -3606,8 +3684,8 @@ int unshare_files(void) + return 0; + } + +-int sysctl_max_threads(struct ctl_table *table, int write, +- void *buffer, size_t *lenp, loff_t *ppos) ++int sysctl_max_threads(struct ctl_table *table, int write, void *buffer, ++ size_t *lenp, loff_t *ppos) + { + struct ctl_table t; + int ret; +diff --git a/kernel/groups.c b/kernel/groups.c +index 9b43da22647d..8045812e8a3c 100644 +--- a/kernel/groups.c ++++ b/kernel/groups.c +@@ -11,6 +11,9 @@ + #include <linux/user_namespace.h> + #include <linux/vmalloc.h> + #include <linux/uaccess.h> ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif + + struct group_info *groups_alloc(int gidsetsize) + { +@@ -119,7 +122,11 @@ void set_groups(struct cred *new, struct group_info *group_info) + { + put_group_info(new->group_info); + get_group_info(group_info); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_group_info(new,group_info); ++ #else + new->group_info = group_info; ++ #endif + } + + EXPORT_SYMBOL(set_groups); +diff --git a/kernel/kthread.c b/kernel/kthread.c +index 1eea53050bab..317eac6eb2f2 100644 +--- a/kernel/kthread.c ++++ b/kernel/kthread.c +@@ -30,6 +30,10 @@ + #include <linux/sched/isolation.h> + #include <trace/events/sched.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#endif ++ + + static DEFINE_SPINLOCK(kthread_create_lock); + static LIST_HEAD(kthread_create_list); +@@ -1429,6 +1433,10 @@ void kthread_use_mm(struct mm_struct *mm) + tsk->active_mm = mm; + tsk->mm = mm; + membarrier_update_current_mm(mm); ++ #ifdef CONFIG_IEE ++ iee_set_token_mm(tsk, mm); ++ iee_set_token_pgd(tsk, mm->pgd); ++ #endif + switch_mm_irqs_off(active_mm, mm, tsk); + local_irq_enable(); + task_unlock(tsk); +@@ -1473,7 +1481,12 @@ void kthread_unuse_mm(struct mm_struct *mm) + local_irq_disable(); + tsk->mm = NULL; + membarrier_update_current_mm(NULL); ++ #ifdef CONFIG_IEE ++ iee_set_token_mm(tsk, mm); ++ iee_set_token_pgd(tsk, NULL); ++ #endif + mmgrab_lazy_tlb(mm); ++ + /* active_mm is still 'mm' */ + enter_lazy_tlb(mm, tsk); + local_irq_enable(); +diff --git a/kernel/smpboot.c b/kernel/smpboot.c +index f47d8f375946..60c7d365c0e1 100644 +--- a/kernel/smpboot.c ++++ b/kernel/smpboot.c +@@ -16,6 +16,10 @@ + #include <linux/kthread.h> + #include <linux/smpboot.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#endif ++ + #include "smpboot.h" + + #ifdef CONFIG_SMP +@@ -57,6 +61,11 @@ static __always_inline void idle_init(unsigned int cpu) + pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); + else + per_cpu(idle_threads, cpu) = tsk; ++ #ifdef CONFIG_IEE ++ // Set the secondary __entry_task. ++ *(struct task_struct **)SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu]) = tsk; ++ iee_set_logical_mem_ro((unsigned long)SHIFT_PERCPU_PTR(__entry_task,__per_cpu_offset[cpu])); ++ #endif + } + } + +diff --git a/kernel/sys.c b/kernel/sys.c +index 44b575990333..fbc47f83af50 100644 +--- a/kernel/sys.c ++++ b/kernel/sys.c +@@ -75,6 +75,10 @@ + #include <asm/io.h> + #include <asm/unistd.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + #include "uid16.h" + + #ifndef SET_UNALIGN_CTL +@@ -395,7 +399,11 @@ long __sys_setregid(gid_t rgid, gid_t egid) + if (gid_eq(old->gid, krgid) || + gid_eq(old->egid, krgid) || + ns_capable_setid(old->user_ns, CAP_SETGID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_gid(new,krgid); ++ #else + new->gid = krgid; ++ #endif + else + goto error; + } +@@ -404,15 +412,27 @@ long __sys_setregid(gid_t rgid, gid_t egid) + gid_eq(old->egid, kegid) || + gid_eq(old->sgid, kegid) || + ns_capable_setid(old->user_ns, CAP_SETGID)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_egid(new,kegid); ++ #else + new->egid = kegid; ++ #endif + else + goto error; + } + + if (rgid != (gid_t) -1 || + (egid != (gid_t) -1 && !gid_eq(kegid, old->gid))) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_sgid(new,new->egid); ++ #else + new->sgid = new->egid; ++ #endif ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsgid(new,new->egid); ++ #else + new->fsgid = new->egid; ++ #endif + + retval = security_task_fix_setgid(new, old, LSM_SETID_RE); + if (retval < 0) +@@ -454,9 +474,25 @@ long __sys_setgid(gid_t gid) + + retval = -EPERM; + if (ns_capable_setid(old->user_ns, CAP_SETGID)) ++ #ifdef CONFIG_CREDP ++ { ++ iee_set_cred_fsgid(new,kgid); ++ iee_set_cred_sgid(new,kgid); ++ iee_set_cred_egid(new,kgid); ++ iee_set_cred_gid(new,kgid); ++ } ++ #else + new->gid = new->egid = new->sgid = new->fsgid = kgid; ++ #endif + else if (gid_eq(kgid, old->gid) || gid_eq(kgid, old->sgid)) ++ #ifdef CONFIG_CREDP ++ { ++ iee_set_cred_fsgid(new,kgid); ++ iee_set_cred_egid(new,kgid); ++ } ++ #else + new->egid = new->fsgid = kgid; ++ #endif + else + goto error; + +@@ -488,7 +524,11 @@ static int set_user(struct cred *new) + return -EAGAIN; + + free_uid(new->user); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_user(new,new_user); ++ #else + new->user = new_user; ++ #endif + return 0; + } + +@@ -549,7 +589,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid) + + retval = -EPERM; + if (ruid != (uid_t) -1) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_uid(new,kruid); ++ #else + new->uid = kruid; ++ #endif + if (!uid_eq(old->uid, kruid) && + !uid_eq(old->euid, kruid) && + !ns_capable_setid(old->user_ns, CAP_SETUID)) +@@ -557,7 +601,11 @@ long __sys_setreuid(uid_t ruid, uid_t euid) + } + + if (euid != (uid_t) -1) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_euid(new,keuid); ++ #else + new->euid = keuid; ++ #endif + if (!uid_eq(old->uid, keuid) && + !uid_eq(old->euid, keuid) && + !uid_eq(old->suid, keuid) && +@@ -572,8 +620,16 @@ long __sys_setreuid(uid_t ruid, uid_t euid) + } + if (ruid != (uid_t) -1 || + (euid != (uid_t) -1 && !uid_eq(keuid, old->uid))) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_suid(new,new->euid); ++ #else + new->suid = new->euid; ++ #endif ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,new->euid); ++ #else + new->fsuid = new->euid; ++ #endif + + retval = security_task_fix_setuid(new, old, LSM_SETID_RE); + if (retval < 0) +@@ -626,7 +682,12 @@ long __sys_setuid(uid_t uid) + + retval = -EPERM; + if (ns_capable_setid(old->user_ns, CAP_SETUID)) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_uid(new,kuid); ++ iee_set_cred_suid(new,kuid); ++ #else + new->suid = new->uid = kuid; ++ #endif + if (!uid_eq(kuid, old->uid)) { + retval = set_user(new); + if (retval < 0) +@@ -636,7 +697,12 @@ long __sys_setuid(uid_t uid) + goto error; + } + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_euid(new,kuid); ++ iee_set_cred_fsuid(new,kuid); ++ #else + new->fsuid = new->euid = kuid; ++ #endif + + retval = security_task_fix_setuid(new, old, LSM_SETID_ID); + if (retval < 0) +@@ -710,7 +776,11 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) + return -ENOMEM; + + if (ruid != (uid_t) -1) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_uid(new,kruid); ++ #else + new->uid = kruid; ++ #endif + if (!uid_eq(kruid, old->uid)) { + retval = set_user(new); + if (retval < 0) +@@ -718,10 +788,22 @@ long __sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) + } + } + if (euid != (uid_t) -1) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_euid(new,keuid); ++ #else + new->euid = keuid; ++ #endif + if (suid != (uid_t) -1) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_suid(new,ksuid); ++ #else + new->suid = ksuid; ++ #endif ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,new->euid); ++ #else + new->fsuid = new->euid; ++ #endif + + retval = security_task_fix_setuid(new, old, LSM_SETID_RES); + if (retval < 0) +@@ -810,12 +892,29 @@ long __sys_setresgid(gid_t rgid, gid_t egid, gid_t sgid) + return -ENOMEM; + + if (rgid != (gid_t) -1) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_gid(new,krgid); ++ #else + new->gid = krgid; ++ #endif + if (egid != (gid_t) -1) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_egid(new,kegid); ++ #else + new->egid = kegid; ++ #endif + if (sgid != (gid_t) -1) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_sgid(new,ksgid); ++ #else + new->sgid = ksgid; ++ #endif ++ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsgid(new,new->egid); ++ #else + new->fsgid = new->egid; ++ #endif + + retval = security_task_fix_setgid(new, old, LSM_SETID_RES); + if (retval < 0) +@@ -882,7 +981,11 @@ long __sys_setfsuid(uid_t uid) + uid_eq(kuid, old->suid) || uid_eq(kuid, old->fsuid) || + ns_capable_setid(old->user_ns, CAP_SETUID)) { + if (!uid_eq(kuid, old->fsuid)) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,kuid); ++ #else + new->fsuid = kuid; ++ #endif + if (security_task_fix_setuid(new, old, LSM_SETID_FS) == 0) + goto change_okay; + } +@@ -926,7 +1029,11 @@ long __sys_setfsgid(gid_t gid) + gid_eq(kgid, old->sgid) || gid_eq(kgid, old->fsgid) || + ns_capable_setid(old->user_ns, CAP_SETGID)) { + if (!gid_eq(kgid, old->fsgid)) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsgid(new,kgid); ++ #else + new->fsgid = kgid; ++ #endif + if (security_task_fix_setgid(new,old,LSM_SETID_FS) == 0) + goto change_okay; + } +diff --git a/kernel/umh.c b/kernel/umh.c +index 1b13c5d34624..32f5c88e10bf 100644 +--- a/kernel/umh.c ++++ b/kernel/umh.c +@@ -32,6 +32,10 @@ + + #include <trace/events/module.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + static kernel_cap_t usermodehelper_bset = CAP_FULL_SET; + static kernel_cap_t usermodehelper_inheritable = CAP_FULL_SET; + static DEFINE_SPINLOCK(umh_sysctl_lock); +@@ -91,9 +95,15 @@ static int call_usermodehelper_exec_async(void *data) + goto out; + + spin_lock(&umh_sysctl_lock); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_bset(new,cap_intersect(usermodehelper_bset, new->cap_bset)); ++ iee_set_cred_cap_inheritable(new,cap_intersect(usermodehelper_inheritable, ++ new->cap_inheritable)); ++ #else + new->cap_bset = cap_intersect(usermodehelper_bset, new->cap_bset); + new->cap_inheritable = cap_intersect(usermodehelper_inheritable, + new->cap_inheritable); ++ #endif + spin_unlock(&umh_sysctl_lock); + + if (sub_info->init) { +diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c +index 1d8e47bed3f1..9f1921025539 100644 +--- a/kernel/user_namespace.c ++++ b/kernel/user_namespace.c +@@ -22,6 +22,10 @@ + #include <linux/bsearch.h> + #include <linux/sort.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + static struct kmem_cache *user_ns_cachep __read_mostly; + static DEFINE_MUTEX(userns_state_mutex); + +@@ -45,6 +49,19 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) + /* Start with the same capabilities as init but useless for doing + * anything as the capabilities are bound to the new user namespace. + */ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_securebits(cred,SECUREBITS_DEFAULT); ++ iee_set_cred_cap_inheritable(cred,CAP_EMPTY_SET); ++ iee_set_cred_cap_permitted(cred,CAP_FULL_SET); ++ iee_set_cred_cap_effective(cred,CAP_FULL_SET); ++ iee_set_cred_cap_ambient(cred,CAP_EMPTY_SET); ++ iee_set_cred_cap_bset(cred,CAP_FULL_SET); ++#ifdef CONFIG_KEYS ++ key_put(cred->request_key_auth); ++ iee_set_cred_request_key_auth(cred,NULL); ++#endif ++ iee_set_cred_user_ns(cred,user_ns); ++ #else + cred->securebits = SECUREBITS_DEFAULT; + cred->cap_inheritable = CAP_EMPTY_SET; + cred->cap_permitted = CAP_FULL_SET; +@@ -57,6 +74,7 @@ static void set_cred_user_ns(struct cred *cred, struct user_namespace *user_ns) + #endif + /* tgcred will be cleared in our caller bc CLONE_THREAD won't be set */ + cred->user_ns = user_ns; ++ #endif + } + + static unsigned long enforced_nproc_rlimit(void) +diff --git a/mm/Kconfig b/mm/Kconfig +index 45d4139c959c..eb9d41768c15 100644 +--- a/mm/Kconfig ++++ b/mm/Kconfig +@@ -530,6 +530,18 @@ config NUMA_KEEP_MEMINFO + config MEMORY_ISOLATION + bool + ++# Config for kernel module isolation ++config KOI ++ depends on ARM64 ++ depends on ARM64_VA_BITS_48 ++ depends on ARM64_4K_PAGES ++ def_bool n ++ ++# Configs for pgtable isolation ++config PTP ++ depends on IEE ++ def_bool y ++ + # IORESOURCE_SYSTEM_RAM regions in the kernel resource tree that are marked + # IORESOURCE_EXCLUSIVE cannot be mapped to user space, for example, via + # /dev/mem. +diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c +index d25d99cb5f2b..2ea51f559d4e 100644 +--- a/mm/damon/ops-common.c ++++ b/mm/damon/ops-common.c +@@ -44,6 +44,7 @@ void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr + if (!folio) + return; + ++ + if (ptep_clear_young_notify(vma, addr, pte)) + folio_set_young(folio); + +diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c +index 13f0d1192707..60dc95c5b286 100644 +--- a/mm/debug_vm_pgtable.c ++++ b/mm/debug_vm_pgtable.c +@@ -452,7 +452,11 @@ static void __init pmd_huge_tests(struct pgtable_debug_args *args) + * X86 defined pmd_set_huge() verifies that the given + * PMD is not a populated non-leaf entry. + */ ++ #ifdef CONFIG_PTP ++ set_pmd(args->pmdp, __pmd(0)); ++ #else + WRITE_ONCE(*args->pmdp, __pmd(0)); ++ #endif + WARN_ON(!pmd_set_huge(args->pmdp, __pfn_to_phys(args->fixed_pmd_pfn), args->page_prot)); + WARN_ON(!pmd_clear_huge(args->pmdp)); + pmd = READ_ONCE(*args->pmdp); +@@ -472,7 +476,11 @@ static void __init pud_huge_tests(struct pgtable_debug_args *args) + * X86 defined pud_set_huge() verifies that the given + * PUD is not a populated non-leaf entry. + */ ++ #ifdef CONFIG_PTP ++ set_pud(args->pudp, __pud(0)); ++ #else + WRITE_ONCE(*args->pudp, __pud(0)); ++ #endif + WARN_ON(!pud_set_huge(args->pudp, __pfn_to_phys(args->fixed_pud_pfn), args->page_prot)); + WARN_ON(!pud_clear_huge(args->pudp)); + pud = READ_ONCE(*args->pudp); +@@ -511,7 +519,11 @@ static void __init pud_clear_tests(struct pgtable_debug_args *args) + + pr_debug("Validating PUD clear\n"); + pud = __pud(pud_val(pud) | RANDOM_ORVALUE); ++ #ifdef CONFIG_PTP ++ set_pud(args->pudp, pud); ++ #else + WRITE_ONCE(*args->pudp, pud); ++ #endif + pud_clear(args->pudp); + pud = READ_ONCE(*args->pudp); + WARN_ON(!pud_none(pud)); +@@ -548,7 +560,11 @@ static void __init p4d_clear_tests(struct pgtable_debug_args *args) + + pr_debug("Validating P4D clear\n"); + p4d = __p4d(p4d_val(p4d) | RANDOM_ORVALUE); ++ #ifdef CONFIG_PTP ++ set_p4d(args->p4dp, p4d); ++ #else + WRITE_ONCE(*args->p4dp, p4d); ++ #endif + p4d_clear(args->p4dp); + p4d = READ_ONCE(*args->p4dp); + WARN_ON(!p4d_none(p4d)); +@@ -582,7 +598,11 @@ static void __init pgd_clear_tests(struct pgtable_debug_args *args) + + pr_debug("Validating PGD clear\n"); + pgd = __pgd(pgd_val(pgd) | RANDOM_ORVALUE); ++ #ifdef CONFIG_PTP ++ set_pgd(args->pgdp, pgd); ++ #else + WRITE_ONCE(*args->pgdp, pgd); ++ #endif + pgd_clear(args->pgdp); + pgd = READ_ONCE(*args->pgdp); + WARN_ON(!pgd_none(pgd)); +@@ -650,7 +670,11 @@ static void __init pmd_clear_tests(struct pgtable_debug_args *args) + + pr_debug("Validating PMD clear\n"); + pmd = __pmd(pmd_val(pmd) | RANDOM_ORVALUE); ++ #ifdef CONFIG_PTP ++ set_pmd(args->pmdp, pmd); ++ #else + WRITE_ONCE(*args->pmdp, pmd); ++ #endif + pmd_clear(args->pmdp); + pmd = READ_ONCE(*args->pmdp); + WARN_ON(!pmd_none(pmd)); +diff --git a/mm/early_ioremap.c b/mm/early_ioremap.c +index ce06b2884789..a039c7a50ec5 100644 +--- a/mm/early_ioremap.c ++++ b/mm/early_ioremap.c +@@ -147,7 +147,11 @@ __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot) + if (after_paging_init) + __late_set_fixmap(idx, phys_addr, prot); + else ++ #ifdef CONFIG_PTP ++ __iee_set_fixmap_pre_init(idx, phys_addr, prot); ++ #else + __early_set_fixmap(idx, phys_addr, prot); ++ #endif + phys_addr += PAGE_SIZE; + --idx; + --nrpages; +@@ -199,13 +203,66 @@ void __init early_iounmap(void __iomem *addr, unsigned long size) + if (after_paging_init) + __late_clear_fixmap(idx); + else ++ #ifdef CONFIG_PTP ++ __iee_set_fixmap_pre_init(idx, 0, FIXMAP_PAGE_CLEAR); ++ #else + __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); ++ #endif + --idx; + --nrpages; + } + prev_map[slot] = NULL; + } + ++#ifdef CONFIG_PTP ++void __init early_iounmap_after_init(void __iomem *addr, unsigned long size) ++{ ++ unsigned long virt_addr; ++ unsigned long offset; ++ unsigned int nrpages; ++ enum fixed_addresses idx; ++ int i, slot; ++ ++ slot = -1; ++ for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { ++ if (prev_map[i] == addr) { ++ slot = i; ++ break; ++ } ++ } ++ ++ if (WARN(slot < 0, "early_iounmap(%p, %08lx) not found slot\n", ++ addr, size)) ++ return; ++ ++ if (WARN(prev_size[slot] != size, ++ "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n", ++ addr, size, slot, prev_size[slot])) ++ return; ++ ++ WARN(early_ioremap_debug, "early_iounmap(%p, %08lx) [%d]\n", ++ addr, size, slot); ++ ++ virt_addr = (unsigned long)addr; ++ if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN))) ++ return; ++ ++ offset = offset_in_page(virt_addr); ++ nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT; ++ ++ idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; ++ while (nrpages > 0) { ++ if (after_paging_init) ++ __late_clear_fixmap(idx); ++ else ++ __early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR); ++ --idx; ++ --nrpages; ++ } ++ prev_map[slot] = NULL; ++} ++#endif ++ + /* Remap an IO device */ + void __init __iomem * + early_ioremap(resource_size_t phys_addr, unsigned long size) +diff --git a/mm/huge_memory.c b/mm/huge_memory.c +index 763bb25e4f99..80bb2c0abeda 100644 +--- a/mm/huge_memory.c ++++ b/mm/huge_memory.c +@@ -39,6 +39,10 @@ + #include <linux/memory-tiers.h> + #include <linux/compat.h> + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + #include <asm/tlb.h> + #include <asm/pgalloc.h> + #include "internal.h" +@@ -2489,6 +2493,10 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, + unsigned long addr; + pte_t *pte; + int i; ++ #ifdef CONFIG_PTP ++ pte_t *ptep; ++ unsigned long iee_addr; ++ #endif + + /* + * Leave pmd empty until pte is filled note that it is fine to delay +@@ -2501,7 +2509,14 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, + old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); + + pgtable = pgtable_trans_huge_withdraw(mm, pmd); +- pmd_populate(mm, &_pmd, pgtable); ++ #ifdef CONFIG_PTP ++ ptep = (pte_t *)page_address(pgtable); ++ iee_addr = __phys_to_iee(__pa(ptep)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)ptep); ++ #endif ++ //pmd_populate(mm, &_pmd, pgtable); ++ _pmd = __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) | PMD_TYPE_TABLE); + + pte = pte_offset_map(&_pmd, haddr); + VM_BUG_ON(!pte); +@@ -2534,6 +2549,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr; + pte_t *pte; + int i; ++ #ifdef CONFIG_PTP ++ pte_t *ptep; ++ unsigned long iee_addr; ++ #endif + + VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); + VM_BUG_ON_VMA(vma->vm_start > haddr, vma); +@@ -2671,7 +2690,14 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, + * This's critical for some architectures (Power). + */ + pgtable = pgtable_trans_huge_withdraw(mm, pmd); +- pmd_populate(mm, &_pmd, pgtable); ++ #ifdef CONFIG_PTP ++ ptep = (pte_t *)page_to_virt(pgtable); ++ iee_addr = __phys_to_iee(__pa(ptep)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)ptep); ++ #endif ++ //pmd_populate(mm, &_pmd, pgtable); ++ _pmd = __pmd(__phys_to_pmd_val(page_to_phys(pgtable)) | PMD_TYPE_TABLE); + + pte = pte_offset_map(&_pmd, haddr); + VM_BUG_ON(!pte); +diff --git a/mm/init-mm.c b/mm/init-mm.c +index 24c809379274..07d060fca6f0 100644 +--- a/mm/init-mm.c ++++ b/mm/init-mm.c +@@ -55,3 +55,20 @@ void setup_initial_init_mm(void *start_code, void *end_code, + init_mm.end_data = (unsigned long)end_data; + init_mm.brk = (unsigned long)brk; + } ++ ++#ifdef CONFIG_KOI ++/* ++ * This is used to init ko_mm when creating pgtable for a ko to be isolated ++ * the ko_mm belongs to a specific ko, pgdp is allocated by koi_pgd_alloc ++ */ ++void init_ko_mm(struct mm_struct *ko_mm, pgd_t *pgdp) { ++ ko_mm->mm_rb = RB_ROOT; ++ ko_mm->pgd = pgdp; ++ ko_mm->mm_users = (atomic_t)ATOMIC_INIT(2); ++ ko_mm->mm_count = (atomic_t)ATOMIC_INIT(1); ++ ko_mm->mmap_lock = (struct rw_semaphore)__RWSEM_INITIALIZER(ko_mm->mmap_lock); ++ ko_mm->page_table_lock = __SPIN_LOCK_UNLOCKED(ko_mm.page_table_lock); ++ ko_mm->arg_lock = __SPIN_LOCK_UNLOCKED(ko_mm->arg_lock); ++ ko_mm->mmlist = (struct list_head)LIST_HEAD_INIT(ko_mm->mmlist); ++} ++#endif +diff --git a/mm/memory.c b/mm/memory.c +index 4ef917a182f9..28da89a19e30 100644 +--- a/mm/memory.c ++++ b/mm/memory.c +@@ -80,6 +80,10 @@ + #include <linux/userswap.h> + #include <linux/dynamic_pool.h> + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + #include <trace/events/kmem.h> + + #include <asm/io.h> +@@ -5872,6 +5876,11 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) + + spin_lock(&mm->page_table_lock); + if (!p4d_present(*p4d)) { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(new)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)new); ++ #endif + mm_inc_nr_puds(mm); + smp_wmb(); /* See comment in pmd_install() */ + p4d_populate(mm, p4d, new); +@@ -5896,6 +5905,11 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) + + ptl = pud_lock(mm, pud); + if (!pud_present(*pud)) { ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr = __phys_to_iee(__pa(new)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)new); ++ #endif + mm_inc_nr_pmds(mm); + smp_wmb(); /* See comment in pmd_install() */ + pud_populate(mm, pud, new); +diff --git a/mm/slub.c b/mm/slub.c +index ee3e32cdb7fd..20a45a7feed5 100644 +--- a/mm/slub.c ++++ b/mm/slub.c +@@ -42,6 +42,11 @@ + #include <kunit/test-bug.h> + #include <linux/sort.h> + ++#ifdef CONFIG_IEE ++#include <linux/iee-func.h> ++#include <asm/iee-access.h> ++#endif ++ + #include <linux/debugfs.h> + #include <trace/events/kmem.h> + +@@ -317,6 +322,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) + /* + * Tracking user of a slab. + */ ++#ifndef CONFIG_IEE + #define TRACK_ADDRS_COUNT 16 + struct track { + unsigned long addr; /* Called from address */ +@@ -329,6 +335,7 @@ struct track { + }; + + enum track_item { TRACK_ALLOC, TRACK_FREE }; ++#endif + + #ifdef SLAB_SUPPORTS_SYSFS + static int sysfs_slab_add(struct kmem_cache *); +@@ -379,7 +386,9 @@ static struct workqueue_struct *flushwq; + * freeptr_t represents a SLUB freelist pointer, which might be encoded + * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled. + */ ++#ifndef CONFIG_IEE + typedef struct { unsigned long v; } freeptr_t; ++#endif + + /* + * Returns freelist pointer (ptr). With hardening, this is obfuscated +@@ -464,7 +473,14 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) + #endif + + freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr); ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ iee_set_freeptr((freeptr_t *)freeptr_addr, freelist_ptr_encode(s, fp, freeptr_addr)); ++ else ++ *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); ++ #else + *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr); ++ #endif + } + + /* Loop over all objects in a slab */ +@@ -809,7 +825,34 @@ static void set_track_update(struct kmem_cache *s, void *object, + depot_stack_handle_t handle) + { + struct track *p = get_track(s, object, alloc); ++#ifdef CONFIG_IEE ++ struct track tmp; ++#endif + ++#ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ { ++ tmp = *p; ++ #ifdef CONFIG_STACKDEPOT ++ tmp.handle = handle; ++ #endif ++ tmp.addr = addr; ++ tmp.cpu = smp_processor_id(); ++ tmp.pid = current->pid; ++ tmp.when = jiffies; ++ iee_set_track(p,&tmp); ++ } ++ else ++ { ++ #ifdef CONFIG_STACKDEPOT ++ p->handle = handle; ++ #endif ++ p->addr = addr; ++ p->cpu = smp_processor_id(); ++ p->pid = current->pid; ++ p->when = jiffies; ++ } ++#else + #ifdef CONFIG_STACKDEPOT + p->handle = handle; + #endif +@@ -817,6 +860,7 @@ static void set_track_update(struct kmem_cache *s, void *object, + p->cpu = smp_processor_id(); + p->pid = current->pid; + p->when = jiffies; ++#endif + } + + static __always_inline void set_track(struct kmem_cache *s, void *object, +@@ -835,7 +879,14 @@ static void init_tracking(struct kmem_cache *s, void *object) + return; + + p = get_track(s, object, TRACK_ALLOC); ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ iee_memset(p, 0, 2*sizeof(struct track)); ++ else ++ memset(p, 0, 2*sizeof(struct track)); ++ #else + memset(p, 0, 2*sizeof(struct track)); ++ #endif + } + + static void print_track(const char *s, struct track *t, unsigned long pr_time) +@@ -1045,7 +1096,14 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) + unsigned int poison_size = s->object_size; + + if (s->flags & SLAB_RED_ZONE) { ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ iee_memset(p - s->red_left_pad, val, s->red_left_pad); ++ else ++ memset(p - s->red_left_pad, val, s->red_left_pad); ++ #else + memset(p - s->red_left_pad, val, s->red_left_pad); ++ #endif + + if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) { + /* +@@ -1058,12 +1116,34 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) + } + + if (s->flags & __OBJECT_POISON) { ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ { ++ iee_memset(p, POISON_FREE, poison_size - 1); ++ iee_memset(&p[poison_size - 1], POISON_END, 1); ++ } ++ else ++ { ++ memset(p, POISON_FREE, poison_size - 1); ++ p[poison_size - 1] = POISON_END; ++ } ++ #else + memset(p, POISON_FREE, poison_size - 1); + p[poison_size - 1] = POISON_END; ++ #endif + } + +- if (s->flags & SLAB_RED_ZONE) ++ if (s->flags & SLAB_RED_ZONE) { ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ iee_memset(p + poison_size, val, s->inuse - poison_size); ++ else ++ memset(p + poison_size, val, s->inuse - poison_size); ++ #else + memset(p + poison_size, val, s->inuse - poison_size); ++ #endif ++ ++ } + } + + static void restore_bytes(struct kmem_cache *s, char *message, u8 data, +@@ -1433,7 +1513,14 @@ void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) + return; + + metadata_access_enable(); ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ iee_memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); ++ else ++ memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); ++ #else + memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab)); ++ #endif + metadata_access_disable(); + } + +@@ -2015,6 +2102,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + void *start, *p, *next; + int idx; + bool shuffle; ++ #ifdef CONFIG_IEE ++ unsigned int order; ++ #endif + + flags &= gfp_allowed_mask; + +@@ -2029,6 +2119,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~__GFP_RECLAIM; + + slab = alloc_slab_page(alloc_gfp, node, oo); ++ #ifdef CONFIG_IEE ++ order = oo_order(oo); ++ #endif + if (unlikely(!slab)) { + oo = s->min; + alloc_gfp = flags; +@@ -2037,6 +2130,9 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + * Try a lower order alloc if possible + */ + slab = alloc_slab_page(alloc_gfp, node, oo); ++ #ifdef CONFIG_IEE ++ order = oo_order(oo); ++ #endif + if (unlikely(!slab)) + return NULL; + stat(s, ORDER_FALLBACK); +@@ -2046,6 +2142,45 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) + slab->inuse = 0; + slab->frozen = 0; + ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ { ++ int i; ++ for(i = 0; i < (0x1 << order); i++) ++ { ++ unsigned long iee_addr = __phys_to_iee(page_to_phys(folio_page(slab_folio(slab), i))); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)page_address(folio_page(slab_folio(slab), i))); ++ } ++ } ++ ++ // If the page belongs to a task_struct, alloc token for it and set iee&lm va. ++ if(strcmp(s->name, "task_struct") == 0) ++ { ++ int i; ++ for(i = 0; i < (0x1 << order); i++) ++ { ++ void *token_addr = (void *)__phys_to_iee(page_to_phys(folio_page(slab_folio(slab), i))); ++ // Get lm va of the page. ++ void *alloc_token = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); ++ iee_set_token_page_valid(token_addr, alloc_token); ++ set_iee_page_valid(__phys_to_iee(__pa(alloc_token))); ++ iee_set_logical_mem_ro((unsigned long)alloc_token); ++ } ++ } ++ #else ++ #ifdef CONFIG_KOI ++ if (strcmp(s->name, "task_struct") == 0) { ++ int i; ++ for (i = 0; i < (0x1 << order); i++) { ++ void *token_addr = __phys_to_virt(page_to_phys(page + i)) + koi_offset; ++ void *alloc_token = __get_free_page(GFP_KERNEL | __GFP_ZERO); ++ koi_add_page_mapping(token_addr, alloc_token); ++ } ++ } ++ #endif ++ #endif ++ + account_slab(slab, oo_order(oo), s, flags); + + slab->slab_cache = s; +@@ -2098,6 +2233,67 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab) + __folio_clear_slab(folio); + mm_account_reclaimed_pages(pages); + unaccount_slab(slab, order, s); ++ ++ #ifdef CONFIG_IEE ++ if(IS_ENABLED(CONFIG_CREDP) && strcmp(s->name, "cred_jar") == 0) ++ { ++ int i; ++ for(i = 0; i < (0x1 << order); i++) ++ { ++ unsigned long iee_addr = __phys_to_iee(page_to_phys(folio_page(folio, i))); ++ set_iee_page_invalid(iee_addr); ++ iee_set_logical_mem_rw((unsigned long)page_address(folio_page(folio, i))); ++ } ++ } ++ // If the page containing this token is empty, free it and restore iee&lm va. ++ if(strcmp(s->name, "task_struct") == 0) ++ { ++ int i; ++ for(i = 0; i < (0x1 << order); i++) ++ { ++ void *token_addr = (void *)__phys_to_iee(page_to_phys(folio_page(folio, i))); ++ unsigned long flags; ++ unsigned long res; ++ local_irq_save(flags); ++ asm volatile("at s1e1r, %0"::"r"(token_addr)); ++ isb(); ++ res = read_sysreg(par_el1); ++ local_irq_restore(flags); ++ if(!(res & 0x1)) ++ { ++ // Get lm va of the page. ++ void *token_page = __va(res & PTE_ADDR_MASK); ++ iee_set_token_page_invalid(token_addr); ++ set_iee_page_invalid(__phys_to_iee(__pa(token_page))); ++ iee_set_logical_mem_rw((unsigned long)token_page); ++ free_page((unsigned long)token_page); ++ } ++ } ++ } ++ #else ++ #ifdef CONFIG_KOI ++ if(strcmp(s->name, "task_struct") == 0) ++ { ++ int i; ++ for(i = 0; i < (0x1 << order); i++) ++ { ++ void *token_addr = __phys_to_virt(page_to_phys(page + i)) + koi_offset; ++ unsigned long flags; ++ local_irq_save(flags); ++ asm volatile("at s1e1r, %0"::"r"(token_addr)); ++ isb(); ++ unsigned long res = read_sysreg(par_el1); ++ local_irq_restore(flags); ++ if(!(res & 0x1)) ++ { ++ koi_remove_page_mapping(token_addr); ++ free_page(__va(res & PTE_ADDR_MASK)); ++ } ++ } ++ } ++ #endif ++ #endif ++ + __free_pages(&folio->page, order); + } + +diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c +index a2cbe44c48e1..7cf05d293312 100644 +--- a/mm/sparse-vmemmap.c ++++ b/mm/sparse-vmemmap.c +@@ -28,6 +28,10 @@ + #include <linux/vmalloc.h> + #include <linux/sched.h> + ++#ifdef CONFIG_PTP ++#include <linux/iee-func.h> ++#endif ++ + #include <asm/dma.h> + #include <asm/pgalloc.h> + +@@ -146,6 +150,9 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, + struct page *reuse) + { + pte_t *pte = pte_offset_kernel(pmd, addr); ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + if (pte_none(ptep_get(pte))) { + pte_t entry; + void *p; +@@ -167,6 +174,11 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, + get_page(reuse); + p = page_to_virt(reuse); + } ++#ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(p)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)p); ++#endif + entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); + set_pte_at(&init_mm, addr, pte, entry); + } +@@ -176,11 +188,20 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, + static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) + { + void *p = vmemmap_alloc_block(size, node); ++ #ifdef CONFIG_PTP ++ unsigned long iee_addr; ++ #endif + + if (!p) + return NULL; + memset(p, 0, size); + ++ #ifdef CONFIG_PTP ++ iee_addr = __phys_to_iee(__pa(p)); ++ set_iee_page_valid(iee_addr); ++ iee_set_logical_mem_ro((unsigned long)p); ++ #endif ++ + return p; + } + +diff --git a/mm/vmalloc.c b/mm/vmalloc.c +index e6058942a084..27a006728009 100644 +--- a/mm/vmalloc.c ++++ b/mm/vmalloc.c +@@ -3431,7 +3431,7 @@ static int vmap_pfn_apply(pte_t *pte, unsigned long addr, void *private) + + if (WARN_ON_ONCE(pfn_valid(pfn))) + return -EINVAL; +- ++ + ptent = pte_mkspecial(pfn_pte(pfn, data->prot)); + set_pte_at(&init_mm, addr, pte, ptent); + +diff --git a/net/dns_resolver/dns_key.c b/net/dns_resolver/dns_key.c +index c42ddd85ff1f..4714b4f2be08 100644 +--- a/net/dns_resolver/dns_key.c ++++ b/net/dns_resolver/dns_key.c +@@ -34,6 +34,10 @@ + #include <keys/user-type.h> + #include "internal.h" + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + MODULE_DESCRIPTION("DNS Resolver"); + MODULE_AUTHOR("Wang Lei"); + MODULE_LICENSE("GPL"); +@@ -365,8 +369,13 @@ static int __init init_dns_resolver(void) + /* instruct request_key() to use this special keyring as a cache for + * the results it looks up */ + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(cred,keyring); ++ iee_set_cred_jit_keyring(cred,KEY_REQKEY_DEFL_THREAD_KEYRING); ++ #else + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; ++ #endif + dns_resolver_cache = cred; + + kdebug("DNS resolver keyring: %d\n", key_serial(keyring)); +diff --git a/security/commoncap.c b/security/commoncap.c +index bc0521104197..d7d3b7cc13e8 100644 +--- a/security/commoncap.c ++++ b/security/commoncap.c +@@ -26,6 +26,10 @@ + #include <linux/personality.h> + #include <linux/mnt_idmapping.h> + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + /* + * If a non-root user executes a setuid-root binary in + * !secure(SECURE_NOROOT) mode, then we raise capabilities. +@@ -266,6 +270,15 @@ int cap_capset(struct cred *new, + if (!cap_issubset(*effective, *permitted)) + return -EPERM; + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,*effective); ++ iee_set_cred_cap_inheritable(new,*inheritable); ++ iee_set_cred_cap_permitted(new,*permitted); ++ ++ iee_set_cred_cap_ambient(new,cap_intersect(new->cap_ambient, ++ cap_intersect(*permitted, ++ *inheritable))); ++ #else + new->cap_effective = *effective; + new->cap_inheritable = *inheritable; + new->cap_permitted = *permitted; +@@ -277,6 +290,7 @@ int cap_capset(struct cred *new, + new->cap_ambient = cap_intersect(new->cap_ambient, + cap_intersect(*permitted, + *inheritable)); ++ #endif + if (WARN_ON(!cap_ambient_invariant_ok(new))) + return -EINVAL; + return 0; +@@ -601,9 +615,16 @@ static inline int bprm_caps_from_vfs_caps(struct cpu_vfs_cap_data *caps, + * pP' = (X & fP) | (pI & fI) + * The addition of pA' is handled later. + */ ++#ifdef CONFIG_CREDP ++ kernel_cap_t temp = new->cap_permitted; ++ temp.val = (new->cap_bset.val & caps->permitted.val) | ++ (new->cap_inheritable.val & caps->inheritable.val); ++ iee_set_cred_cap_permitted(new,temp); ++#else + new->cap_permitted.val = + (new->cap_bset.val & caps->permitted.val) | + (new->cap_inheritable.val & caps->inheritable.val); ++#endif + + if (caps->permitted.val & ~new->cap_permitted.val) + /* insufficient to execute correctly */ +@@ -726,7 +747,15 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, + int rc = 0; + struct cpu_vfs_cap_data vcaps; + ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = bprm->cred->cap_permitted; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_permitted(bprm->cred, tmp_cap); ++ } while (0); ++ #else + cap_clear(bprm->cred->cap_permitted); ++ #endif + + if (!file_caps_enabled) + return 0; +@@ -757,7 +786,15 @@ static int get_file_caps(struct linux_binprm *bprm, struct file *file, + + out: + if (rc) ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = bprm->cred->cap_permitted; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_permitted(bprm->cred, tmp_cap); ++ } while (0); ++ #else + cap_clear(bprm->cred->cap_permitted); ++ #endif + + return rc; + } +@@ -809,8 +846,13 @@ static void handle_privileged_root(struct linux_binprm *bprm, bool has_fcap, + */ + if (__is_eff(root_uid, new) || __is_real(root_uid, new)) { + /* pP' = (cap_bset & ~0) | (pI & ~0) */ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_permitted(new,cap_combine(old->cap_bset, ++ old->cap_inheritable)); ++ #else + new->cap_permitted = cap_combine(old->cap_bset, + old->cap_inheritable); ++ #endif + } + /* + * If only the real uid is 0, we do not set the effective bit. +@@ -919,34 +961,71 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file) + /* downgrade; they get no more than they had, and maybe less */ + if (!ns_capable(new->user_ns, CAP_SETUID) || + (bprm->unsafe & LSM_UNSAFE_NO_NEW_PRIVS)) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_euid(new,new->uid); ++ iee_set_cred_egid(new,new->gid); ++ #else + new->euid = new->uid; + new->egid = new->gid; ++ #endif + } ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_permitted(new,cap_intersect(new->cap_permitted, ++ old->cap_permitted)); ++ #else + new->cap_permitted = cap_intersect(new->cap_permitted, + old->cap_permitted); ++ #endif + } + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_fsuid(new,new->euid); ++ iee_set_cred_suid(new,new->euid); ++ iee_set_cred_fsgid(new,new->egid); ++ iee_set_cred_sgid(new,new->egid); ++ #else + new->suid = new->fsuid = new->euid; + new->sgid = new->fsgid = new->egid; ++ #endif + + /* File caps or setid cancels ambient. */ + if (has_fcap || is_setid) ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = new->cap_ambient; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_ambient(new, tmp_cap); ++ } while (0); ++ #else + cap_clear(new->cap_ambient); ++ #endif + + /* + * Now that we've computed pA', update pP' to give: + * pP' = (X & fP) | (pI & fI) | pA' + */ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_permitted(new,cap_combine(new->cap_permitted, new->cap_ambient)); ++ #else + new->cap_permitted = cap_combine(new->cap_permitted, new->cap_ambient); ++ #endif + + /* + * Set pE' = (fE ? pP' : pA'). Because pA' is zero if fE is set, + * this is the same as pE' = (fE ? pP' : 0) | pA'. + */ + if (effective) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,new->cap_permitted); ++ #else + new->cap_effective = new->cap_permitted; ++ #endif + else ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,new->cap_ambient); ++ #else + new->cap_effective = new->cap_ambient; ++ #endif + + if (WARN_ON(!cap_ambient_invariant_ok(new))) + return -EPERM; +@@ -957,7 +1036,11 @@ int cap_bprm_creds_from_file(struct linux_binprm *bprm, struct file *file) + return ret; + } + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_securebits(new,new->securebits & ~issecure_mask(SECURE_KEEP_CAPS)); ++ #else + new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS); ++ #endif + + if (WARN_ON(!cap_ambient_invariant_ok(new))) + return -EPERM; +@@ -1092,8 +1175,21 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old) + !uid_eq(new->euid, root_uid) && + !uid_eq(new->suid, root_uid))) { + if (!issecure(SECURE_KEEP_CAPS)) { ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = new->cap_permitted; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_permitted(new, tmp_cap); ++ } while (0); ++ do { ++ kernel_cap_t tmp_cap = new->cap_effective; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_effective(new, tmp_cap); ++ } while (0); ++ #else + cap_clear(new->cap_permitted); + cap_clear(new->cap_effective); ++ #endif + } + + /* +@@ -1101,12 +1197,32 @@ static inline void cap_emulate_setxuid(struct cred *new, const struct cred *old) + * by exec to drop capabilities. We should make sure that + * this remains the case. + */ ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = new->cap_ambient; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_ambient(new, tmp_cap); ++ } while (0); ++ #else + cap_clear(new->cap_ambient); ++ #endif + } + if (uid_eq(old->euid, root_uid) && !uid_eq(new->euid, root_uid)) ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = new->cap_effective; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_effective(new, tmp_cap); ++ } while (0); ++ #else + cap_clear(new->cap_effective); ++ #endif + if (!uid_eq(old->euid, root_uid) && uid_eq(new->euid, root_uid)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,new->cap_permitted); ++ #else + new->cap_effective = new->cap_permitted; ++ #endif + } + + /** +@@ -1142,13 +1258,22 @@ int cap_task_fix_setuid(struct cred *new, const struct cred *old, int flags) + if (!issecure(SECURE_NO_SETUID_FIXUP)) { + kuid_t root_uid = make_kuid(old->user_ns, 0); + if (uid_eq(old->fsuid, root_uid) && !uid_eq(new->fsuid, root_uid)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,cap_drop_fs_set(new->cap_effective)); ++ #else + new->cap_effective = + cap_drop_fs_set(new->cap_effective); ++ #endif + + if (!uid_eq(old->fsuid, root_uid) && uid_eq(new->fsuid, root_uid)) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_cap_effective(new,cap_raise_fs_set(new->cap_effective, ++ new->cap_permitted)); ++ #else + new->cap_effective = + cap_raise_fs_set(new->cap_effective, + new->cap_permitted); ++ #endif + } + break; + +@@ -1243,7 +1368,15 @@ static int cap_prctl_drop(unsigned long cap) + new = prepare_creds(); + if (!new) + return -ENOMEM; ++ #ifdef CONFIG_CREDP ++ { ++ kernel_cap_t tmp = new->cap_bset; ++ cap_lower(tmp, cap); ++ iee_set_cred_cap_bset(new, tmp); ++ } ++ #else + cap_lower(new->cap_bset, cap); ++ #endif + return commit_creds(new); + } + +@@ -1319,7 +1452,11 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, + new = prepare_creds(); + if (!new) + return -ENOMEM; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_securebits(new,arg2); ++ #else + new->securebits = arg2; ++ #endif + return commit_creds(new); + + case PR_GET_SECUREBITS: +@@ -1338,9 +1475,17 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, + if (!new) + return -ENOMEM; + if (arg2) ++ #ifdef CONFIG_CREDP ++ iee_set_cred_securebits(new,new->securebits | issecure_mask(SECURE_KEEP_CAPS)); ++ #else + new->securebits |= issecure_mask(SECURE_KEEP_CAPS); ++ #endif + else ++ #ifdef CONFIG_CREDP ++ iee_set_cred_securebits(new,new->securebits & ~issecure_mask(SECURE_KEEP_CAPS)); ++ #else + new->securebits &= ~issecure_mask(SECURE_KEEP_CAPS); ++ #endif + return commit_creds(new); + + case PR_CAP_AMBIENT: +@@ -1351,7 +1496,15 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, + new = prepare_creds(); + if (!new) + return -ENOMEM; ++ #ifdef CONFIG_CREDP ++ do { ++ kernel_cap_t tmp_cap = new->cap_ambient; ++ tmp_cap.val = 0; ++ iee_set_cred_cap_ambient(new, tmp_cap); ++ } while (0); ++ #else + cap_clear(new->cap_ambient); ++ #endif + return commit_creds(new); + } + +@@ -1375,9 +1528,25 @@ int cap_task_prctl(int option, unsigned long arg2, unsigned long arg3, + if (!new) + return -ENOMEM; + if (arg2 == PR_CAP_AMBIENT_RAISE) ++ #ifdef CONFIG_CREDP ++ { ++ kernel_cap_t tmp = new->cap_ambient; ++ cap_raise(tmp, arg3); ++ iee_set_cred_cap_ambient(new, tmp); ++ } ++ #else + cap_raise(new->cap_ambient, arg3); ++ #endif + else ++ #ifdef CONFIG_CREDP ++ { ++ kernel_cap_t tmp = new->cap_ambient; ++ cap_lower(tmp, arg3); ++ iee_set_cred_cap_ambient(new, tmp); ++ } ++ #else + cap_lower(new->cap_ambient, arg3); ++ #endif + return commit_creds(new); + } + +diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c +index 19be69fa4d05..6cb164dfc19b 100644 +--- a/security/keys/keyctl.c ++++ b/security/keys/keyctl.c +@@ -23,6 +23,9 @@ + #include <linux/uaccess.h> + #include <keys/request_key_auth-type.h> + #include "internal.h" ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif + + #define KEY_MAX_DESC_SIZE 4096 + +@@ -1155,7 +1158,11 @@ static int keyctl_change_reqkey_auth(struct key *key) + return -ENOMEM; + + key_put(new->request_key_auth); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_request_key_auth(new,key_get(key)); ++ #else + new->request_key_auth = key_get(key); ++ #endif + + return commit_creds(new); + } +@@ -1432,7 +1439,11 @@ long keyctl_set_reqkey_keyring(int reqkey_defl) + } + + set: ++ #ifdef CONFIG_CREDP ++ iee_set_cred_jit_keyring(new,reqkey_defl); ++ #else + new->jit_keyring = reqkey_defl; ++ #endif + commit_creds(new); + return old_setting; + error: +@@ -1644,9 +1655,17 @@ long keyctl_session_to_parent(void) + cred = cred_alloc_blank(); + if (!cred) + goto error_keyring; ++ #ifdef CONFIG_CREDP ++ newwork = (struct rcu_head *)(cred->rcu.func); ++ #else + newwork = &cred->rcu; ++ #endif + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_session_keyring(cred,key_ref_to_ptr(keyring_r)); ++ #else + cred->session_keyring = key_ref_to_ptr(keyring_r); ++ #endif + keyring_r = NULL; + init_task_work(newwork, key_change_session_keyring); + +@@ -1705,7 +1724,11 @@ long keyctl_session_to_parent(void) + write_unlock_irq(&tasklist_lock); + rcu_read_unlock(); + if (oldwork) ++ #ifdef CONFIG_CREDP ++ put_cred(*(struct cred **)(oldwork + 1)); ++ #else + put_cred(container_of(oldwork, struct cred, rcu)); ++ #endif + if (newwork) + put_cred(cred); + return ret; +diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c +index b5d5333ab330..aaa2a2347d84 100644 +--- a/security/keys/process_keys.c ++++ b/security/keys/process_keys.c +@@ -19,6 +19,10 @@ + #include <keys/request_key_auth-type.h> + #include "internal.h" + ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif ++ + /* Session keyring create vs join semaphore */ + static DEFINE_MUTEX(key_session_mutex); + +@@ -232,7 +236,11 @@ int install_thread_keyring_to_cred(struct cred *new) + if (IS_ERR(keyring)) + return PTR_ERR(keyring); + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_thread_keyring(new,keyring); ++ #else + new->thread_keyring = keyring; ++ #endif + return 0; + } + +@@ -279,7 +287,11 @@ int install_process_keyring_to_cred(struct cred *new) + if (IS_ERR(keyring)) + return PTR_ERR(keyring); + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_process_keyring(new,keyring); ++ #else + new->process_keyring = keyring; ++ #endif + return 0; + } + +@@ -338,7 +350,11 @@ int install_session_keyring_to_cred(struct cred *cred, struct key *keyring) + + /* install the keyring */ + old = cred->session_keyring; ++ #ifdef CONFIG_CREDP ++ iee_set_cred_session_keyring(cred,keyring); ++ #else + cred->session_keyring = keyring; ++ #endif + + if (old) + key_put(old); +@@ -911,7 +927,11 @@ long join_session_keyring(const char *name) + void key_change_session_keyring(struct callback_head *twork) + { + const struct cred *old = current_cred(); ++ #ifdef CONFIG_CREDP ++ struct cred *new = *(struct cred **)(twork + 1); ++ #else + struct cred *new = container_of(twork, struct cred, rcu); ++ #endif + + if (unlikely(current->flags & PF_EXITING)) { + put_cred(new); +@@ -925,6 +945,38 @@ void key_change_session_keyring(struct callback_head *twork) + return; + } + ++ /* If get_ucounts fails more bits are needed in the refcount */ ++ if (unlikely(!get_ucounts(old->ucounts))) { ++ WARN_ONCE(1, "In %s get_ucounts failed\n", __func__); ++ put_cred(new); ++ return; ++ } ++ ++ #ifdef CONFIG_CREDP ++ iee_set_cred_uid(new,old-> uid); ++ iee_set_cred_euid(new,old-> euid); ++ iee_set_cred_suid(new,old-> suid); ++ iee_set_cred_fsuid(new,old->fsuid); ++ iee_set_cred_gid(new,old-> gid); ++ iee_set_cred_egid(new,old-> egid); ++ iee_set_cred_sgid(new,old-> sgid); ++ iee_set_cred_fsgid(new,old->fsgid); ++ iee_set_cred_user(new,get_uid(old->user)); ++ iee_set_cred_ucounts(new, old->ucounts); ++ iee_set_cred_user_ns(new,get_user_ns(old->user_ns)); ++ iee_set_cred_group_info(new,get_group_info(old->group_info)); ++ ++ iee_set_cred_securebits(new,old->securebits); ++ iee_set_cred_cap_inheritable(new,old->cap_inheritable); ++ iee_set_cred_cap_permitted(new,old->cap_permitted); ++ iee_set_cred_cap_effective(new,old->cap_effective); ++ iee_set_cred_cap_ambient(new,old->cap_ambient); ++ iee_set_cred_cap_bset(new,old->cap_bset); ++ ++ iee_set_cred_jit_keyring(new,old->jit_keyring); ++ iee_set_cred_thread_keyring(new,key_get(old->thread_keyring)); ++ iee_set_cred_process_keyring(new,key_get(old->process_keyring)); ++ #else + new-> uid = old-> uid; + new-> euid = old-> euid; + new-> suid = old-> suid; +@@ -948,6 +1000,7 @@ void key_change_session_keyring(struct callback_head *twork) + new->jit_keyring = old->jit_keyring; + new->thread_keyring = key_get(old->thread_keyring); + new->process_keyring = key_get(old->process_keyring); ++ #endif + + security_transfer_creds(new, old); + +diff --git a/security/security.c b/security/security.c +index 407b51719f79..74ffd7ea3f37 100644 +--- a/security/security.c ++++ b/security/security.c +@@ -30,6 +30,9 @@ + #include <linux/string.h> + #include <linux/msg.h> + #include <net/flow.h> ++#ifdef CONFIG_CREDP ++#include <asm/iee-cred.h> ++#endif + + /* How many LSMs were built into the kernel? */ + #define LSM_COUNT (__end_lsm_info - __start_lsm_info) +@@ -570,11 +573,19 @@ EXPORT_SYMBOL(unregister_blocking_lsm_notifier); + static int lsm_cred_alloc(struct cred *cred, gfp_t gfp) + { + if (blob_sizes.lbs_cred == 0) { ++ #ifdef CONFIG_CREDP ++ iee_set_cred_security(cred,NULL); ++ #else + cred->security = NULL; ++ #endif + return 0; + } + ++ #ifdef CONFIG_CREDP ++ iee_set_cred_security(cred,kzalloc(blob_sizes.lbs_cred, gfp)); ++ #else + cred->security = kzalloc(blob_sizes.lbs_cred, gfp); ++ #endif + if (cred->security == NULL) + return -ENOMEM; + return 0; +@@ -2950,7 +2961,11 @@ void security_cred_free(struct cred *cred) + call_void_hook(cred_free, cred); + + kfree(cred->security); ++ #ifdef CONFIG_CREDP ++ iee_set_cred_security(cred,NULL); ++ #else + cred->security = NULL; ++ #endif + } + + /** +-- +2.34.1 + |