From c9910a21ebb63403332c415850c1640bc056cfaa Mon Sep 17 00:00:00 2001 From: Mahesh Salgaonkar Date: Mon, 8 Jan 2018 14:27:27 +0530 Subject: [PATCH 01/26] UBUNTU: SAUCE: powerpc: Prevent Meltdown attack with L1-D$ flush CVE-2017-5754 This commit inserts nops before each rfid/hrfid and patches in an L1-D cache flush instruction when firmware advises us that is possible. It provides /sys/devices/system/cpu/rfi_flush which can report and can enabled/disable the rfi flushes at runtime. Includes support for querying the device tree, or hypervisor, to determine the platform's capabilities and requirements. The RFI flush mitigation can be disabled by booting with either 'no_rfi_flush' or 'nopti' on the kernel command line. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Signed-off-by: Michael Neuling Signed-off-by: Oliver O'Halloran Signed-off-by: Mahesh Salgaonkar Signed-off-by: Stefan Bader --- arch/powerpc/Kconfig.debug | 10 ++ arch/powerpc/include/asm/exception-64e.h | 6 ++ arch/powerpc/include/asm/exception-64s.h | 70 +++++++++++++- arch/powerpc/include/asm/feature-fixups.h | 12 +++ arch/powerpc/include/asm/hvcall.h | 14 +++ arch/powerpc/include/asm/paca.h | 11 +++ arch/powerpc/include/asm/plpar_wrappers.h | 16 ++++ arch/powerpc/include/asm/ppc_asm.h | 6 +- arch/powerpc/include/asm/setup.h | 13 +++ arch/powerpc/kernel/asm-offsets.c | 4 + arch/powerpc/kernel/entry_64.S | 44 +++++++-- arch/powerpc/kernel/exceptions-64s.S | 147 +++++++++++++++++++++++++++--- arch/powerpc/kernel/setup_64.c | 97 ++++++++++++++++++++ arch/powerpc/kernel/sysfs.c | 43 +++++++++ arch/powerpc/kernel/vmlinux.lds.S | 9 ++ arch/powerpc/kvm/book3s_hv_rmhandlers.S | 7 +- arch/powerpc/kvm/book3s_rmhandlers.S | 4 +- arch/powerpc/lib/feature-fixups.c | 42 +++++++++ arch/powerpc/platforms/powernv/setup.c | 49 ++++++++++ arch/powerpc/platforms/pseries/setup.c | 34 +++++++ 20 files changed, 606 insertions(+), 32 deletions(-) diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index 21c9f30..418c38d 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -363,4 +363,14 @@ config FAIL_IOMMU If you are unsure, say N. +config PPC_DEBUG_RFI + bool "Debug RFIs (Return From Interrupt)" + depends on PPC_BOOK3S_64 + help + The enables extra debug code in some RFI (Return From Interrupt) + sequences, to detect that we are returning to the correct context + (user, kernel or guest). This adds some performance overhead. + + If unsure, say N. + endmenu diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h index 371a77f..fa193c1 100644 --- a/arch/powerpc/include/asm/exception-64e.h +++ b/arch/powerpc/include/asm/exception-64e.h @@ -230,5 +230,11 @@ exc_##label##_book3e: mtspr SPRN_IVOR##vector_number,r3; #endif /* CONFIG_RELOCATABLE */ +#define RFI_TO_KERNEL \ + rfi + +#define RFI_TO_USER \ + rfi + #endif /* _ASM_POWERPC_EXCEPTION_64E_H */ diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index aeaa56c..9118863 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -34,6 +34,7 @@ * exception handlers (including pSeries LPAR) and iSeries LPAR * implementations as possible. */ +#include #define EX_R9 0 #define EX_R10 8 @@ -50,6 +51,73 @@ #define EX_PPR 88 /* SMT thread status register (priority) */ #define EX_CTR 96 +/* + * The nop instructions allow us to insert one or more instructions to flush the + * L1-D cache when return to userspace or a guest. + */ +#define RFI_FLUSH_SLOT \ + RFI_FLUSH_FIXUP_SECTION; \ + nop; \ + nop; \ + nop + +#ifdef CONFIG_PPC_DEBUG_RFI +#define CHECK_TARGET_MSR_PR(srr_reg, expected_pr) \ + SET_SCRATCH0(r3); \ + mfspr r3,srr_reg; \ + extrdi r3,r3,1,63-MSR_PR_LG; \ +666: tdnei r3,expected_pr; \ + EMIT_BUG_ENTRY 666b,__FILE__,__LINE__,0; \ + GET_SCRATCH0(r3); +#else +#define CHECK_TARGET_MSR_PR(srr_reg, expected_pr) +#endif + +#define RFI_TO_KERNEL \ + CHECK_TARGET_MSR_PR(SPRN_SRR1, 0); \ + rfid + +#define RFI_TO_USER \ + CHECK_TARGET_MSR_PR(SPRN_SRR1, 1); \ + RFI_FLUSH_SLOT; \ + rfid; \ + b rfi_flush_fallback + +#define RFI_TO_USER_OR_KERNEL \ + RFI_FLUSH_SLOT; \ + rfid; \ + b rfi_flush_fallback + +#define RFI_TO_GUEST \ + RFI_FLUSH_SLOT; \ + rfid; \ + b rfi_flush_fallback + +#define HRFI_TO_KERNEL \ + CHECK_TARGET_MSR_PR(SPRN_HSRR1, 0); \ + hrfid + +#define HRFI_TO_USER \ + CHECK_TARGET_MSR_PR(SPRN_HSRR1, 1); \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define HRFI_TO_USER_OR_KERNEL \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define HRFI_TO_GUEST \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + +#define HRFI_TO_UNKNOWN \ + RFI_FLUSH_SLOT; \ + hrfid; \ + b hrfi_flush_fallback + #ifdef CONFIG_RELOCATABLE #define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ ld r12,PACAKBASE(r13); /* get high part of &label */ \ @@ -191,7 +259,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) mtspr SPRN_##h##SRR0,r12; \ mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ mtspr SPRN_##h##SRR1,r10; \ - h##rfid; \ + h##RFI_TO_KERNEL; \ b . /* prevent speculative execution */ #define EXCEPTION_PROLOG_PSERIES_1(label, h) \ __EXCEPTION_PROLOG_PSERIES_1(label, h) diff --git a/arch/powerpc/include/asm/feature-fixups.h b/arch/powerpc/include/asm/feature-fixups.h index 9a67a38..2a962ce 100644 --- a/arch/powerpc/include/asm/feature-fixups.h +++ b/arch/powerpc/include/asm/feature-fixups.h @@ -144,7 +144,19 @@ label##5: \ #define ALT_FW_FTR_SECTION_END_IFCLR(msk) \ ALT_FW_FTR_SECTION_END_NESTED_IFCLR(msk, 97) +#define RFI_FLUSH_FIXUP_SECTION \ +951: \ + .pushsection __rfi_flush_fixup,"a"; \ + .align 2; \ +952: \ + FTR_ENTRY_OFFSET 951b-952b; \ + .popsection; + + #ifndef __ASSEMBLY__ +#include + +extern long __start___rfi_flush_fixup, __stop___rfi_flush_fixup; #define ASM_FTR_IF(section_if, section_else, msk, val) \ stringify_in_c(BEGIN_FTR_SECTION) \ diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index a9bd4b3..60ac63b 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -239,6 +239,7 @@ #define H_GET_HCA_INFO 0x1B8 #define H_GET_PERF_COUNT 0x1BC #define H_MANAGE_TRACE 0x1C0 +#define H_GET_CPU_CHARACTERISTICS 0x1C8 #define H_FREE_LOGICAL_LAN_BUFFER 0x1D4 #define H_QUERY_INT_STATE 0x1E4 #define H_POLL_PENDING 0x1D8 @@ -280,6 +281,19 @@ #define H_SET_MODE_RESOURCE_ADDR_TRANS_MODE 3 #define H_SET_MODE_RESOURCE_LE 4 +/* H_GET_CPU_CHARACTERISTICS return values */ +#define H_GET_CPU_CHAR_CHAR_ORI31_SPEC_BAR PPC_BIT(0) +#define H_GET_CPU_CHAR_CHAR_BCCTR_SERIAL PPC_BIT(1) +#define H_GET_CPU_CHAR_CHAR_ORI30_L1_FLUSH PPC_BIT(2) +#define H_GET_CPU_CHAR_CHAR_MTTRIG2_L1_FLUSH PPC_BIT(3) +#define H_GET_CPU_CHAR_CHAR_L1D_PRIVATE PPC_BIT(4) +#define H_GET_CPU_CHAR_CHAR_BC_HINTS_HONORED PPC_BIT(5) +#define H_GET_CPU_CHAR_CHAR_MTTRID01_THR_CFG PPC_BIT(6) + +#define H_GET_CPU_CHAR_BEHAV_FAV_SEC_VS_PERF PPC_BIT(0) +#define H_GET_CPU_CHAR_BEHAV_L1_FLUSH_LOW_PRIV PPC_BIT(1) +#define H_GET_CPU_CHAR_BEHAV_SPEC_BAR_BNDS_CHK PPC_BIT(2) + #ifndef __ASSEMBLY__ /** diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index c3523d1..20aa58b 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -153,6 +153,8 @@ struct paca_struct { struct opal_machine_check_event *opal_mc_evt; #endif #ifdef CONFIG_PPC_BOOK3S_64 + void *rfi_flush_fallback_area; + /* Exclusive emergency stack pointer for machine check exception. */ void *mc_emergency_sp; /* @@ -181,6 +183,15 @@ struct paca_struct { #endif struct kvmppc_host_state kvm_hstate; #endif +#ifdef CONFIG_PPC_BOOK3S_64 + /* + * rfi fallback flush must be in its own cacheline to prevent + * other paca data leaking into the L1d + */ + u64 exrfi[13] __aligned(0x80); + u64 l1d_flush_congruence; + u64 l1d_flush_sets; +#endif }; extern struct paca_struct *paca; diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h index 12c32c5..8ea880a 100644 --- a/arch/powerpc/include/asm/plpar_wrappers.h +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -323,4 +323,20 @@ static inline long plapr_set_watchpoint0(unsigned long dawr0, unsigned long dawr return plpar_set_mode(0, 2, dawr0, dawrx0); } +static inline long plpar_get_cpu_characteristics(unsigned long *character, + unsigned long *behavior) +{ + long rc; + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + + rc = plpar_hcall(H_GET_CPU_CHARACTERISTICS, retbuf); + + if (character) + *character = retbuf[0]; + if (behavior) + *behavior = retbuf[1]; + + return rc; +} + #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 7c04486..0425c6d 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -11,9 +11,7 @@ #include #include -#ifndef __ASSEMBLY__ -#error __FILE__ should only be used in assembler files -#else +#ifdef __ASSEMBLY__ #define SZL (BITS_PER_LONG/8) @@ -790,5 +788,7 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,945) .long 0xa6037b7d; /* mtsrr1 r11 */ \ .long 0x2400004c /* rfid */ #endif /* !CONFIG_PPC_BOOK3E */ + #endif /* __ASSEMBLY__ */ + #endif /* _ASM_POWERPC_PPC_ASM_H */ diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index 703a841..e84bf9e 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -27,6 +27,19 @@ void check_for_initrd(void); void do_init_bootmem(void); void setup_panic(void); +void rfi_flush_enable(bool enable); + +/* These are bit flags */ +enum l1d_flush_type { + L1D_FLUSH_NONE = 0x1, + L1D_FLUSH_FALLBACK = 0x2, + L1D_FLUSH_ORI = 0x4, + L1D_FLUSH_MTTRIG = 0x8, +}; + +void __init setup_rfi_flush(enum l1d_flush_type, bool enable); +void do_rfi_flush_fixups(enum l1d_flush_type types); + #endif /* !__ASSEMBLY__ */ #endif /* _ASM_POWERPC_SETUP_H */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index d188ebf..c5081f2 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -235,6 +235,10 @@ int main(void) #ifdef CONFIG_PPC_BOOK3S_64 DEFINE(PACAMCEMERGSP, offsetof(struct paca_struct, mc_emergency_sp)); DEFINE(PACA_IN_MCE, offsetof(struct paca_struct, in_mce)); + OFFSET(PACA_RFI_FLUSH_FALLBACK_AREA, paca_struct, rfi_flush_fallback_area); + OFFSET(PACA_EXRFI, paca_struct, exrfi); + OFFSET(PACA_L1D_FLUSH_CONGRUENCE, paca_struct, l1d_flush_congruence); + OFFSET(PACA_L1D_FLUSH_SETS, paca_struct, l1d_flush_sets); #endif DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id)); DEFINE(PACAKEXECSTATE, offsetof(struct paca_struct, kexec_state)); diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 145f39c..8f400c1d 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -35,6 +35,11 @@ #include #include #include +#ifdef CONFIG_PPC_BOOK3S +#include +#else +#include +#endif /* * System calls. @@ -238,13 +243,23 @@ END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) ACCOUNT_CPU_USER_EXIT(r11, r12) HMT_MEDIUM_LOW_HAS_PPR ld r13,GPR13(r1) /* only restore r13 if returning to usermode */ + ld r2,GPR2(r1) + ld r1,GPR1(r1) + mtlr r4 + mtcr r5 + mtspr SPRN_SRR0,r7 + mtspr SPRN_SRR1,r8 + RFI_TO_USER + b . /* prevent speculative execution */ + + /* exit to kernel */ 1: ld r2,GPR2(r1) ld r1,GPR1(r1) mtlr r4 mtcr r5 mtspr SPRN_SRR0,r7 mtspr SPRN_SRR1,r8 - RFI + RFI_TO_KERNEL b . /* prevent speculative execution */ syscall_error: @@ -354,8 +369,7 @@ tabort_syscall: mtmsrd r10, 1 mtspr SPRN_SRR0, r11 mtspr SPRN_SRR1, r12 - - rfid + RFI_TO_USER b . /* prevent speculative execution */ #endif @@ -892,7 +906,7 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ACCOUNT_CPU_USER_EXIT(r2, r4) REST_GPR(13, r1) -1: + mtspr SPRN_SRR1,r3 ld r2,_CCR(r1) @@ -905,8 +919,22 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ld r3,GPR3(r1) ld r4,GPR4(r1) ld r1,GPR1(r1) + RFI_TO_USER + b . /* prevent speculative execution */ - rfid +1: mtspr SPRN_SRR1,r3 + + ld r2,_CCR(r1) + mtcrf 0xFF,r2 + ld r2,_NIP(r1) + mtspr SPRN_SRR0,r2 + + ld r0,GPR0(r1) + ld r2,GPR2(r1) + ld r3,GPR3(r1) + ld r4,GPR4(r1) + ld r1,GPR1(r1) + RFI_TO_KERNEL b . /* prevent speculative execution */ #endif /* CONFIG_PPC_BOOK3E */ @@ -1077,7 +1105,7 @@ _GLOBAL(enter_rtas) mtspr SPRN_SRR0,r5 mtspr SPRN_SRR1,r6 - rfid + RFI_TO_KERNEL b . /* prevent speculative execution */ _STATIC(rtas_return_loc) @@ -1102,7 +1130,7 @@ _STATIC(rtas_return_loc) mtspr SPRN_SRR0,r3 mtspr SPRN_SRR1,r4 - rfid + RFI_TO_KERNEL b . /* prevent speculative execution */ .align 3 @@ -1173,7 +1201,7 @@ _GLOBAL(enter_prom) LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE) andc r11,r11,r12 mtsrr1 r11 - rfid + RFI_TO_KERNEL #endif /* CONFIG_PPC_BOOK3E */ 1: /* Return from OF */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index a113575..2d2fbd4 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -45,7 +45,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \ mtspr SPRN_SRR0,r10 ; \ ld r10,PACAKMSR(r13) ; \ mtspr SPRN_SRR1,r10 ; \ - rfid ; \ + RFI_TO_KERNEL ; \ b . ; /* prevent speculative execution */ #define SYSCALL_PSERIES_3 \ @@ -53,7 +53,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \ 1: mfspr r12,SPRN_SRR1 ; \ xori r12,r12,MSR_LE ; \ mtspr SPRN_SRR1,r12 ; \ - rfid ; /* return to userspace */ \ + RFI_TO_USER ; /* return to userspace */ \ b . ; \ 2: mfspr r12,SPRN_SRR1 ; \ andi. r12,r12,MSR_PR ; \ @@ -61,7 +61,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \ mtspr SPRN_SRR0,r3 ; \ mtspr SPRN_SRR1,r4 ; \ mtspr SPRN_SDR1,r5 ; \ - rfid ; \ + RFI_TO_KERNEL ; \ b . ; /* prevent speculative execution */ #if defined(CONFIG_RELOCATABLE) @@ -486,7 +486,7 @@ BEGIN_FTR_SECTION LOAD_HANDLER(r12, machine_check_handle_early) mtspr SPRN_SRR0,r12 mtspr SPRN_SRR1,r11 - rfid + RFI_TO_KERNEL b . /* prevent speculative execution */ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) @@ -595,7 +595,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r11,PACA_EXGEN+EX_R11(r13) ld r12,PACA_EXGEN+EX_R12(r13) ld r13,PACA_EXGEN+EX_R13(r13) - HRFID + HRFI_TO_UNKNOWN b . #endif @@ -655,12 +655,92 @@ masked_##_H##interrupt: \ ld r10,PACA_EXGEN+EX_R10(r13); \ ld r11,PACA_EXGEN+EX_R11(r13); \ GET_SCRATCH0(r13); \ - ##_H##rfid; \ + ##_H##RFI_TO_KERNEL; \ b . MASKED_INTERRUPT() MASKED_INTERRUPT(H) + .globl rfi_flush_fallback +rfi_flush_fallback: + SET_SCRATCH0(r13); + GET_PACA(r13); + std r9,PACA_EXRFI+EX_R9(r13) + std r10,PACA_EXRFI+EX_R10(r13) + std r11,PACA_EXRFI+EX_R11(r13) + std r12,PACA_EXRFI+EX_R12(r13) + std r8,PACA_EXRFI+EX_R13(r13) + mfctr r9 + ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) + ld r11,PACA_L1D_FLUSH_SETS(r13) + ld r12,PACA_L1D_FLUSH_CONGRUENCE(r13) + /* + * The load adresses are at staggered offsets within cachelines, + * which suits some pipelines better (on others it should not + * hurt. + */ + addi r12,r12,8 + mtctr r11 + DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ + /* XXX: Should an instruction synchronizing operation be done here? */ + +1: li r8,0 + .rept 8 /* 8-way set associative */ + ldx r11,r10,r8 + add r8,r8,r12 + .endr + addi r10,r10,128 /* 128 byte cache line */ + bdnz 1b + + mtctr r9 + ld r9,PACA_EXRFI+EX_R9(r13) + ld r10,PACA_EXRFI+EX_R10(r13) + ld r11,PACA_EXRFI+EX_R11(r13) + ld r12,PACA_EXRFI+EX_R12(r13) + ld r8,PACA_EXRFI+EX_R13(r13) + GET_SCRATCH0(r13); + rfid + + .globl hrfi_flush_fallback +hrfi_flush_fallback: + SET_SCRATCH0(r13); + GET_PACA(r13); + std r9,PACA_EXRFI+EX_R9(r13) + std r10,PACA_EXRFI+EX_R10(r13) + std r11,PACA_EXRFI+EX_R11(r13) + std r12,PACA_EXRFI+EX_R12(r13) + std r8,PACA_EXRFI+EX_R13(r13) + mfctr r9 + ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) + ld r11,PACA_L1D_FLUSH_SETS(r13) + ld r12,PACA_L1D_FLUSH_CONGRUENCE(r13) + /* + * The load adresses are at staggered offsets within cachelines, + * which suits some pipelines better (on others it should not + * hurt. + */ + addi r12,r12,8 + mtctr r11 + DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ + /* XXX: Should an instruction synchronizing operation be done here? */ + +1: li r8,0 + .rept 8 /* 8-way set associative */ + ldx r11,r10,r8 + add r8,r8,r12 + .endr + addi r10,r10,128 /* 128 byte cache line */ + bdnz 1b + + mtctr r9 + ld r9,PACA_EXRFI+EX_R9(r13) + ld r10,PACA_EXRFI+EX_R10(r13) + ld r11,PACA_EXRFI+EX_R11(r13) + ld r12,PACA_EXRFI+EX_R12(r13) + ld r8,PACA_EXRFI+EX_R13(r13) + GET_SCRATCH0(r13); + hrfid + /* * Called from arch_local_irq_enable when an interrupt needs * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate @@ -731,7 +811,7 @@ slb_miss_user_pseries: mtspr SRR0,r12 mfspr r12,SRR1 /* and SRR1 */ mtspr SRR1,r10 - rfid + rfid /* No change for disabled code*/ b . /* prevent spec. execution */ #endif /* __DISABLED__ */ @@ -745,7 +825,7 @@ kvmppc_skip_interrupt: addi r13, r13, 4 mtspr SPRN_SRR0, r13 GET_SCRATCH0(r13) - rfid + RFI_TO_KERNEL b . kvmppc_skip_Hinterrupt: @@ -757,7 +837,7 @@ kvmppc_skip_Hinterrupt: addi r13, r13, 4 mtspr SPRN_HSRR0, r13 GET_SCRATCH0(r13) - hrfid + HRFI_TO_KERNEL b . #endif @@ -1119,7 +1199,7 @@ slb_miss_user_common: ld r11,PACA_EXGEN+EX_R11(r13) ld r12,PACA_EXGEN+EX_R12(r13) ld r13,PACA_EXGEN+EX_R13(r13) - rfid + rfid /* No change for disabled code */ b . slb_miss_fault: @@ -1456,7 +1536,7 @@ machine_check_handle_early: */ bl .machine_check_queue_event MACHINE_CHECK_HANDLER_WINDUP - rfid + RFI_TO_USER_OR_KERNEL 9: /* Deliver the machine check to host kernel in V mode. */ MACHINE_CHECK_HANDLER_WINDUP @@ -1492,6 +1572,9 @@ _GLOBAL(slb_miss_realmode) andi. r10,r12,MSR_RI /* check for unrecoverable exception */ beq- 2f + andi. r10,r12,MSR_PR /* check for exception from userspace */ + beq 1f + .machine push .machine "power4" mtcrf 0x80,r9 @@ -1504,7 +1587,23 @@ _GLOBAL(slb_miss_realmode) ld r11,PACA_EXSLB+EX_R11(r13) ld r12,PACA_EXSLB+EX_R12(r13) ld r13,PACA_EXSLB+EX_R13(r13) - rfid + RFI_TO_USER + b . /* prevent speculative execution */ + +1: +.machine push +.machine "power4" + mtcrf 0x80,r9 + mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */ +.machine pop + + RESTORE_PPR_PACA(PACA_EXSLB, r9) + ld r9,PACA_EXSLB+EX_R9(r13) + ld r10,PACA_EXSLB+EX_R10(r13) + ld r11,PACA_EXSLB+EX_R11(r13) + ld r12,PACA_EXSLB+EX_R12(r13) + ld r13,PACA_EXSLB+EX_R13(r13) + RFI_TO_KERNEL b . /* prevent speculative execution */ 2: mfspr r11,SPRN_SRR0 @@ -1513,7 +1612,7 @@ _GLOBAL(slb_miss_realmode) mtspr SPRN_SRR0,r10 ld r10,PACAKMSR(r13) mtspr SPRN_SRR1,r10 - rfid + RFI_TO_KERNEL b . unrecov_slb: @@ -1728,6 +1827,9 @@ _GLOBAL(do_stab_bolted) andi. r10,r12,MSR_RI beq- unrecov_slb + andi. r10,r12,MSR_PR /* check for exception from userspace */ + beq 1f /* returning to kernel */ + mtcrf 0x80,r9 /* restore CR */ mfmsr r10 @@ -1741,5 +1843,22 @@ _GLOBAL(do_stab_bolted) ld r11,PACA_EXSLB+EX_R11(r13) ld r12,PACA_EXSLB+EX_R12(r13) ld r13,PACA_EXSLB+EX_R13(r13) - rfid + RFI_TO_USER + b . /* prevent speculative execution */ + +1: + mtcrf 0x80,r9 /* restore CR */ + + mfmsr r10 + clrrdi r10,r10,2 + mtmsrd r10,1 + + mtspr SPRN_SRR0,r11 + mtspr SPRN_SRR1,r12 + ld r9,PACA_EXSLB+EX_R9(r13) + ld r10,PACA_EXSLB+EX_R10(r13) + ld r11,PACA_EXSLB+EX_R11(r13) + ld r12,PACA_EXSLB+EX_R12(r13) + ld r13,PACA_EXSLB+EX_R13(r13) + RFI_TO_KERNEL b . /* prevent speculative execution */ diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 3973991..c909cc7 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -726,3 +726,100 @@ void __init setup_per_cpu_areas(void) struct ppc_pci_io ppc_pci_io; EXPORT_SYMBOL(ppc_pci_io); #endif + +#ifdef CONFIG_PPC_BOOK3S_64 +static enum l1d_flush_type enabled_flush_types; +static void *l1d_flush_fallback_area; +static bool no_rfi_flush; +bool rfi_flush; + +static int __init handle_no_rfi_flush(char *p) +{ + pr_info("rfi-flush: disabled on command line."); + no_rfi_flush = true; + return 0; +} +early_param("no_rfi_flush", handle_no_rfi_flush); + +/* + * The RFI flush is not KPTI, but because users will see doco that says to use + * nopti we hijack that option here to also disable the RFI flush. + */ +static int __init handle_no_pti(char *p) +{ + pr_info("rfi-flush: disabling due to 'nopti' on command line.\n"); + handle_no_rfi_flush(NULL); + return 0; +} +early_param("nopti", handle_no_pti); + +static void do_nothing(void *unused) +{ + /* + * We don't need to do the flush explicitly, just enter+exit kernel is + * sufficient, the RFI exit handlers will do the right thing. + */ +} + +void rfi_flush_enable(bool enable) +{ + if (rfi_flush == enable) + return; + + if (enable) { + do_rfi_flush_fixups(enabled_flush_types); + on_each_cpu(do_nothing, NULL, 1); + } else + do_rfi_flush_fixups(L1D_FLUSH_NONE); + + rfi_flush = enable; +} + +void __init setup_rfi_flush(enum l1d_flush_type types, bool enable) +{ + if (types & L1D_FLUSH_FALLBACK) { + int cpu; + u64 l1d_size = ppc64_caches.dsize; + u64 limit = min(safe_stack_limit(), ppc64_rma_size); + + pr_info("rfi-flush: Using fallback displacement flush\n"); + + /* + * Align to L1d size, and size it at 2x L1d size, to + * catch possible hardware prefetch runoff. We don't + * have a recipe for load patterns to reliably avoid + * the prefetcher. + */ + l1d_flush_fallback_area = + __va(memblock_alloc_base(l1d_size * 2, l1d_size, limit)); + memset(l1d_flush_fallback_area, 0, l1d_size * 2); + + for_each_possible_cpu(cpu) { + /* + * The fallback flush is currently coded for 8-way + * associativity. Different associativity is possible, + * but it will be treated as 8-way and may not evict + * the lines as effectively. + * + * 128 byte lines are mandatory. + */ + u64 c = l1d_size / 8; + + paca[cpu].rfi_flush_fallback_area = l1d_flush_fallback_area; + paca[cpu].l1d_flush_congruence = c; + paca[cpu].l1d_flush_sets = c / 128; + } + } + + if (types & L1D_FLUSH_ORI) + pr_info("rfi-flush: Using ori type flush\n"); + + if (types & L1D_FLUSH_MTTRIG) + pr_info("rfi-flush: Using mttrig type flush\n"); + + enabled_flush_types = types; + + if (!no_rfi_flush) + rfi_flush_enable(enable); +} +#endif /* CONFIG_PPC_BOOK3S_64 */ diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index b4e6676..66025932 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include "cacheinfo.h" @@ -180,6 +182,44 @@ SYSFS_PMCSETUP(spurr, SPRN_SPURR); SYSFS_PMCSETUP(dscr, SPRN_DSCR); SYSFS_PMCSETUP(pir, SPRN_PIR); +#ifdef CONFIG_PPC_BOOK3S_64 +extern bool rfi_flush; +static ssize_t show_rfi_flush(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", rfi_flush ? 1 : 0); +} + +static ssize_t __used store_rfi_flush(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t count) +{ + int val; + int ret = 0; + + ret = sscanf(buf, "%d", &val); + if (ret != 1) + return -EINVAL; + + if (val == 1) + rfi_flush_enable(true); + else if (val == 0) + rfi_flush_enable(false); + else + return -EINVAL; + + return count; +} + +static DEVICE_ATTR(rfi_flush, 0600, + show_rfi_flush, store_rfi_flush); + +static void sysfs_create_rfi_flush(void) +{ + device_create_file(cpu_subsys.dev_root, &dev_attr_rfi_flush); +} +#endif /* CONFIG_PPC_BOOK3S_64 */ + /* Lets only enable read for phyp resources and enable write when needed with a separate function. @@ -683,6 +723,9 @@ static int __init topology_init(void) } #ifdef CONFIG_PPC64 sysfs_create_dscr_default(); +#ifdef CONFIG_PPC_BOOK3S + sysfs_create_rfi_flush(); +#endif #endif /* CONFIG_PPC64 */ return 0; diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index 1db6851..b542a80 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -72,6 +72,15 @@ SECTIONS /* Read-only data */ RODATA +#ifdef CONFIG_PPC64 + . = ALIGN(8); + __rfi_flush_fixup : AT(ADDR(__rfi_flush_fixup) - LOAD_OFFSET) { + __start___rfi_flush_fixup = .; + *(__rfi_flush_fixup) + __stop___rfi_flush_fixup = .; + } +#endif + EXCEPTION_TABLE(0) NOTES :kernel :notes diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 5f19b88..f85684f 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -54,7 +54,7 @@ _GLOBAL(kvmppc_hv_entry_trampoline) mtmsrd r0,1 /* clear RI in MSR */ mtsrr0 r5 mtsrr1 r6 - RFI + RFI_TO_KERNEL kvmppc_call_hv_entry: bl kvmppc_hv_entry @@ -144,7 +144,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) mtsrr1 r7 beqa 0x500 /* external interrupt (PPC970) */ beq cr1, 13f /* machine check */ - RFI + RFI_TO_KERNEL /* On POWER7, we have external interrupts set to use HSRR0/1 */ 11: mtspr SPRN_HSRR0, r8 @@ -731,8 +731,7 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ld r0, VCPU_GPR(R0)(r4) ld r4, VCPU_GPR(R4)(r4) - - hrfid + HRFI_TO_GUEST b . /****************************************************************************** diff --git a/arch/powerpc/kvm/book3s_rmhandlers.S b/arch/powerpc/kvm/book3s_rmhandlers.S index c3c5231..ccf92b8 100644 --- a/arch/powerpc/kvm/book3s_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_rmhandlers.S @@ -137,7 +137,7 @@ kvmppc_handler_skip_ins: GET_SCRATCH0(r13) /* And get back into the code */ - RFI + RFI_TO_GUEST #endif /* @@ -160,7 +160,7 @@ _GLOBAL(kvmppc_entry_trampoline) ori r5, r5, MSR_EE mtsrr0 r7 mtsrr1 r6 - RFI + RFI_TO_KERNEL #if defined(CONFIG_PPC_BOOK3S_32) #define STACK_LR INT_FRAME_SIZE+4 diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index 7a8a748..2aca8d8 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -20,6 +20,7 @@ #include #include #include +#include struct fixup_entry { @@ -113,6 +114,47 @@ void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) } } +#ifdef CONFIG_PPC_BOOK3S_64 +void do_rfi_flush_fixups(enum l1d_flush_type types) +{ + unsigned int instrs[3], *dest; + long *start, *end; + int i; + + start = PTRRELOC(&__start___rfi_flush_fixup), + end = PTRRELOC(&__stop___rfi_flush_fixup); + + instrs[0] = 0x60000000; /* nop */ + instrs[1] = 0x60000000; /* nop */ + instrs[2] = 0x60000000; /* nop */ + + if (types & L1D_FLUSH_FALLBACK) + /* b .+16 to fallback flush */ + instrs[0] = 0x48000010; + + i = 0; + if (types & L1D_FLUSH_ORI) { + instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */ + instrs[i++] = 0x63de0000; /* ori 30,30,0 L1d flush*/ + } + + if (types & L1D_FLUSH_MTTRIG) + instrs[i++] = 0x7c12dba6; /* mtspr TRIG2,r0 (SPR #882) */ + + for (i = 0; start < end; start++, i++) { + dest = (void *)start + *start; + + pr_devel("patching dest %lx\n", (unsigned long)dest); + + patch_instruction(dest, instrs[0]); + patch_instruction(dest + 1, instrs[1]); + patch_instruction(dest + 2, instrs[2]); + } + + printk(KERN_DEBUG "rfi-flush: patched %d locations\n", i); +} +#endif /* CONFIG_PPC_BOOK3S_64 */ + void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) { long *start, *end; diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index e8e9f54..27dc1a1 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -34,11 +34,60 @@ #include #include #include +#include #include "powernv.h" +static void pnv_setup_rfi_flush(void) +{ + struct device_node *np, *fw_features; + enum l1d_flush_type type; + int enable; + + /* Default to fallback in case fw-features are not available */ + type = L1D_FLUSH_FALLBACK; + enable = 1; + + np = of_find_node_by_name(NULL, "ibm,opal"); + fw_features = of_get_child_by_name(np, "fw-features"); + of_node_put(np); + + if (fw_features) { + np = of_get_child_by_name(fw_features, "inst-l1d-flush-trig2"); + if (np && of_property_read_bool(np, "enabled")) + type = L1D_FLUSH_MTTRIG; + + of_node_put(np); + + np = of_get_child_by_name(fw_features, "inst-l1d-flush-ori30,30,0"); + if (np && of_property_read_bool(np, "enabled")) + type = L1D_FLUSH_ORI; + + of_node_put(np); + + /* Enable unless firmware says NOT to */ + enable = 2; + np = of_get_child_by_name(fw_features, "needs-l1d-flush-msr-hv-1-to-0"); + if (np && of_property_read_bool(np, "disabled")) + enable--; + + of_node_put(np); + + np = of_get_child_by_name(fw_features, "needs-l1d-flush-msr-pr-0-to-1"); + if (np && of_property_read_bool(np, "disabled")) + enable--; + + of_node_put(np); + of_node_put(fw_features); + } + + setup_rfi_flush(type, enable > 0); +} + static void __init pnv_setup_arch(void) { + pnv_setup_rfi_flush(); + /* Initialize SMP */ pnv_smp_init(); diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index c1f1908..7cd0e13 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -468,6 +468,38 @@ static long pseries_little_endian_exceptions(void) } #endif +static void pSeries_setup_rfi_flush(void) +{ + unsigned long character, behaviour, rc; + enum l1d_flush_type types; + bool enable; + + /* Enable by default */ + enable = true; + + rc = plpar_get_cpu_characteristics(&character, &behaviour); + if (rc == H_SUCCESS) { + types = L1D_FLUSH_NONE; + + if (character & H_GET_CPU_CHAR_CHAR_MTTRIG2_L1_FLUSH) + types |= L1D_FLUSH_MTTRIG; + if (character & H_GET_CPU_CHAR_CHAR_ORI30_L1_FLUSH) + types |= L1D_FLUSH_ORI; + + /* Use fallback if nothing set in hcall */ + if (types == L1D_FLUSH_NONE) + types = L1D_FLUSH_FALLBACK; + + if (!(behaviour & H_GET_CPU_CHAR_BEHAV_L1_FLUSH_LOW_PRIV)) + enable = false; + } else { + /* Default to fallback if case hcall is not available */ + types = L1D_FLUSH_FALLBACK; + } + + setup_rfi_flush(types, enable); +} + static void __init pSeries_setup_arch(void) { panic_timeout = 10; @@ -484,6 +516,8 @@ static void __init pSeries_setup_arch(void) fwnmi_init(); + pSeries_setup_rfi_flush(); + /* By default, only probe PCI (can be overriden by rtas_pci) */ pci_add_flags(PCI_PROBE_ONLY); -- 2.7.4