Commit f84a52ee authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'x86_bugs_for_6.7_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 hw mitigation updates from Borislav Petkov:

 - A bunch of improvements, cleanups and fixlets to the SRSO mitigation
   machinery and other, general cleanups to the hw mitigations code, by
   Josh Poimboeuf

 - Improve the return thunk detection by objtool as it is absolutely
   important that the default return thunk is not used after returns
   have been patched. Future work to detect and report this better is
   pending

 - Other misc cleanups and fixes

* tag 'x86_bugs_for_6.7_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (21 commits)
  x86/retpoline: Document some thunk handling aspects
  x86/retpoline: Make sure there are no unconverted return thunks due to KCSAN
  x86/callthunks: Delete unused "struct thunk_desc"
  x86/vdso: Run objtool on vdso32-setup.o
  objtool: Fix return thunk patching in retpolines
  x86/srso: Remove unnecessary semicolon
  x86/pti: Fix kernel warnings for pti= and nopti cmdline options
  x86/calldepth: Rename __x86_return_skl() to call_depth_return_thunk()
  x86/nospec: Refactor UNTRAIN_RET[_*]
  x86/rethunk: Use SYM_CODE_START[_LOCAL]_NOALIGN macros
  x86/srso: Disentangle rethunk-dependent options
  x86/srso: Move retbleed IBPB check into existing 'has_microcode' code block
  x86/bugs: Remove default case for fully switched enums
  x86/srso: Remove 'pred_cmd' label
  x86/srso: Unexport untraining functions
  x86/srso: Improve i-cache locality for alias mitigation
  x86/srso: Fix unret validation dependencies
  x86/srso: Fix vulnerability reporting for missing microcode
  x86/srso: Print mitigation for retbleed IBPB case
  x86/srso: Print actual mitigation if requested mitigation isn't possible
  ...
parents 01ae815c 9d9c22cc

Documentation/admin-guide/hw-vuln/srso.rst

+17 −7
Original line number Diff line number Diff line
@@ -46,12 +46,22 @@ The possible values in this file are: 


   The processor is not vulnerable



 * 'Vulnerable: no microcode':

* 'Vulnerable':



   The processor is vulnerable and no mitigations have been applied.



 * 'Vulnerable: No microcode':



   The processor is vulnerable, no microcode extending IBPB

   functionality to address the vulnerability has been applied.



 * 'Mitigation: microcode':

 * 'Vulnerable: Safe RET, no microcode':



   The "Safe RET" mitigation (see below) has been applied to protect the

   kernel, but the IBPB-extending microcode has not been applied.  User

   space tasks may still be vulnerable.



 * 'Vulnerable: Microcode, no safe RET':



   Extended IBPB functionality microcode patch has been applied. It does

   not address User->Kernel and Guest->Host transitions protection but it
@@ -72,11 +82,11 @@ The possible values in this file are: 


   (spec_rstack_overflow=microcode)



 * 'Mitigation: safe RET':

 * 'Mitigation: Safe RET':



   Software-only mitigation. It complements the extended IBPB microcode

   patch functionality by addressing User->Kernel and Guest->Host

   transitions protection.

   Combined microcode/software mitigation. It complements the

   extended IBPB microcode patch functionality by addressing

   User->Kernel and Guest->Host transitions protection.



   Selected by default or by spec_rstack_overflow=safe-ret
@@ -129,7 +139,7 @@ an indrect branch prediction barrier after having applied the required 
microcode patch for one's system. This mitigation comes also at

a performance cost.



Mitigation: safe RET

Mitigation: Safe RET

--------------------



The mitigation works by ensuring all RET instructions speculate to

arch/x86/entry/vdso/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ vdso_img-$(VDSOX32-y) += x32 
vdso_img-$(VDSO32-y)		+= 32



obj-$(VDSO32-y)				 += vdso32-setup.o

OBJECT_FILES_NON_STANDARD_vdso32-setup.o := n



vobjs := $(foreach F,$(vobjs-y),$(obj)/$F)

vobjs32 := $(foreach F,$(vobjs32-y),$(obj)/$F)

arch/x86/include/asm/nospec-branch.h

+29 −38
Original line number Diff line number Diff line
@@ -271,7 +271,7 @@ 
.Lskip_rsb_\@:

.endm



#ifdef CONFIG_CPU_UNRET_ENTRY

#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO)

#define CALL_UNTRAIN_RET	"call entry_untrain_ret"

#else

#define CALL_UNTRAIN_RET	""
@@ -288,38 +288,24 @@ 
 * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point

 * where we have a stack but before any RET instruction.

 */

.macro UNTRAIN_RET

#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \

	defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)

.macro __UNTRAIN_RET ibpb_feature, call_depth_insns

#if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY)

	VALIDATE_UNRET_END

	ALTERNATIVE_3 "",						\

		      CALL_UNTRAIN_RET, X86_FEATURE_UNRET,		\

		      "call entry_ibpb", X86_FEATURE_ENTRY_IBPB,	\

		      __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH

		      "call entry_ibpb", \ibpb_feature,			\

		     __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH

#endif

.endm



.macro UNTRAIN_RET_VM

#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \

	defined(CONFIG_CALL_DEPTH_TRACKING) || defined(CONFIG_CPU_SRSO)

	VALIDATE_UNRET_END

	ALTERNATIVE_3 "",						\

		      CALL_UNTRAIN_RET, X86_FEATURE_UNRET,		\

		      "call entry_ibpb", X86_FEATURE_IBPB_ON_VMEXIT,	\

		      __stringify(RESET_CALL_DEPTH), X86_FEATURE_CALL_DEPTH

#endif

.endm

#define UNTRAIN_RET \

	__UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH)



.macro UNTRAIN_RET_FROM_CALL

#if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY) || \

	defined(CONFIG_CALL_DEPTH_TRACKING)

	VALIDATE_UNRET_END

	ALTERNATIVE_3 "",						\

		      CALL_UNTRAIN_RET, X86_FEATURE_UNRET,		\

		      "call entry_ibpb", X86_FEATURE_ENTRY_IBPB,	\

		      __stringify(RESET_CALL_DEPTH_FROM_CALL), X86_FEATURE_CALL_DEPTH

#endif

.endm

#define UNTRAIN_RET_VM \

	__UNTRAIN_RET X86_FEATURE_IBPB_ON_VMEXIT, __stringify(RESET_CALL_DEPTH)



#define UNTRAIN_RET_FROM_CALL \

	__UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL)





.macro CALL_DEPTH_ACCOUNT
@@ -348,13 +334,23 @@ extern void __x86_return_thunk(void); 
static inline void __x86_return_thunk(void) {}

#endif



#ifdef CONFIG_CPU_UNRET_ENTRY

extern void retbleed_return_thunk(void);

#else

static inline void retbleed_return_thunk(void) {}

#endif



#ifdef CONFIG_CPU_SRSO

extern void srso_return_thunk(void);

extern void srso_alias_return_thunk(void);

#else

static inline void srso_return_thunk(void) {}

static inline void srso_alias_return_thunk(void) {}

#endif



extern void retbleed_untrain_ret(void);

extern void srso_untrain_ret(void);

extern void srso_alias_untrain_ret(void);

extern void retbleed_return_thunk(void);

extern void srso_return_thunk(void);

extern void srso_alias_return_thunk(void);



extern void entry_untrain_ret(void);

extern void entry_ibpb(void);
@@ -362,12 +358,7 @@ extern void entry_ibpb(void); 
extern void (*x86_return_thunk)(void);



#ifdef CONFIG_CALL_DEPTH_TRACKING

extern void __x86_return_skl(void);



static inline void x86_set_skl_return_thunk(void)

{

	x86_return_thunk = &__x86_return_skl;

}

extern void call_depth_return_thunk(void);



#define CALL_DEPTH_ACCOUNT					\

	ALTERNATIVE("",						\
@@ -380,12 +371,12 @@ DECLARE_PER_CPU(u64, __x86_ret_count); 
DECLARE_PER_CPU(u64, __x86_stuffs_count);

DECLARE_PER_CPU(u64, __x86_ctxsw_count);

#endif

#else

static inline void x86_set_skl_return_thunk(void) {}

#else /* !CONFIG_CALL_DEPTH_TRACKING */



static inline void call_depth_return_thunk(void) {}

#define CALL_DEPTH_ACCOUNT ""



#endif

#endif /* CONFIG_CALL_DEPTH_TRACKING */



#ifdef CONFIG_RETPOLINE

arch/x86/kernel/callthunks.c

+0 −5
Original line number Diff line number Diff line
@@ -48,11 +48,6 @@ EXPORT_SYMBOL_GPL(__x86_call_count); 


extern s32 __call_sites[], __call_sites_end[];



struct thunk_desc {

	void		*template;

	unsigned int	template_size;

};



struct core_text {

	unsigned long	base;

	unsigned long	end;

arch/x86/kernel/cpu/bugs.c

+50 −45
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ EXPORT_SYMBOL_GPL(x86_pred_cmd); 


static DEFINE_MUTEX(spec_ctrl_mutex);



void (*x86_return_thunk)(void) __ro_after_init = &__x86_return_thunk;

void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;



/* Update SPEC_CTRL MSR and its cached copy unconditionally */

static void update_spec_ctrl(u64 val)
@@ -717,7 +717,7 @@ void update_gds_msr(void) 
	case GDS_MITIGATION_UCODE_NEEDED:

	case GDS_MITIGATION_HYPERVISOR:

		return;

	};

	}



	wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
@@ -1019,7 +1019,6 @@ static void __init retbleed_select_mitigation(void) 


do_cmd_auto:

	case RETBLEED_CMD_AUTO:

	default:

		if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||

		    boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {

			if (IS_ENABLED(CONFIG_CPU_UNRET_ENTRY))
@@ -1042,7 +1041,6 @@ static void __init retbleed_select_mitigation(void) 
		setup_force_cpu_cap(X86_FEATURE_RETHUNK);

		setup_force_cpu_cap(X86_FEATURE_UNRET);



		if (IS_ENABLED(CONFIG_RETHUNK))

		x86_return_thunk = retbleed_return_thunk;



		if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
@@ -1061,7 +1059,8 @@ static void __init retbleed_select_mitigation(void) 
	case RETBLEED_MITIGATION_STUFF:

		setup_force_cpu_cap(X86_FEATURE_RETHUNK);

		setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);

		x86_set_skl_return_thunk();



		x86_return_thunk = call_depth_return_thunk;

		break;



	default:
@@ -1290,6 +1289,8 @@ spectre_v2_user_select_mitigation(void) 


		spectre_v2_user_ibpb = mode;

		switch (cmd) {

		case SPECTRE_V2_USER_CMD_NONE:

			break;

		case SPECTRE_V2_USER_CMD_FORCE:

		case SPECTRE_V2_USER_CMD_PRCTL_IBPB:

		case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
@@ -1301,8 +1302,6 @@ spectre_v2_user_select_mitigation(void) 
		case SPECTRE_V2_USER_CMD_SECCOMP:

			static_branch_enable(&switch_mm_cond_ibpb);

			break;

		default:

			break;

		}



		pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
@@ -2160,6 +2159,10 @@ static int l1d_flush_prctl_get(struct task_struct *task) 
static int ssb_prctl_get(struct task_struct *task)

{

	switch (ssb_mode) {

	case SPEC_STORE_BYPASS_NONE:

		if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))

			return PR_SPEC_ENABLE;

		return PR_SPEC_NOT_AFFECTED;

	case SPEC_STORE_BYPASS_DISABLE:

		return PR_SPEC_DISABLE;

	case SPEC_STORE_BYPASS_SECCOMP:
@@ -2171,11 +2174,8 @@ static int ssb_prctl_get(struct task_struct *task) 
		if (task_spec_ssb_disable(task))

			return PR_SPEC_PRCTL | PR_SPEC_DISABLE;

		return PR_SPEC_PRCTL | PR_SPEC_ENABLE;

	default:

		if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))

			return PR_SPEC_ENABLE;

		return PR_SPEC_NOT_AFFECTED;

	}

	BUG();

}



static int ib_prctl_get(struct task_struct *task)
@@ -2353,6 +2353,8 @@ early_param("l1tf", l1tf_cmdline); 


enum srso_mitigation {

	SRSO_MITIGATION_NONE,

	SRSO_MITIGATION_UCODE_NEEDED,

	SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED,

	SRSO_MITIGATION_MICROCODE,

	SRSO_MITIGATION_SAFE_RET,

	SRSO_MITIGATION_IBPB,
@@ -2369,8 +2371,10 @@ enum srso_mitigation_cmd { 


static const char * const srso_strings[] = {

	[SRSO_MITIGATION_NONE]			= "Vulnerable",

	[SRSO_MITIGATION_MICROCODE]      = "Mitigation: microcode",

	[SRSO_MITIGATION_SAFE_RET]	 = "Mitigation: safe RET",

	[SRSO_MITIGATION_UCODE_NEEDED]		= "Vulnerable: No microcode",

	[SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED]	= "Vulnerable: Safe RET, no microcode",

	[SRSO_MITIGATION_MICROCODE]		= "Vulnerable: Microcode, no safe RET",

	[SRSO_MITIGATION_SAFE_RET]		= "Mitigation: Safe RET",

	[SRSO_MITIGATION_IBPB]			= "Mitigation: IBPB",

	[SRSO_MITIGATION_IBPB_ON_VMEXIT]	= "Mitigation: IBPB on VMEXIT only"

};
@@ -2406,34 +2410,44 @@ static void __init srso_select_mitigation(void) 
{

	bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE);



	if (!boot_cpu_has_bug(X86_BUG_SRSO) || cpu_mitigations_off())

		goto pred_cmd;

	if (cpu_mitigations_off())

		return;



	if (!has_microcode) {

		pr_warn("IBPB-extending microcode not applied!\n");

		pr_warn(SRSO_NOTICE);

	} else {

	if (!boot_cpu_has_bug(X86_BUG_SRSO)) {

		if (boot_cpu_has(X86_FEATURE_SBPB))

			x86_pred_cmd = PRED_CMD_SBPB;

		return;

	}



	if (has_microcode) {

		/*

		 * Zen1/2 with SMT off aren't vulnerable after the right

		 * IBPB microcode has been applied.

		 *

		 * Zen1/2 don't have SBPB, no need to try to enable it here.

		 */

		if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {

			setup_force_cpu_cap(X86_FEATURE_SRSO_NO);

			return;

		}

	}



		if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {

		if (has_microcode) {

			pr_err("Retbleed IBPB mitigation enabled, using same for SRSO\n");

			srso_mitigation = SRSO_MITIGATION_IBPB;

			goto pred_cmd;

			goto out;

		}

	} else {

		pr_warn("IBPB-extending microcode not applied!\n");

		pr_warn(SRSO_NOTICE);



		/* may be overwritten by SRSO_CMD_SAFE_RET below */

		srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED;

	}



	switch (srso_cmd) {

	case SRSO_CMD_OFF:

		goto pred_cmd;

		if (boot_cpu_has(X86_FEATURE_SBPB))

			x86_pred_cmd = PRED_CMD_SBPB;

		return;



	case SRSO_CMD_MICROCODE:

		if (has_microcode) {
@@ -2458,10 +2472,12 @@ static void __init srso_select_mitigation(void) 
				setup_force_cpu_cap(X86_FEATURE_SRSO);

				x86_return_thunk = srso_return_thunk;

			}

			if (has_microcode)

				srso_mitigation = SRSO_MITIGATION_SAFE_RET;

			else

				srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED;

		} else {

			pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");

			goto pred_cmd;

		}

		break;
@@ -2473,7 +2489,6 @@ static void __init srso_select_mitigation(void) 
			}

		} else {

			pr_err("WARNING: kernel not compiled with CPU_IBPB_ENTRY.\n");

			goto pred_cmd;

		}

		break;
@@ -2485,20 +2500,12 @@ static void __init srso_select_mitigation(void) 
			}

		} else {

			pr_err("WARNING: kernel not compiled with CPU_SRSO.\n");

			goto pred_cmd;

                }

		break;



	default:

		break;

	}



	pr_info("%s%s\n", srso_strings[srso_mitigation], (has_microcode ? "" : ", no microcode"));



pred_cmd:

	if ((boot_cpu_has(X86_FEATURE_SRSO_NO) || srso_cmd == SRSO_CMD_OFF) &&

	     boot_cpu_has(X86_FEATURE_SBPB))

		x86_pred_cmd = PRED_CMD_SBPB;

out:

	pr_info("%s\n", srso_strings[srso_mitigation]);

}



#undef pr_fmt
@@ -2704,9 +2711,7 @@ static ssize_t srso_show_state(char *buf) 
	if (boot_cpu_has(X86_FEATURE_SRSO_NO))

		return sysfs_emit(buf, "Mitigation: SMT disabled\n");



	return sysfs_emit(buf, "%s%s\n",

			  srso_strings[srso_mitigation],

			  boot_cpu_has(X86_FEATURE_IBPB_BRTYPE) ? "" : ", no microcode");

	return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]);

}



static ssize_t gds_show_state(char *buf)