Commit ed6b0b45 authored by David S. Miller's avatar David S. Miller
Browse files

[SPARC64]: SUN4V memory exception trap handlers. 



Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent 618e9ed9

arch/sparc64/kernel/sun4v_tlb_miss.S

+170 −0
Original line number Diff line number Diff line
@@ -237,6 +237,167 @@ sun4v_tsb_miss_common: 
	ba,pt	%xcc, tsb_miss_page_table_walk

	 add	%g1, %g2, %g1



	/* Instruction Access Exception, tl0. */

sun4v_iacc:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_I_TYPE_OFFSET], %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_I_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_I_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	sun4v_insn_access_exception

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Instruction Access Exception, tl1. */

sun4v_iacc_tl1:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_I_TYPE_OFFSET], %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_I_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_I_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5

	ba,pt	%xcc, etraptl1

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	sun4v_insn_access_exception_tl1

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Data Access Exception, tl0. */

sun4v_dacc:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_TYPE_OFFSET], %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	sun4v_data_access_exception

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Data Access Exception, tl1. */

sun4v_dacc_tl1:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_TYPE_OFFSET], %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5

	ba,pt	%xcc, etraptl1

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	sun4v_data_access_exception_tl1

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Memory Address Unaligned.  */

sun4v_mna:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	mov	HV_FAULT_TYPE_UNALIGNED, %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5



	/* Window fixup? */

	rdpr	%tl, %g2

	cmp	%g2, 1

	bgu,pn	%icc, winfix_mna

	 rdpr	%tpc, %g3



	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	sun4v_mna

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Privileged Action.  */

sun4v_privact:

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	call	do_privact

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Unaligned ldd float, tl0. */

sun4v_lddfmna:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_TYPE_OFFSET], %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	handle_lddfmna

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



	/* Unaligned std float, tl0. */

sun4v_stdfmna:

	mov	SCRATCHPAD_CPUID, %g1

	ldxa	[%g1] ASI_SCRATCHPAD, %g3

	sethi	%hi(trap_block), %g2

	or	%g2, %lo(trap_block), %g2

	sllx	%g3, TRAP_BLOCK_SZ_SHIFT, %g3

	add	%g2, %g3, %g2

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_TYPE_OFFSET], %g3

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_ADDR_OFFSET], %g4

	ldx	[%g2 + TRAP_PER_CPU_FAULT_INFO + HV_FAULT_D_CTX_OFFSET], %g5

	sllx	%g3, 16, %g3

	or	%g5, %g3, %g5

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	mov	%l4, %o1

	mov	%l5, %o2

	call	handle_stdfmna

	 add	%sp, PTREGS_OFF, %o0

	ba,a,pt	%xcc, rtrap_clr_l6



#define BRANCH_ALWAYS	0x10680000

#define NOP		0x01000000
@@ -265,6 +426,15 @@ sun4v_patch_tlb_handlers: 
	SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)

	SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)

	SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)

	SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)

	SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)

	SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)

	SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)

	SUN4V_DO_PATCH(tl0_mna, sun4v_mna)

	SUN4V_DO_PATCH(tl1_mna, sun4v_mna)

	SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)

	SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)

	SUN4V_DO_PATCH(tl0_privact, sun4v_privact)

	retl

	 nop

	.size	sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers

arch/sparc64/kernel/traps.c

+101 −7
Original line number Diff line number Diff line
@@ -180,6 +180,45 @@ void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr 
	spitfire_insn_access_exception(regs, sfsr, sfar);

}



void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)

{

	unsigned short type = (type_ctx >> 16);

	unsigned short ctx  = (type_ctx & 0xffff);

	siginfo_t info;



	if (notify_die(DIE_TRAP, "instruction access exception", regs,

		       0, 0x8, SIGTRAP) == NOTIFY_STOP)

		return;



	if (regs->tstate & TSTATE_PRIV) {

		printk("sun4v_insn_access_exception: ADDR[%016lx] "

		       "CTX[%04x] TYPE[%04x], going.\n",

		       addr, ctx, type);

		die_if_kernel("Iax", regs);

	}



	if (test_thread_flag(TIF_32BIT)) {

		regs->tpc &= 0xffffffff;

		regs->tnpc &= 0xffffffff;

	}

	info.si_signo = SIGSEGV;

	info.si_errno = 0;

	info.si_code = SEGV_MAPERR;

	info.si_addr = (void __user *) addr;

	info.si_trapno = 0;

	force_sig_info(SIGSEGV, &info, current);

}



void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)

{

	if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,

		       0, 0x8, SIGTRAP) == NOTIFY_STOP)

		return;



	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));

	sun4v_insn_access_exception(regs, addr, type_ctx);

}



void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)

{

	siginfo_t info;
@@ -228,6 +267,45 @@ void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr 
	spitfire_data_access_exception(regs, sfsr, sfar);

}



void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)

{

	unsigned short type = (type_ctx >> 16);

	unsigned short ctx  = (type_ctx & 0xffff);

	siginfo_t info;



	if (notify_die(DIE_TRAP, "data access exception", regs,

		       0, 0x8, SIGTRAP) == NOTIFY_STOP)

		return;



	if (regs->tstate & TSTATE_PRIV) {

		printk("sun4v_data_access_exception: ADDR[%016lx] "

		       "CTX[%04x] TYPE[%04x], going.\n",

		       addr, ctx, type);

		die_if_kernel("Iax", regs);

	}



	if (test_thread_flag(TIF_32BIT)) {

		regs->tpc &= 0xffffffff;

		regs->tnpc &= 0xffffffff;

	}

	info.si_signo = SIGSEGV;

	info.si_errno = 0;

	info.si_code = SEGV_MAPERR;

	info.si_addr = (void __user *) addr;

	info.si_trapno = 0;

	force_sig_info(SIGSEGV, &info, current);

}



void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)

{

	if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,

		       0, 0x8, SIGTRAP) == NOTIFY_STOP)

		return;



	dump_tl1_traplog((struct tl1_traplog *)(regs + 1));

	sun4v_data_access_exception(regs, addr, type_ctx);

}



#ifdef CONFIG_PCI

/* This is really pathetic... */

extern volatile int pci_poke_in_progress;
@@ -2150,6 +2228,8 @@ void do_illegal_instruction(struct pt_regs *regs) 
	force_sig_info(SIGILL, &info, current);

}



extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn);



void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)

{

	siginfo_t info;
@@ -2159,13 +2239,7 @@ void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned lo 
		return;



	if (regs->tstate & TSTATE_PRIV) {

		extern void kernel_unaligned_trap(struct pt_regs *regs,

						  unsigned int insn, 

						  unsigned long sfar,

						  unsigned long sfsr);



		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc),

				      sfar, sfsr);

		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));

		return;

	}

	info.si_signo = SIGBUS;
@@ -2176,6 +2250,26 @@ void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned lo 
	force_sig_info(SIGBUS, &info, current);

}



void sun4v_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)

{

	siginfo_t info;



	if (notify_die(DIE_TRAP, "memory address unaligned", regs,

		       0, 0x34, SIGSEGV) == NOTIFY_STOP)

		return;



	if (regs->tstate & TSTATE_PRIV) {

		kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));

		return;

	}

	info.si_signo = SIGBUS;

	info.si_errno = 0;

	info.si_code = BUS_ADRALN;

	info.si_addr = (void __user *) addr;

	info.si_trapno = 0;

	force_sig_info(SIGBUS, &info, current);

}



void do_privop(struct pt_regs *regs)

{

	siginfo_t info;

arch/sparc64/kernel/unaligned.c

+34 −11
Original line number Diff line number Diff line
@@ -277,7 +277,7 @@ static void kernel_mna_trap_fault(void) 
	regs->tstate |= (ASI_AIUS << 24UL);

}



asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn, unsigned long sfar, unsigned long sfsr)

asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn)

{

	enum direction dir = decode_direction(insn);

	int size = decode_access_size(insn);
@@ -405,6 +405,9 @@ extern void do_privact(struct pt_regs *regs); 
extern void spitfire_data_access_exception(struct pt_regs *regs,

					   unsigned long sfsr,

					   unsigned long sfar);

extern void sun4v_data_access_exception(struct pt_regs *regs,

					unsigned long addr,

					unsigned long type_ctx);



int handle_ldf_stq(u32 insn, struct pt_regs *regs)

{
@@ -447,6 +450,9 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) 
				break;

			}

		default:

			if (tlb_type == hypervisor)

				sun4v_data_access_exception(regs, addr, 0);

			else

				spitfire_data_access_exception(regs, 0, addr);

			return 1;

		}
@@ -454,6 +460,9 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) 
		    __put_user ((u32)first, (u32 __user *)(addr + 4)) ||

		    __put_user (second >> 32, (u32 __user *)(addr + 8)) ||

		    __put_user ((u32)second, (u32 __user *)(addr + 12))) {

			if (tlb_type == hypervisor)

				sun4v_data_access_exception(regs, addr, 0);

			else

				spitfire_data_access_exception(regs, 0, addr);

		    	return 1;

		}
@@ -467,6 +476,9 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) 
			do_privact(regs);

			return 1;

		} else if (asi > ASI_SNFL) {

			if (tlb_type == hypervisor)

				sun4v_data_access_exception(regs, addr, 0);

			else

				spitfire_data_access_exception(regs, 0, addr);

			return 1;

		}
@@ -484,6 +496,9 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) 
				err |= __get_user (data[i], (u32 __user *)(addr + 4*i));

		}

		if (err && !(asi & 0x2 /* NF */)) {

			if (tlb_type == hypervisor)

				sun4v_data_access_exception(regs, addr, 0);

			else

				spitfire_data_access_exception(regs, 0, addr);

			return 1;

		}
@@ -548,7 +563,7 @@ void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr 
	u32 insn;

	u32 first, second;

	u64 value;

	u8 asi, freg;

	u8 freg;

	int flag;

	struct fpustate *f = FPUSTATE;
@@ -557,7 +572,7 @@ void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr 
	if (test_thread_flag(TIF_32BIT))

		pc = (u32)pc;

	if (get_user(insn, (u32 __user *) pc) != -EFAULT) {

		asi = sfsr >> 16;

		int asi = decode_asi(insn, regs);

		if ((asi > ASI_SNFL) ||

		    (asi < ASI_P))

			goto daex;
@@ -587,7 +602,11 @@ void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr 
		*(u64 *)(f->regs + freg) = value;

		current_thread_info()->fpsaved[0] |= flag;

	} else {

daex:		spitfire_data_access_exception(regs, sfsr, sfar);

daex:

		if (tlb_type == hypervisor)

			sun4v_data_access_exception(regs, sfar, sfsr);

		else

			spitfire_data_access_exception(regs, sfsr, sfar);

		return;

	}

	advance(regs);
@@ -600,7 +619,7 @@ void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr 
	unsigned long tstate = regs->tstate;

	u32 insn;

	u64 value;

	u8 asi, freg;

	u8 freg;

	int flag;

	struct fpustate *f = FPUSTATE;
@@ -609,8 +628,8 @@ void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr 
	if (test_thread_flag(TIF_32BIT))

		pc = (u32)pc;

	if (get_user(insn, (u32 __user *) pc) != -EFAULT) {

		int asi = decode_asi(insn, regs);

		freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);

		asi = sfsr >> 16;

		value = 0;

		flag = (freg < 32) ? FPRS_DL : FPRS_DU;

		if ((asi > ASI_SNFL) ||
@@ -631,7 +650,11 @@ void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr 
		    __put_user ((u32)value, (u32 __user *)(sfar + 4)))

			goto daex;

	} else {

daex:		spitfire_data_access_exception(regs, sfsr, sfar);

daex:

		if (tlb_type == hypervisor)

			sun4v_data_access_exception(regs, sfar, sfsr);

		else

			spitfire_data_access_exception(regs, sfsr, sfar);

		return;

	}

	advance(regs);

arch/sparc64/kernel/winfixup.S

+20 −6
Original line number Diff line number Diff line
@@ -109,16 +109,23 @@ winfix_mna: 
	done



fill_fixup_mna:

	TRAP_LOAD_THREAD_REG(%g6, %g1)

	rdpr	%tstate, %g1

	and	%g1, TSTATE_CWP, %g1

	wrpr	%g1, %cwp

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	mov	%l4, %o2

	mov	%l5, %o1

	call	mem_address_unaligned

	sethi	%hi(tlb_type), %g1

	mov	%l4, %o1

	lduw	[%g1 + %lo(tlb_type)], %g1

	mov	%l5, %o2

	cmp	%g1, 3

	bne,pt	%icc, 1f

	 add	%sp, PTREGS_OFF, %o0

	call	sun4v_mna

	 nop

	ba,a,pt	%xcc, rtrap_clr_l6

1:	call	mem_address_unaligned

	 nop

	ba,a,pt	%xcc, rtrap_clr_l6



winfix_dax:
@@ -128,14 +135,21 @@ winfix_dax: 
	done



fill_fixup_dax:

	TRAP_LOAD_THREAD_REG(%g6, %g1)

	rdpr	%tstate, %g1

	and	%g1, TSTATE_CWP, %g1

	wrpr	%g1, %cwp

	ba,pt	%xcc, etrap

	 rd	%pc, %g7

	sethi	%hi(tlb_type), %g1

	mov	%l4, %o1

	lduw	[%g1 + %lo(tlb_type)], %g1

	mov	%l5, %o2

	call	spitfire_data_access_exception

	cmp	%g1, 3

	bne,pt	%icc, 1f

	 add	%sp, PTREGS_OFF, %o0

	call	sun4v_data_access_exception

	 nop

	ba,a,pt	%xcc, rtrap_clr_l6

1:	call	spitfire_data_access_exception

	 nop

	ba,a,pt	%xcc, rtrap_clr_l6