arch/s390/kernel/fpu.c

/*
 * In-kernel vector facility support functions
 *
 * Copyright IBM Corp. 2015
 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
 */
#include <linux/kernel.h>
#include <linux/cpu.h>
#include <linux/sched.h>
#include <asm/fpu/types.h>
#include <asm/fpu/api.h>

/*
 * Per-CPU variable to maintain FPU register ranges that are in use
 * by the kernel.
 */
static DEFINE_PER_CPU(u32, kernel_fpu_state);

#define KERNEL_FPU_STATE_MASK	(KERNEL_FPU_MASK|KERNEL_FPC)


void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
{
	if (!__this_cpu_read(kernel_fpu_state)) {
		/*
		 * Save user space FPU state and register contents.  Multiple
		 * calls because of interruptions do not matter and return
		 * immediately.  This also sets CIF_FPU to lazy restore FP/VX
		 * register contents when returning to user space.
		 */
		save_fpu_regs();
	}

	/* Update flags to use the vector facility for KERNEL_FPR */
	if (MACHINE_HAS_VX && (state->mask & KERNEL_FPR)) {
		flags |= KERNEL_VXR_LOW | KERNEL_FPC;
		flags &= ~KERNEL_FPR;
	}

	/* Save and update current kernel VX state */
	state->mask = __this_cpu_read(kernel_fpu_state);
	__this_cpu_or(kernel_fpu_state, flags & KERNEL_FPU_STATE_MASK);

	/*
	 * If this is the first call to __kernel_fpu_begin(), no additional
	 * work is required.
	 */
	if (!(state->mask & KERNEL_FPU_STATE_MASK))
		return;

	/*
	 * If KERNEL_FPR is still set, the vector facility is not available
	 * and, thus, save floating-point control and registers only.
	 */
	if (state->mask & KERNEL_FPR) {
		asm volatile("stfpc %0" : "=Q" (state->fpc));
		asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
		asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
		asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
		asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
		asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
		asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
		asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
		asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
		asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
		asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
		asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
		asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
		asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
		asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
		asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
		asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
		return;
	}

	/*
	 * If this is a nested call to __kernel_fpu_begin(), check the saved
	 * state mask to save and later restore the vector registers that
	 * are already in use.	Let's start with checking floating-point
	 * controls.
	 */
	if (state->mask & KERNEL_FPC)
		asm volatile("stfpc %0" : "=m" (state->fpc));

	/* Test and save vector registers */
	asm volatile (
		/*
		 * Test if any vector register must be saved and, if so,
		 * test if all register can be saved.
		 */
		"	tmll	%[m],15\n"	/* KERNEL_VXR_MASK */
		"	jz	20f\n"		/* no work -> done */
		"	la	1,%[vxrs]\n"	/* load save area */
		"	jo	18f\n"		/* -> save V0..V31 */

		/*
		 * Test if V8..V23 can be saved at once... this speeds up
		 * for KERNEL_fpu_MID only. Otherwise continue to split the
		 * range of vector registers into two halves and test them
		 * separately.
		 */
		"	tmll	%[m],6\n"	/* KERNEL_VXR_MID */
		"	jo	17f\n"		/* -> save V8..V23 */

		/* Test and save the first half of 16 vector registers */
		"1:	tmll	%[m],3\n"	/* KERNEL_VXR_LOW */
		"	jz	10f\n"		/* -> KERNEL_VXR_HIGH */
		"	jo	2f\n"		/* 11 -> save V0..V15 */
		"	brc	4,3f\n"		/* 01 -> save V0..V7  */
		"	brc	2,4f\n"		/* 10 -> save V8..V15 */

		/* Test and save the second half of 16 vector registers */
		"10:	tmll	%[m],12\n"	/* KERNEL_VXR_HIGH */
		"	jo	19f\n"		/* 11 -> save V16..V31 */
		"	brc	4,11f\n"	/* 01 -> save V16..V23	*/
		"	brc	2,12f\n"	/* 10 -> save V24..V31 */
		"	j	20f\n"		/* 00 -> done */

		/*
		 * Below are the vstm combinations to save multiple vector
		 * registers at once.
		 */
		"2:	.word	0xe70f,0x1000,0x003e\n"	/* vstm 0,15,0(1) */
		"	j	10b\n"			/* -> VXR_HIGH */
		"3:	.word	0xe707,0x1000,0x003e\n" /* vstm 0,7,0(1) */
		"	j	10b\n"			/* -> VXR_HIGH */
		"4:	.word	0xe78f,0x1080,0x003e\n" /* vstm 8,15,128(1) */
		"	j	10b\n"			/* -> VXR_HIGH */
		"\n"
		"11:	.word	0xe707,0x1100,0x0c3e\n"	/* vstm 16,23,256(1) */
		"	j	20f\n"			/* -> done */
		"12:	.word	0xe78f,0x1180,0x0c3e\n" /* vstm 24,31,384(1) */
		"	j	20f\n"			/* -> done */
		"\n"
		"17:	.word	0xe787,0x1080,0x043e\n"	/* vstm 8,23,128(1) */
		"	nill	%[m],249\n"		/* m &= ~VXR_MID    */
		"	j	1b\n"			/* -> VXR_LOW */
		"\n"
		"18:	.word	0xe70f,0x1000,0x003e\n"	/* vstm 0,15,0(1) */
		"19:	.word	0xe70f,0x1100,0x0c3e\n"	/* vstm 16,31,256(1) */
		"20:"
		: [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
		: [m] "d" (state->mask)
		: "1", "cc");
}
EXPORT_SYMBOL(__kernel_fpu_begin);

void __kernel_fpu_end(struct kernel_fpu *state)
{
	/* Just update the per-CPU state if there is nothing to restore */
	if (!(state->mask & KERNEL_FPU_STATE_MASK))
		goto update_fpu_state;

	/*
	 * If KERNEL_FPR is specified, the vector facility is not available
	 * and, thus, restore floating-point control and registers only.
	 */
	if (state->mask & KERNEL_FPR) {
		asm volatile("lfpc %0" : : "Q" (state->fpc));
		asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
		asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
		asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
		asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
		asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
		asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
		asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
		asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
		asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
		asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
		asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
		asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
		asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
		asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
		asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
		asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
		goto update_fpu_state;
	}

	/* Test and restore floating-point controls */
	if (state->mask & KERNEL_FPC)
		asm volatile("lfpc %0" : : "Q" (state->fpc));

	/* Test and restore (load) vector registers */
	asm volatile (
		/*
		 * Test if any vector registers must be loaded and, if so,
		 * test if all registers can be loaded at once.
		 */
		"	tmll	%[m],15\n"	/* KERNEL_VXR_MASK */
		"	jz	20f\n"		/* no work -> done */
		"	la	1,%[vxrs]\n"	/* load load area */
		"	jo	18f\n"		/* -> load V0..V31 */

		/*
		 * Test if V8..V23 can be restored at once... this speeds up
		 * for KERNEL_VXR_MID only. Otherwise continue to split the
		 * range of vector registers into two halves and test them
		 * separately.
		 */
		"	tmll	%[m],6\n"	/* KERNEL_VXR_MID */
		"	jo	17f\n"		/* -> load V8..V23 */

		/* Test and load the first half of 16 vector registers */
		"1:	tmll	%[m],3\n"	/* KERNEL_VXR_LOW */
		"	jz	10f\n"		/* -> KERNEL_VXR_HIGH */
		"	jo	2f\n"		/* 11 -> load V0..V15 */
		"	brc	4,3f\n"		/* 01 -> load V0..V7  */
		"	brc	2,4f\n"		/* 10 -> load V8..V15 */

		/* Test and load the second half of 16 vector registers */
		"10:	tmll	%[m],12\n"	/* KERNEL_VXR_HIGH */
		"	jo	19f\n"		/* 11 -> load V16..V31 */
		"	brc	4,11f\n"	/* 01 -> load V16..V23	*/
		"	brc	2,12f\n"	/* 10 -> load V24..V31 */
		"	j	20f\n"		/* 00 -> done */

		/*
		 * Below are the vstm combinations to load multiple vector
		 * registers at once.
		 */
		"2:	.word	0xe70f,0x1000,0x0036\n"	/* vlm 0,15,0(1) */
		"	j	10b\n"			/* -> VXR_HIGH */
		"3:	.word	0xe707,0x1000,0x0036\n" /* vlm 0,7,0(1) */
		"	j	10b\n"			/* -> VXR_HIGH */
		"4:	.word	0xe78f,0x1080,0x0036\n" /* vlm 8,15,128(1) */
		"	j	10b\n"			/* -> VXR_HIGH */
		"\n"
		"11:	.word	0xe707,0x1100,0x0c36\n"	/* vlm 16,23,256(1) */
		"	j	20f\n"			/* -> done */
		"12:	.word	0xe78f,0x1180,0x0c36\n" /* vlm 24,31,384(1) */
		"	j	20f\n"			/* -> done */
		"\n"
		"17:	.word	0xe787,0x1080,0x0436\n"	/* vlm 8,23,128(1) */
		"	nill	%[m],249\n"		/* m &= ~VXR_MID    */
		"	j	1b\n"			/* -> VXR_LOW */
		"\n"
		"18:	.word	0xe70f,0x1000,0x0036\n"	/* vlm 0,15,0(1) */
		"19:	.word	0xe70f,0x1100,0x0c36\n"	/* vlm 16,31,256(1) */
		"20:"
		:
		: [vxrs] "Q" (*(struct vx_array *) &state->vxrs),
		  [m] "d" (state->mask)
		: "1", "cc");

update_fpu_state:
	/* Update current kernel VX state */
	__this_cpu_write(kernel_fpu_state, state->mask);
}
EXPORT_SYMBOL(__kernel_fpu_end);