in include/asm/uaccess.h [131:392]
__asm__ __volatile__( \
"1: "op" %1,[%2]\n" \
"2: ;nop\n" \
" .section .fixup, \"ax\"\n" \
" .align 4\n" \
"3: mov %0, %3\n" \
" j 2b\n" \
" .previous\n" \
" .section __ex_table, \"a\"\n" \
" .align 4\n" \
" .word 1b,3b\n" \
" .previous\n" \
\
: "+r" (ret) \
: "r" (src), "r" (dst), "ir" (-EFAULT))
#define __arc_put_user_one_64(src, dst, ret) \
__asm__ __volatile__( \
"1: st %1,[%2]\n" \
"4: st %R1,[%2, 4]\n" \
"2: ;nop\n" \
" .section .fixup, \"ax\"\n" \
" .align 4\n" \
"3: mov %0, %3\n" \
" j 2b\n" \
" .previous\n" \
" .section __ex_table, \"a\"\n" \
" .align 4\n" \
" .word 1b,3b\n" \
" .word 4b,3b\n" \
" .previous\n" \
\
: "+r" (ret) \
: "r" (src), "r" (dst), "ir" (-EFAULT))
static inline unsigned long
raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
long res = 0;
char val;
unsigned long tmp1, tmp2, tmp3, tmp4;
unsigned long orig_n = n;
if (n == 0)
return 0;
/* unaligned */
if (((unsigned long)to & 0x3) || ((unsigned long)from & 0x3)) {
unsigned char tmp;
__asm__ __volatile__ (
" mov.f lp_count, %0 \n"
" lpnz 2f \n"
"1: ldb.ab %1, [%3, 1] \n"
" stb.ab %1, [%2, 1] \n"
" sub %0,%0,1 \n"
"2: ;nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"3: j 2b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 1b, 3b \n"
" .previous \n"
: "+r" (n),
/*
* Note as an '&' earlyclobber operand to make sure the
* temporary register inside the loop is not the same as
* FROM or TO.
*/
"=&r" (tmp), "+r" (to), "+r" (from)
:
: "lp_count", "memory");
return n;
}
/*
* Hand-crafted constant propagation to reduce code sz of the
* laddered copy 16x,8,4,2,1
*/
if (__builtin_constant_p(orig_n)) {
res = orig_n;
if (orig_n / 16) {
orig_n = orig_n % 16;
__asm__ __volatile__(
" lsr lp_count, %7,4 \n"
" lp 3f \n"
"1: ld.ab %3, [%2, 4] \n"
"11: ld.ab %4, [%2, 4] \n"
"12: ld.ab %5, [%2, 4] \n"
"13: ld.ab %6, [%2, 4] \n"
" st.ab %3, [%1, 4] \n"
" st.ab %4, [%1, 4] \n"
" st.ab %5, [%1, 4] \n"
" st.ab %6, [%1, 4] \n"
" sub %0,%0,16 \n"
"3: ;nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"4: j 3b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 1b, 4b \n"
" .word 11b,4b \n"
" .word 12b,4b \n"
" .word 13b,4b \n"
" .previous \n"
: "+r" (res), "+r"(to), "+r"(from),
"=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
: "ir"(n)
: "lp_count", "memory");
}
if (orig_n / 8) {
orig_n = orig_n % 8;
__asm__ __volatile__(
"14: ld.ab %3, [%2,4] \n"
"15: ld.ab %4, [%2,4] \n"
" st.ab %3, [%1,4] \n"
" st.ab %4, [%1,4] \n"
" sub %0,%0,8 \n"
"31: ;nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"4: j 31b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 14b,4b \n"
" .word 15b,4b \n"
" .previous \n"
: "+r" (res), "+r"(to), "+r"(from),
"=r"(tmp1), "=r"(tmp2)
:
: "memory");
}
if (orig_n / 4) {
orig_n = orig_n % 4;
__asm__ __volatile__(
"16: ld.ab %3, [%2,4] \n"
" st.ab %3, [%1,4] \n"
" sub %0,%0,4 \n"
"32: ;nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"4: j 32b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 16b,4b \n"
" .previous \n"
: "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
:
: "memory");
}
if (orig_n / 2) {
orig_n = orig_n % 2;
__asm__ __volatile__(
"17: ldw.ab %3, [%2,2] \n"
" stw.ab %3, [%1,2] \n"
" sub %0,%0,2 \n"
"33: ;nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"4: j 33b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 17b,4b \n"
" .previous \n"
: "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
:
: "memory");
}
if (orig_n & 1) {
__asm__ __volatile__(
"18: ldb.ab %3, [%2,2] \n"
" stb.ab %3, [%1,2] \n"
" sub %0,%0,1 \n"
"34: ; nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"4: j 34b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 18b,4b \n"
" .previous \n"
: "+r" (res), "+r"(to), "+r"(from), "=r"(tmp1)
:
: "memory");
}
} else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
__asm__ __volatile__(
" mov %0,%3 \n"
" lsr.f lp_count, %3,4 \n" /* 16x bytes */
" lpnz 3f \n"
"1: ld.ab %5, [%2, 4] \n"
"11: ld.ab %6, [%2, 4] \n"
"12: ld.ab %7, [%2, 4] \n"
"13: ld.ab %8, [%2, 4] \n"
" st.ab %5, [%1, 4] \n"
" st.ab %6, [%1, 4] \n"
" st.ab %7, [%1, 4] \n"
" st.ab %8, [%1, 4] \n"
" sub %0,%0,16 \n"
"3: and.f %3,%3,0xf \n" /* stragglers */
" bz 34f \n"
" bbit0 %3,3,31f \n" /* 8 bytes left */
"14: ld.ab %5, [%2,4] \n"
"15: ld.ab %6, [%2,4] \n"
" st.ab %5, [%1,4] \n"
" st.ab %6, [%1,4] \n"
" sub.f %0,%0,8 \n"
"31: bbit0 %3,2,32f \n" /* 4 bytes left */
"16: ld.ab %5, [%2,4] \n"
" st.ab %5, [%1,4] \n"
" sub.f %0,%0,4 \n"
"32: bbit0 %3,1,33f \n" /* 2 bytes left */
"17: ldw.ab %5, [%2,2] \n"
" stw.ab %5, [%1,2] \n"
" sub.f %0,%0,2 \n"
"33: bbit0 %3,0,34f \n"
"18: ldb.ab %5, [%2,1] \n" /* 1 byte left */
" stb.ab %5, [%1,1] \n"
" sub.f %0,%0,1 \n"
"34: ;nop \n"
" .section .fixup, \"ax\" \n"
" .align 4 \n"
"4: j 34b \n"
" .previous \n"
" .section __ex_table, \"a\" \n"
" .align 4 \n"
" .word 1b, 4b \n"
" .word 11b,4b \n"
" .word 12b,4b \n"
" .word 13b,4b \n"
" .word 14b,4b \n"
" .word 15b,4b \n"
" .word 16b,4b \n"
" .word 17b,4b \n"
" .word 18b,4b \n"
" .previous \n"
: "=r" (res), "+r"(to), "+r"(from), "+r"(n), "=r"(val),
"=r"(tmp1), "=r"(tmp2), "=r"(tmp3), "=r"(tmp4)
:
: "lp_count", "memory");
}
return res;
}