in simd/mips64/jdsample-mmi.c [107:239]
void jsimd_h2v2_fancy_upsample_mmi(int max_v_samp_factor,
JDIMENSION downsampled_width,
JSAMPARRAY input_data,
JSAMPARRAY *output_data_ptr)
{
JSAMPARRAY output_data = *output_data_ptr;
JSAMPROW inptr_1, inptr0, inptr1, outptr0, outptr1;
int inrow, outrow, incol, tmp, tmp1;
__m64 this_1l, this_1h, this_1, thiscolsum_1l, thiscolsum_1h;
__m64 this0l, this0h, this0;
__m64 this1l, this1h, this1, thiscolsum1l, thiscolsum1h;
__m64 next_1l, next_1h, next_1, nextcolsum_1l, nextcolsum_1h;
__m64 next0l, next0h, next0;
__m64 next1l, next1h, next1, nextcolsum1l, nextcolsum1h;
__m64 mask0 = 0.0, masklast, samp0123, samp4567, wk[4], zero = 0.0;
mask0 = _mm_cmpeq_pi8(mask0, mask0);
masklast = _mm_slli_si64(mask0, (SIZEOF_MMWORD - 2) * BYTE_BIT);
mask0 = _mm_srli_si64(mask0, (SIZEOF_MMWORD - 2) * BYTE_BIT);
for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) {
inptr_1 = input_data[inrow - 1];
inptr0 = input_data[inrow];
inptr1 = input_data[inrow + 1];
outptr0 = output_data[outrow++];
outptr1 = output_data[outrow++];
if (downsampled_width & 7) {
tmp = (downsampled_width - 1) * sizeof(JSAMPLE);
tmp1 = downsampled_width * sizeof(JSAMPLE);
asm(PTR_ADDU "$8, %3, %6\r\n"
"lb $9, ($8)\r\n"
PTR_ADDU "$8, %3, %7\r\n"
"sb $9, ($8)\r\n"
PTR_ADDU "$8, %4, %6\r\n"
"lb $9, ($8)\r\n"
PTR_ADDU "$8, %4, %7\r\n"
"sb $9, ($8)\r\n"
PTR_ADDU "$8, %5, %6\r\n"
"lb $9, ($8)\r\n"
PTR_ADDU "$8, %5, %7\r\n"
"sb $9, ($8)\r\n"
: "=m" (*inptr_1), "=m" (*inptr0), "=m" (*inptr1)
: "r" (inptr_1), "r" (inptr0), "r" (inptr1), "r" (tmp), "r" (tmp1)
: "$8", "$9"
);
}
/* process the first column block */
this0 = _mm_load_si64((__m64 *)inptr0); /* row[ 0][0] */
this_1 = _mm_load_si64((__m64 *)inptr_1); /* row[-1][0] */
this1 = _mm_load_si64((__m64 *)inptr1); /* row[ 1][0] */
this0l = _mm_unpacklo_pi8(this0, zero); /* row[ 0][0]( 0 1 2 3) */
this0h = _mm_unpackhi_pi8(this0, zero); /* row[ 0][0]( 4 5 6 7) */
this_1l = _mm_unpacklo_pi8(this_1, zero); /* row[-1][0]( 0 1 2 3) */
this_1h = _mm_unpackhi_pi8(this_1, zero); /* row[-1][0]( 4 5 6 7) */
this1l = _mm_unpacklo_pi8(this1, zero); /* row[+1][0]( 0 1 2 3) */
this1h = _mm_unpackhi_pi8(this1, zero); /* row[+1][0]( 4 5 6 7) */
this0l = _mm_mullo_pi16(this0l, PW_THREE);
this0h = _mm_mullo_pi16(this0h, PW_THREE);
thiscolsum_1l = _mm_add_pi16(this_1l, this0l); /* ( 0 1 2 3) */
thiscolsum_1h = _mm_add_pi16(this_1h, this0h); /* ( 4 5 6 7) */
thiscolsum1l = _mm_add_pi16(this0l, this1l); /* ( 0 1 2 3) */
thiscolsum1h = _mm_add_pi16(this0h, this1h); /* ( 4 5 6 7) */
/* temporarily save the intermediate data */
_mm_store_si64((__m64 *)outptr0, thiscolsum_1l);
_mm_store_si64((__m64 *)outptr0 + 1, thiscolsum_1h);
_mm_store_si64((__m64 *)outptr1, thiscolsum1l);
_mm_store_si64((__m64 *)outptr1 + 1, thiscolsum1h);
wk[0] = _mm_and_si64(thiscolsum_1l, mask0); /* ( 0 - - -) */
wk[1] = _mm_and_si64(thiscolsum1l, mask0); /* ( 0 - - -) */
for (incol = downsampled_width; incol > 0;
incol -= 8, inptr_1 += 8, inptr0 += 8, inptr1 += 8,
outptr0 += 16, outptr1 += 16) {
if (incol > 8) {
/* process the next column block */
next0 = _mm_load_si64((__m64 *)inptr0 + 1); /* row[ 0][1] */
next_1 = _mm_load_si64((__m64 *)inptr_1 + 1); /* row[-1][1] */
next1 = _mm_load_si64((__m64 *)inptr1 + 1); /* row[+1][1] */
next0l = _mm_unpacklo_pi8(next0, zero); /* row[ 0][1]( 0 1 2 3) */
next0h = _mm_unpackhi_pi8(next0, zero); /* row[ 0][1]( 4 5 6 7) */
next_1l = _mm_unpacklo_pi8(next_1, zero); /* row[-1][1]( 0 1 2 3) */
next_1h = _mm_unpackhi_pi8(next_1, zero); /* row[-1][1]( 4 5 6 7) */
next1l = _mm_unpacklo_pi8(next1, zero); /* row[+1][1]( 0 1 2 3) */
next1h = _mm_unpackhi_pi8(next1, zero); /* row[+1][1]( 4 5 6 7) */
next0l = _mm_mullo_pi16(next0l, PW_THREE);
next0h = _mm_mullo_pi16(next0h, PW_THREE);
nextcolsum_1l = _mm_add_pi16(next_1l, next0l); /* ( 0 1 2 3) */
nextcolsum_1h = _mm_add_pi16(next_1h, next0h); /* ( 4 5 6 7) */
nextcolsum1l = _mm_add_pi16(next0l, next1l); /* ( 0 1 2 3) */
nextcolsum1h = _mm_add_pi16(next0h, next1h); /* ( 4 5 6 7) */
/* temporarily save the intermediate data */
_mm_store_si64((__m64 *)outptr0 + 2, nextcolsum_1l);
_mm_store_si64((__m64 *)outptr0 + 3, nextcolsum_1h);
_mm_store_si64((__m64 *)outptr1 + 2, nextcolsum1l);
_mm_store_si64((__m64 *)outptr1 + 3, nextcolsum1h);
wk[2] = _mm_slli_si64(nextcolsum_1l, (SIZEOF_MMWORD - 2) * BYTE_BIT); /* ( - - - 0) */
wk[3] = _mm_slli_si64(nextcolsum1l, (SIZEOF_MMWORD - 2) * BYTE_BIT); /* ( - - - 0) */
} else {
__m64 tmp;
/* process the last column block */
tmp = _mm_load_si64((__m64 *)outptr0 + 1);
wk[2] = _mm_and_si64(masklast, tmp); /* ( - - - 7) */
tmp = _mm_load_si64((__m64 *)outptr1 + 1);
wk[3] = _mm_and_si64(masklast, tmp); /* ( - - - 7) */
}
/* process the upper row */
samp0123 = _mm_load_si64((__m64 *)outptr0); /* ( 0 1 2 3) */ \
samp4567 = _mm_load_si64((__m64 *)outptr0 + 1); /* ( 4 5 6 7) */ \
PROCESS_ROW(0, 2, PW_EIGHT, PW_SEVEN, 4)
/* process the lower row */
samp0123 = _mm_load_si64((__m64 *)outptr1); /* ( 0 1 2 3) */ \
samp4567 = _mm_load_si64((__m64 *)outptr1 + 1); /* ( 4 5 6 7) */ \
PROCESS_ROW(1, 2, PW_EIGHT, PW_SEVEN, 4)
}
}
}