sass/hgemm_16x64x64

# Kernel: hgemm_16x64x64_NT [- our $vec; sub vector { return $vec; } -] <CONSTANT_MAPPING> addr_zero : 4x<(16*64 + 32)*2 + (64*64 + 32)*2> szShareA : (16*64 + 32) szShareB : (64*64 + 32) param_C[0] : c[0x0][0x140] param_C[1] : c[0x0][0x144] param_A[0] : c[0x0][0x148] param_A[1] : c[0x0][0x14c] param_B[0] : c[0x0][0x150] param_B[1] : c[0x0][0x154] param_alpha : c[0x0][0x158] param_beta : c[0x0][0x15c] param_cda : c[0x0][0x160] param_cdb : c[0x0][0x164] param_cdc : c[0x0][0x168] param_m : c[0x0][0x16c] param_n : c[0x0][0x170] param_k : c[0x0][0x174] param_blk_a : c[0x0][0x178] param_blk_b : c[0x0][0x17c] </CONSTANT_MAPPING> <REGISTER_MAPPING> 3, 2,11,10,19,18,27,26 : cx<0-7>y0 7, 6,15,14,23,22,31,30 : cx<0-7>y1 1, 0, 9, 8,17,16,25,24 : cx<0-7>y2 5, 4,13,12,21,20,29,28 : cx<0-7>y3 35,34,43,42,51,50,59,58 : cx<0-7>y4 39,38,47,46,55,54,63,62 : cx<0-7>y5 33,32,41,40,49,48,57,56 : cx<0-7>y6 37,36,45,44,53,52,61,60 : cx<0-7>y7 0-63 : czero<00-63> 64-79 : j0Ay<0-7>, j0Bx<0-7> 80-95 : j1Ay<0-7>, j1Bx<0-7> 64-95 ~ idx_ab, idx_ab_f, idx_a, neg_blk_b, rcp_blk_b, idx_b, tidX, tidY, tidY<1-3>, cda, cdb, cdb16, tid16_8, ta, txa, tb<00|16|32|48>, txb<00|16|32|48>, xmad_ta, xmad_tb, shiftX, predsY0, predsY4, partialK 96-135 : load0A<0-7>, load0B<0-7>, load1B<0-7>, load2B<0-7>, load3B<0-7> 136-145 : track0A<0-1>, track0B<0-1>, track1B<0-1>, track2B<0-1>, track3B<0-1> 146-152 ~ swapBuf, readAs, readBs, writeAs, writeBs, k 153-159 ~ tid, idx_A, idx_B, blkZ, writeCs, preds, tid16 0-31 : part0C<0-3>, part1C<0-3>, part2C<0-3>, part3C<0-3>, part4C<0-3>, part5C<0-3>, part6C<0-3>, part7C<0-3> 64-95 : shuffle_x<0-7>y0, shuffle_x<0-7>y1, shuffle_x<0-7>y2, shuffle_x<0-7>y3 64-95 : shuffle_x<0-7>y4, shuffle_x<0-7>y5, shuffle_x<0-7>y6, shuffle_x<0-7>y7 96-99 : loadC<0-3> 100-103 : b<0-3> 104-107 : c<0-3> 108-109 : C<0-1> 110-152 ~ cdc, cx, cx<1-3>, cy, ci, xmad_c, cdc8, readCs, alpha, beta, tid15 </REGISTER_MAPPING> --:-:5:-:1 I2F.F32.S32 rcp_blk_b, param_blk_b; --:-:1:-:1 S2R tid, SR_TID.X; --:-:2:-:1 S2R idx_ab, SR_CTAID.X; --:-:3:-:1 S2R idx_A, SR_CTAID.Z; --:-:4:-:1 S2R idx_B, SR_CTAID.Y; <SCHEDULE_BLOCK> 10:-:5:-:1 MUFU.RCP rcp_blk_b, rcp_blk_b; --:-:-:-:1 MOV k, param_k; --:-:-:-:1 MOV cda, param_cda; --:-:-:-:1 MOV cdb, param_cdb; --:-:-:-:1 SHL cdb16, cdb, 4; # idx_a = idx_ab // blk_b 02:-:2:-:1 I2F.F32.S32 idx_ab_f, idx_ab; 12:-:-:-:1 FMUL idx_a, idx_ab_f, rcp_blk_b; --:-:-:-:1 FFMA idx_a, idx_a, 5.9604644775390625e-08, idx_a; --:-:2:-:1 F2I.S32.F32.TRUNC idx_a, idx_a; # idx_b = idx_AB % blk_b --:-:-:-:1 IADD neg_blk_b, RZ, -param_blk_b; 02:-:-:-:1 XMAD.S16.U16 idx_b, neg_blk_b, idx_a, idx_ab; # idx_A = idx_A * blk_a + idx_a # idx_B = idx_B * blk_b + idx_b 06:-:-:-:1 XMAD.U16.U16 idx_A, idx_A, param_blk_a, idx_a; 08:-:-:-:1 XMAD.U16.U16 idx_B, idx_B, param_blk_b, idx_b; --:-:-:-:1 STS.128 [addr_zero], RZ; [+ join '', map sprintf("--:-:-:-:1 LDS.U.128 czero%02d, [addr_zero];\n", $_ * 4), 0..15; +] // tidX = tid >> 3 // tidY = (tid & 7) << 3 // shiftX = (tid & 7) << 2 01:-:-:-:1 SHR.U32 tidX, tid, 3; --:-:-:-:1 LOP.AND tidY, tid, 7; --:-:-:-:1 SHL shiftX, tidY, 2; --:-:-:-:1 SHL tidY, tidY, 3; // trackA += ((idx_A*16 + tidX) * cda + tidY) * 2 --:-:-:-:1 ISCADD txa, idx_A, tidX, 4; --:-:-:-:1 XMAD.LO ta, cda, txa, tidY, xmad_ta; --:-:-:-:1 LEA track0A0.CC, ta, param_A[0], 1; --:-:-:-:1 LEA.HI.X track0A1, ta, param_A[1], RZ, 1; --:-:-:-:1 ISETP.LT.AND P2, PT, txa, param_m, PT; // trackB += ((idx_B*64 + tidX) * cdb + tidY) * 2 --:-:-:-:1 ISCADD txb00, idx_B, tidX, 6; --:-:-:-:1 IADD txb16, txb00, 16; --:-:-:-:1 IADD txb32, txb00, 32; --:-:-:-:1 IADD txb48, txb00, 48; --:-:-:-:1 XMAD.LO tb00, cdb, txb00, tidY, xmad_tb; --:-:-:-:1 IADD tb16, tb00, cdb16; --:-:-:-:1 IADD tb32, tb16, cdb16; --:-:-:-:1 IADD tb48, tb32, cdb16; --:-:-:-:1 LEA track0B0.CC, tb00, param_B[0], 1; --:-:-:-:1 LEA.HI.X track0B1, tb00, param_B[1], RZ, 1; --:-:-:-:1 LEA track1B0.CC, tb16, param_B[0], 1; --:-:-:-:1 LEA.HI.X track1B1, tb16, param_B[1], RZ, 1; --:-:-:-:1 LEA track2B0.CC, tb32, param_B[0], 1; --:-:-:-:1 LEA.HI.X track2B1, tb32, param_B[1], RZ, 1; --:-:-:-:1 LEA track3B0.CC, tb48, param_B[0], 1; --:-:-:-:1 LEA.HI.X track3B1, tb48, param_B[1], RZ, 1; --:-:-:-:1 ISETP.LT.AND P3, PT, txb00, param_n, PT; --:-:-:-:1 ISETP.LT.AND P4, PT, txb16, param_n, PT; --:-:-:-:1 ISETP.LT.AND P5, PT, txb32, param_n, PT; --:-:-:-:1 ISETP.LT.AND P6, PT, txb48, param_n, PT; --:-:-:-:1 P2R preds, PR, RZ, 0x7c; // writeAs = (tidY*16 + tidX + shiftX) * 4 --:-:-:-:1 ISCADD writeAs, tidY, tidX, 4; --:-:-:-:1 IADD writeAs, writeAs, shiftX; --:-:-:-:1 SHL writeAs, writeAs, 2; // writeBs = (tidY*64 + tidX + shiftX) * 4 --:-:-:-:1 ISCADD writeBs, tidY, tidX, 6; --:-:-:-:1 IADD writeBs, writeBs, shiftX; --:-:-:-:1 ISCADD writeBs, writeBs, 4x<szShareA>, 2; // readAs = (tid & 1) << 4 --:-:-:-:1 LOP.AND readAs, tid, 1; --:-:-:-:1 SHL readAs, readAs, 4; // readBs = (((tid >> 1) & 7) << 4 --:-:-:-:1 BFE.U32 readBs, tid, 0x301; // 3 bits at position 1 --:-:-:-:1 SHL readBs, readBs, 4; // tid16 = tid & -16 // tid16_8 = tid16 / 2 * 4 --:-:-:-:1 LOP.AND tid16, tid, -16; --:-:-:-:1 SHL tid16_8, tid16, 1; // writeCs = (readAs + tid16*2) * 64 + readBs; --:-:-:-:1 ISCADD writeCs, tid16, readAs, 1; --:-:-:-:1 ISCADD writeCs, writeCs, readBs, 6; // Each block of 16 threads works on 8 lines, shifted over by 4 // readAs += tid16_8 * 16 + tid16 // readBs += tid16_8 * 64 + tid16 + 4x<szShareA> --:-:-:-:1 ISCADD readAs, tid16_8, readAs, 4; --:-:-:-:1 ISCADD readBs, tid16_8, readBs, 6; --:-:-:-:1 IADD readAs, tid16, readAs; --:-:-:-:1 IADD3 readBs, tid16, 4x<szShareA>, readBs; --:-:-:-:1 MOV32I swapBuf, 4x<szShareA + szShareB>; // If k is not a multiple of 64 we want to grab the partial amount on the first fetch. // If it is a multiple of 64 then make a full 64 line fetch. --:-:-:-:1 LOP.AND.Z P0, partialK, k, 63; --:-:-:-:1 @P0 MOV partialK, 64; --:-:-:-:1 IADD k, k, -partialK; [+ return vector() ? q{ --:-:-:-:1 ISETP.LT.AND P1, PT, tidY, partialK, PT; --:-:-:-:1 @P1 R2P PR, preds, 0x7c; --:-:-:-:1 @!P1 R2P PR, RZ, 0x7c; <ORDERED> --:-:2:-:1 @P2 LDG.E.CI.128 load0A, [track0A]; --:-:3:-:1 @P3 LDG.E.CI.128 load0B, [track0B]; --:-:4:-:1 @P4 LDG.E.CI.128 load1B, [track1B]; --:-:5:-:1 @P5 LDG.E.CI.128 load2B, [track2B]; --:-:6:-:1 @P6 LDG.E.CI.128 load3B, [track3B]; </ORDERED> <ORDERED> --:-:-:-:1 @!P2 LDS.U.128 load0A, [addr_zero]; --:-:-:-:1 @!P3 LDS.U.128 load0B, [addr_zero]; --:-:-:-:1 @!P4 LDS.U.128 load1B, [addr_zero]; --:-:-:-:1 @!P5 LDS.U.128 load2B, [addr_zero]; --:-:1:-:1 @!P6 LDS.U.128 load3B, [addr_zero]; </ORDERED> } : q{ --:-:-:-:1 IADD tidY1, tidY, 1; --:-:-:-:1 IADD tidY2, tidY, 2; --:-:-:-:1 IADD tidY3, tidY, 3; --:-:-:-:1 ISETP.LT.AND P0, PT, tidY, partialK, PT; --:-:-:-:1 ISETP.LT.AND P1, PT, tidY1, partialK, PT; --:-:-:-:1 ISETP.LT.AND P2, PT, tidY2, partialK, PT; --:-:-:-:1 ISETP.LT.AND P3, PT, tidY3, partialK, PT; --:-:-:-:1 P2R predsY0, PR, RZ, 0x0f; --:-:-:-:1 IADD tidY, tidY, 4; --:-:-:-:1 IADD tidY1, tidY1, 4; --:-:-:-:1 IADD tidY2, tidY2, 4; --:-:-:-:1 IADD tidY3, tidY3, 4; --:-:-:-:1 ISETP.LT.AND P0, PT, tidY, partialK, PT; --:-:-:-:1 ISETP.LT.AND P1, PT, tidY1, partialK, PT; --:-:-:-:1 ISETP.LT.AND P2, PT, tidY2, partialK, PT; --:-:-:-:1 ISETP.LT.AND P3, PT, tidY3, partialK, PT; --:-:-:-:1 P2R predsY4, PR, RZ, 0x0f; --:-:-:-:1 ISETP.LT.AND P4, PT, txa, param_m, PT; --:-:-:-:1 @P4 R2P PR, predsY0, 0x0f; --:-:-:-:1 @!P4 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load0A0, [track0A + 2x<0>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load0A1, [track0A + 2x<1>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A2, [track0A + 2x<2>]; --:-:2:-:1 @P3 LDG.E.CI.U16 load0A3, [track0A + 2x<3>]; </ORDERED> --:-:-:-:1 @!P0 MOV load0A0, RZ; --:-:-:-:1 @!P1 MOV load0A1, RZ; --:-:-:-:1 @!P2 MOV load0A2, RZ; --:-:-:-:1 @!P3 MOV load0A3, RZ; --:-:-:-:1 @P4 R2P PR, predsY4, 0x0f; --:-:-:-:1 @!P4 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load0A4, [track0A + 2x<4>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load0A5, [track0A + 2x<5>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A6, [track0A + 2x<6>]; --:-:2:-:1 @P3 LDG.E.CI.U16 load0A7, [track0A + 2x<7>]; </ORDERED> --:-:-:-:1 @!P0 MOV load0A4, RZ; --:-:-:-:1 @!P1 MOV load0A5, RZ; --:-:-:-:1 @!P2 MOV load0A6, RZ; --:-:-:-:1 @!P3 MOV load0A7, RZ; --:-:-:-:1 ISETP.LT.AND P5, PT, txb00, param_n, PT; --:-:-:-:1 @P5 R2P PR, predsY0, 0x0f; --:-:-:-:1 @!P5 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load0B0, [track0B + 2x<0>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load0B1, [track0B + 2x<1>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0B2, [track0B + 2x<2>]; --:-:3:-:1 @P3 LDG.E.CI.U16 load0B3, [track0B + 2x<3>]; </ORDERED> --:-:-:-:1 @!P0 MOV load0B0, RZ; --:-:-:-:1 @!P1 MOV load0B1, RZ; --:-:-:-:1 @!P2 MOV load0B2, RZ; --:-:-:-:1 @!P3 MOV load0B3, RZ; --:-:-:-:1 @P5 R2P PR, predsY4, 0x0f; --:-:-:-:1 @!P5 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load0B4, [track0B + 2x<4>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load0B5, [track0B + 2x<5>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0B6, [track0B + 2x<6>]; --:-:3:-:1 @P3 LDG.E.CI.U16 load0B7, [track0B + 2x<7>]; </ORDERED> --:-:-:-:1 @!P0 MOV load0B4, RZ; --:-:-:-:1 @!P1 MOV load0B5, RZ; --:-:-:-:1 @!P2 MOV load0B6, RZ; --:-:-:-:1 @!P3 MOV load0B7, RZ; --:-:-:-:1 ISETP.LT.AND P6, PT, txb16, param_n, PT; --:-:-:-:1 @P6 R2P PR, predsY0, 0x0f; --:-:-:-:1 @!P6 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load1B0, [track1B + 2x<0>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load1B1, [track1B + 2x<1>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load1B2, [track1B + 2x<2>]; --:-:4:-:1 @P3 LDG.E.CI.U16 load1B3, [track1B + 2x<3>]; </ORDERED> --:-:-:-:1 @!P0 MOV load1B0, RZ; --:-:-:-:1 @!P1 MOV load1B1, RZ; --:-:-:-:1 @!P2 MOV load1B2, RZ; --:-:-:-:1 @!P3 MOV load1B3, RZ; --:-:-:-:1 @P6 R2P PR, predsY4, 0x0f; --:-:-:-:1 @!P6 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load1B4, [track1B + 2x<4>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load1B5, [track1B + 2x<5>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load1B6, [track1B + 2x<6>]; --:-:4:-:1 @P3 LDG.E.CI.U16 load1B7, [track1B + 2x<7>]; </ORDERED> --:-:-:-:1 @!P0 MOV load1B4, RZ; --:-:-:-:1 @!P1 MOV load1B5, RZ; --:-:-:-:1 @!P2 MOV load1B6, RZ; --:-:-:-:1 @!P3 MOV load1B7, RZ; --:-:-:-:1 ISETP.LT.AND P4, PT, txb32, param_n, PT; --:-:-:-:1 @P4 R2P PR, predsY0, 0x0f; --:-:-:-:1 @!P4 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load2B0, [track2B + 2x<0>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load2B1, [track2B + 2x<1>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load2B2, [track2B + 2x<2>]; --:-:5:-:1 @P3 LDG.E.CI.U16 load2B3, [track2B + 2x<3>]; </ORDERED> --:-:-:-:1 @!P0 MOV load2B0, RZ; --:-:-:-:1 @!P1 MOV load2B1, RZ; --:-:-:-:1 @!P2 MOV load2B2, RZ; --:-:-:-:1 @!P3 MOV load2B3, RZ; --:-:-:-:1 @P4 R2P PR, predsY4, 0x0f; --:-:-:-:1 @!P4 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load2B4, [track2B + 2x<4>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load2B5, [track2B + 2x<5>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load2B6, [track2B + 2x<6>]; --:-:5:-:1 @P3 LDG.E.CI.U16 load2B7, [track2B + 2x<7>]; </ORDERED> --:-:-:-:1 @!P0 MOV load2B4, RZ; --:-:-:-:1 @!P1 MOV load2B5, RZ; --:-:-:-:1 @!P2 MOV load2B6, RZ; --:-:-:-:1 @!P3 MOV load2B7, RZ; --:-:-:-:1 ISETP.LT.AND P6, PT, txb48, param_n, PT; --:-:-:-:1 @P6 R2P PR, predsY0, 0x0f; --:-:-:-:1 @!P6 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load3B0, [track3B + 2x<0>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load3B1, [track3B + 2x<1>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load3B2, [track3B + 2x<2>]; --:-:6:-:1 @P3 LDG.E.CI.U16 load3B3, [track3B + 2x<3>]; </ORDERED> --:-:-:-:1 @!P0 MOV load3B0, RZ; --:-:-:-:1 @!P1 MOV load3B1, RZ; --:-:-:-:1 @!P2 MOV load3B2, RZ; --:-:-:-:1 @!P3 MOV load3B3, RZ; --:-:-:-:1 @P6 R2P PR, predsY4, 0x0f; --:-:-:-:1 @!P6 R2P PR, RZ, 0x0f; <ORDERED> --:-:-:-:1 @P0 LDG.E.CI.U16 load3B4, [track3B + 2x<4>]; --:-:-:-:1 @P1 LDG.E.CI.U16 load3B5, [track3B + 2x<5>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load3B6, [track3B + 2x<6>]; --:-:6:-:1 @P3 LDG.E.CI.U16 load3B7, [track3B + 2x<7>]; </ORDERED> --:-:-:-:1 @!P0 MOV load3B4, RZ; --:-:-:-:1 @!P1 MOV load3B5, RZ; --:-:-:-:1 @!P2 MOV load3B6, RZ; --:-:-:-:1 @!P3 MOV load3B7, RZ; }; +] --:-:-:-:1 SHL partialK, partialK, 1; --:-:-:-:1 ISETP.GE.AND P0, PT, k, 64, PT; --:-:-:-:1 IADD k, k, -64; --:-:-:-:1 @P0 R2P PR, preds, 0x7c; --:-:-:-:1 @!P0 R2P PR, RZ, 0x7c; </SCHEDULE_BLOCK> [+ return vector() ? q{ 03:-:-:-:1 F2F.F32.F16 load0A7, load0A3.H1; --:-:-:-:1 F2F.F32.F16 load0A6, load0A3.H0; --:-:-:-:1 F2F.F32.F16 load0A5, load0A2.H1; --:-:1:-:1 F2F.F32.F16 load0A4, load0A2.H0; --:-:-:-:1 F2F.F32.F16 load0A3, load0A1.H1; --:-:-:-:1 F2F.F32.F16 load0A2, load0A1.H0; --:-:-:-:1 F2F.F32.F16 load0A1, load0A0.H1; --:-:2:-:1 F2F.F32.F16 load0A0, load0A0.H0; } : q{ 02:-:-:-:1 F2F.F32.F16 load0A7, load0A7; --:-:-:-:1 F2F.F32.F16 load0A6, load0A6; --:-:-:-:1 F2F.F32.F16 load0A5, load0A5; --:-:1:-:1 F2F.F32.F16 load0A4, load0A4; --:-:-:-:1 F2F.F32.F16 load0A3, load0A3; --:-:-:-:1 F2F.F32.F16 load0A2, load0A2; --:-:-:-:1 F2F.F32.F16 load0A1, load0A1; --:-:2:-:1 F2F.F32.F16 load0A0, load0A0; }; +] --:-:-:-:0 IADD track0A0.CC, track0A0, partialK; 01:-:-:-:1 STS [writeAs + 4x<7*16>], load0A7; --:-:-:-:1 STS [writeAs + 4x<6*16>], load0A6; --:-:-:-:1 STS [writeAs + 4x<5*16>], load0A5; --:-:-:-:1 STS [writeAs + 4x<4*16>], load0A4; 02:-:-:-:1 STS [writeAs + 4x<3*16>], load0A3; --:-:-:-:1 STS [writeAs + 4x<2*16>], load0A2; --:-:-:-:1 STS [writeAs + 4x<1*16>], load0A1; --:-:-:-:1 STS [writeAs + 4x<0*16>], load0A0; --:-:-:-:0 IADD.X track0A1, track0A1, RZ; [+ return vector() ? q{ 04:-:-:-:1 F2F.F32.F16 load0B7, load0B3.H1; --:-:-:-:1 F2F.F32.F16 load0B6, load0B3.H0; --:-:-:-:1 F2F.F32.F16 load0B5, load0B2.H1; --:-:1:-:1 F2F.F32.F16 load0B4, load0B2.H0; --:-:-:-:1 F2F.F32.F16 load0B3, load0B1.H1; --:-:-:-:1 F2F.F32.F16 load0B2, load0B1.H0; --:-:-:-:1 F2F.F32.F16 load0B1, load0B0.H1; --:-:2:-:1 F2F.F32.F16 load0B0, load0B0.H0; } : q{ 04:-:-:-:1 F2F.F32.F16 load0B7, load0B7; --:-:-:-:1 F2F.F32.F16 load0B6, load0B6; --:-:-:-:1 F2F.F32.F16 load0B5, load0B5; --:-:1:-:1 F2F.F32.F16 load0B4, load0B4; --:-:-:-:1 F2F.F32.F16 load0B3, load0B3; --:-:-:-:1 F2F.F32.F16 load0B2, load0B2; --:-:-:-:1 F2F.F32.F16 load0B1, load0B1; --:-:2:-:1 F2F.F32.F16 load0B0, load0B0; }; +] --:-:-:-:0 IADD track0B0.CC, track0B0, partialK; 01:-:-:-:1 STS [writeBs + 4x<7*64 + 0*16>], load0B7; --:-:-:-:1 STS [writeBs + 4x<6*64 + 0*16>], load0B6; --:-:-:-:1 STS [writeBs + 4x<5*64 + 0*16>], load0B5; --:-:-:-:1 STS [writeBs + 4x<4*64 + 0*16>], load0B4; 02:-:-:-:1 STS [writeBs + 4x<3*64 + 0*16>], load0B3; --:-:-:-:1 STS [writeBs + 4x<2*64 + 0*16>], load0B2; --:-:-:-:1 STS [writeBs + 4x<1*64 + 0*16>], load0B1; --:-:-:-:1 STS [writeBs + 4x<0*64 + 0*16>], load0B0; --:-:-:-:0 IADD.X track0B1, track0B1, RZ; [+ return vector() ? q{ 08:-:-:-:1 F2F.F32.F16 load1B7, load1B3.H1; --:-:-:-:1 F2F.F32.F16 load1B6, load1B3.H0; --:-:-:-:1 F2F.F32.F16 load1B5, load1B2.H1; --:-:1:-:1 F2F.F32.F16 load1B4, load1B2.H0; --:-:-:-:1 F2F.F32.F16 load1B3, load1B1.H1; --:-:-:-:1 F2F.F32.F16 load1B2, load1B1.H0; --:-:-:-:1 F2F.F32.F16 load1B1, load1B0.H1; --:-:2:-:1 F2F.F32.F16 load1B0, load1B0.H0; } : q{ 08:-:-:-:1 F2F.F32.F16 load1B7, load1B7; --:-:-:-:1 F2F.F32.F16 load1B6, load1B6; --:-:-:-:1 F2F.F32.F16 load1B5, load1B5; --:-:1:-:1 F2F.F32.F16 load1B4, load1B4; --:-:-:-:1 F2F.F32.F16 load1B3, load1B3; --:-:-:-:1 F2F.F32.F16 load1B2, load1B2; --:-:-:-:1 F2F.F32.F16 load1B1, load1B1; --:-:2:-:1 F2F.F32.F16 load1B0, load1B0; }; +] --:-:-:-:0 IADD track1B0.CC, track1B0, partialK; 01:-:-:-:1 STS [writeBs + 4x<7*64 + 1*16>], load1B7; --:-:-:-:1 STS [writeBs + 4x<6*64 + 1*16>], load1B6; --:-:-:-:1 STS [writeBs + 4x<5*64 + 1*16>], load1B5; --:-:-:-:1 STS [writeBs + 4x<4*64 + 1*16>], load1B4; 02:-:-:-:1 STS [writeBs + 4x<3*64 + 1*16>], load1B3; --:-:-:-:1 STS [writeBs + 4x<2*64 + 1*16>], load1B2; --:-:-:-:1 STS [writeBs + 4x<1*64 + 1*16>], load1B1; --:-:-:-:1 STS [writeBs + 4x<0*64 + 1*16>], load1B0; --:-:-:-:0 IADD.X track1B1, track1B1, RZ; [+ return vector() ? q{ 10:-:-:-:1 F2F.F32.F16 load2B7, load2B3.H1; --:-:-:-:1 F2F.F32.F16 load2B6, load2B3.H0; --:-:-:-:1 F2F.F32.F16 load2B5, load2B2.H1; --:-:1:-:1 F2F.F32.F16 load2B4, load2B2.H0; --:-:-:-:1 F2F.F32.F16 load2B3, load2B1.H1; --:-:-:-:1 F2F.F32.F16 load2B2, load2B1.H0; --:-:-:-:1 F2F.F32.F16 load2B1, load2B0.H1; --:-:2:-:1 F2F.F32.F16 load2B0, load2B0.H0; } : q{ 10:-:-:-:1 F2F.F32.F16 load2B7, load2B7; --:-:-:-:1 F2F.F32.F16 load2B6, load2B6; --:-:-:-:1 F2F.F32.F16 load2B5, load2B5; --:-:1:-:1 F2F.F32.F16 load2B4, load2B4; --:-:-:-:1 F2F.F32.F16 load2B3, load2B3; --:-:-:-:1 F2F.F32.F16 load2B2, load2B2; --:-:-:-:1 F2F.F32.F16 load2B1, load2B1; --:-:2:-:1 F2F.F32.F16 load2B0, load2B0; }; +] --:-:-:-:0 IADD track2B0.CC, track2B0, partialK; 01:-:-:-:1 STS [writeBs + 4x<7*64 + 2*16>], load2B7; --:-:-:-:1 STS [writeBs + 4x<6*64 + 2*16>], load2B6; --:-:-:-:1 STS [writeBs + 4x<5*64 + 2*16>], load2B5; --:-:-:-:1 STS [writeBs + 4x<4*64 + 2*16>], load2B4; 02:-:-:-:1 STS [writeBs + 4x<3*64 + 2*16>], load2B3; --:-:-:-:1 STS [writeBs + 4x<2*64 + 2*16>], load2B2; --:-:-:-:1 STS [writeBs + 4x<1*64 + 2*16>], load2B1; --:-:-:-:1 STS [writeBs + 4x<0*64 + 2*16>], load2B0; --:-:-:-:0 IADD.X track2B1, track2B1, RZ; [+ return vector() ? q{ 20:-:-:-:1 F2F.F32.F16 load3B7, load3B3.H1; --:-:-:-:1 F2F.F32.F16 load3B6, load3B3.H0; --:-:-:-:1 F2F.F32.F16 load3B5, load3B2.H1; --:-:1:-:1 F2F.F32.F16 load3B4, load3B2.H0; --:-:-:-:1 F2F.F32.F16 load3B3, load3B1.H1; --:-:-:-:1 F2F.F32.F16 load3B2, load3B1.H0; --:-:-:-:1 F2F.F32.F16 load3B1, load3B0.H1; --:-:2:-:1 F2F.F32.F16 load3B0, load3B0.H0; } : q{ 20:-:-:-:1 F2F.F32.F16 load3B7, load3B7; --:-:-:-:1 F2F.F32.F16 load3B6, load3B6; --:-:-:-:1 F2F.F32.F16 load3B5, load3B5; --:-:1:-:1 F2F.F32.F16 load3B4, load3B4; --:-:-:-:1 F2F.F32.F16 load3B3, load3B3; --:-:-:-:1 F2F.F32.F16 load3B2, load3B2; --:-:-:-:1 F2F.F32.F16 load3B1, load3B1; --:-:2:-:1 F2F.F32.F16 load3B0, load3B0; }; +] --:-:-:-:0 IADD track3B0.CC, track3B0, partialK; 01:-:-:-:1 STS [writeBs + 4x<7*64 + 3*16>], load3B7; --:-:-:-:1 STS [writeBs + 4x<6*64 + 3*16>], load3B6; --:-:-:-:1 STS [writeBs + 4x<5*64 + 3*16>], load3B5; --:-:-:-:1 STS [writeBs + 4x<4*64 + 3*16>], load3B4; 02:-:-:-:1 STS [writeBs + 4x<3*64 + 3*16>], load3B3; --:-:-:-:1 STS [writeBs + 4x<2*64 + 3*16>], load3B2; --:-:-:-:1 STS [writeBs + 4x<1*64 + 3*16>], load3B1; --:-:-:-:1 STS [writeBs + 4x<0*64 + 3*16>], load3B0; --:-:-:-:0 IADD.X track3B1, track3B1, RZ; --:-:-:-:5 BAR.SYNC 0; --:-:-:-:1 IADD writeBs, writeBs, swapBuf; --:-:-:-:1 IADD writeAs, writeAs, swapBuf; --:-:-:-:0 IADD swapBuf, RZ, -swapBuf; --:-:-:-:1 LDS.U.128 j0Ay0, [readAs + 4x<0*16 + 00>]; --:-:-:-:1 LDS.U.128 j0Bx0, [readBs + 4x<0*64 + 00>]; --:-:-:-:1 LDS.U.128 j0Ay4, [readAs + 4x<0*16 + 08>]; --:-:1:-:1 LDS.U.128 j0Bx4, [readBs + 4x<0*64 + 32>]; [+ return vector() ? q{ --:-:2:-:1 @P2 LDG.E.CI.128 load0A, [track0A]; --:-:3:-:1 @P3 LDG.E.CI.128 load0B, [track0B]; --:-:4:-:1 @P4 LDG.E.CI.128 load1B, [track1B]; --:-:5:-:1 @P5 LDG.E.CI.128 load2B, [track2B]; --:-:6:-:1 @P6 LDG.E.CI.128 load3B, [track3B]; } : q{ --:-:-:-:1 @P2 LDG.E.CI.U16 load0A0, [track0A + 2x<0>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A1, [track0A + 2x<1>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A2, [track0A + 2x<2>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A3, [track0A + 2x<3>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A4, [track0A + 2x<4>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A5, [track0A + 2x<5>]; --:-:-:-:1 @P2 LDG.E.CI.U16 load0A6, [track0A + 2x<6>]; --:-:2:-:1 @P2 LDG.E.CI.U16 load0A7, [track0A + 2x<7>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B0, [track0B + 2x<0>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B1, [track0B + 2x<1>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B2, [track0B + 2x<2>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B3, [track0B + 2x<3>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B4, [track0B + 2x<4>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B5, [track0B + 2x<5>]; --:-:-:-:1 @P3 LDG.E.CI.U16 load0B6, [track0B + 2x<6>]; --:-:3:-:1 @P3 LDG.E.CI.U16 load0B7, [track0B + 2x<7>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B0, [track1B + 2x<0>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B1, [track1B + 2x<1>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B2, [track1B + 2x<2>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B3, [track1B + 2x<3>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B4, [track1B + 2x<4>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B5, [track1B + 2x<5>]; --:-:-:-:1 @P4 LDG.E.CI.U16 load1B6, [track1B + 2x<6>]; --:-:4:-:1 @P4 LDG.E.CI.U16 load1B7, [track1B + 2x<7>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B0, [track2B + 2x<0>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B1, [track2B + 2x<1>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B2, [track2B + 2x<2>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B3, [track2B + 2x<3>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B4, [track2B + 2x<4>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B5, [track2B + 2x<5>]; --:-:-:-:1 @P5 LDG.E.CI.U16 load2B6, [track2B + 2x<6>]; --:-:5:-:1 @P5 LDG.E.CI.U16 load2B7, [track2B + 2x<7>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B0, [track3B + 2x<0>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B1, [track3B + 2x<1>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B2, [track3B + 2x<2>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B3, [track3B + 2x<3>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B4, [track3B + 2x<4>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B5, [track3B + 2x<5>]; --:-:-:-:1 @P6 LDG.E.CI.U16 load3B6, [track3B + 2x<6>]; --:-:6:-:1 @P6 LDG.E.CI.U16 load3B7, [track3B + 2x<7>]; }; +] LOOP: [+ our %insert = ( j0c8 => "--:-:-:-:1 ISETP.GE.AND P0, PT, k, RZ, PT;\n", j0c10 => "--:-:-:-:1 ISETP.GE.AND P1, PT, k, 64, PT;\n" . "--:-:-:-:1 IADD k, k, -64;\n", j0c23 => "--:-:-:-:1 \@P1 R2P PR, preds, 0x7c;\n", j0c24 => "--:-:-:-:1 \@!P1 R2P PR, RZ, 0x7c;\n", j2c32 => "--:-:-:-:1 \@P2 IADD track0A0.CC, track0A0, 2x<64>;\n", j2c37 => "--:-:-:-:1 \@P2 IADD.X track0A1, track0A1, RZ;\n", j3c32 => "--:-:-:-:1 \@P3 IADD track0B0.CC, track0B0, 2x<64>;\n", j3c37 => "--:-:-:-:1 \@P3 IADD.X track0B1, track0B1, RZ;\n", j4c32 => "--:-:-:-:1 \@P4 IADD track1B0.CC, track1B0, 2x<64>;\n", j4c37 => "--:-:-:-:1 \@P4 IADD.X track1B1, track1B1, RZ;\n", j5c32 => "--:-:-:-:1 \@P5 IADD track2B0.CC, track2B0, 2x<64>;\n", j5c37 => "--:-:-:-:1 \@P5 IADD.X track2B1, track2B1, RZ;\n", j6c32 => "--:-:-:-:1 \@P6 IADD track3B0.CC, track3B0, 2x<64>;\n", j6c37 => "--:-:-:-:1 \@P6 IADD.X track3B1, track3B1, RZ;\n", j6c63 => "--:-:-:-:5 BAR.SYNC 0;\n" . "--:-:-:-:1 \@P0 IADD readAs, readAs, -swapBuf;\n" . "--:-:-:-:1 \@P0 IADD readBs, readBs, -swapBuf;\n" . "--:-:-:-:1 \@P0 IADD writeAs, writeAs, swapBuf;\n" . "--:-:-:-:1 \@P0 IADD writeBs, writeBs, swapBuf;\n" . "--:-:-:-:1 \@P0 IADD swapBuf, RZ, -swapBuf;\n", (vector() ? ( j1c35 => "02:-:-:-:1 \@P0 F2F.F32.F16 load0A7, load0A3.H1;\n", j1c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A6, load0A3.H0;\n", j1c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A5, load0A2.H1;\n", j1c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A4, load0A2.H0;\n", j1c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A3, load0A1.H1;\n", j1c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A2, load0A1.H0;\n", j1c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A1, load0A0.H1;\n", j1c63 => "--:-:2:-:1 \@P0 F2F.F32.F16 load0A0, load0A0.H0;\n", j2c36 => "04:-:-:-:1 \@P0 F2F.F32.F16 load0B7, load0B3.H1;\n", j2c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B6, load0B3.H0;\n", j2c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B5, load0B2.H1;\n", j2c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B4, load0B2.H0;\n", j2c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B3, load0B1.H1;\n", j2c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B2, load0B1.H0;\n", j2c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B1, load0B0.H1;\n", j2c63 => "--:-:3:-:1 \@P0 F2F.F32.F16 load0B0, load0B0.H0;\n", j3c36 => "08:-:-:-:1 \@P0 F2F.F32.F16 load1B7, load1B3.H1;\n", j3c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B6, load1B3.H0;\n", j3c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B5, load1B2.H1;\n", j3c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B4, load1B2.H0;\n", j3c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B3, load1B1.H1;\n", j3c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B2, load1B1.H0;\n", j3c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B1, load1B0.H1;\n", j3c63 => "--:-:4:-:1 \@P0 F2F.F32.F16 load1B0, load1B0.H0;\n", j4c36 => "10:-:-:-:1 \@P0 F2F.F32.F16 load2B7, load2B3.H1;\n", j4c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B6, load2B3.H0;\n", j4c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B5, load2B2.H1;\n", j4c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B4, load2B2.H0;\n", j4c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B3, load2B1.H1;\n", j4c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B2, load2B1.H0;\n", j4c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B1, load2B0.H1;\n", j4c63 => "--:-:5:-:1 \@P0 F2F.F32.F16 load2B0, load2B0.H0;\n", j5c36 => "20:-:-:-:1 \@P0 F2F.F32.F16 load3B7, load3B3.H1;\n", j5c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B6, load3B3.H0;\n", j5c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B5, load3B2.H1;\n", j5c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B4, load3B2.H0;\n", j5c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B3, load3B1.H1;\n", j5c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B2, load3B1.H0;\n", j5c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B1, load3B0.H1;\n", j5c63 => "--:-:6:-:1 \@P0 F2F.F32.F16 load3B0, load3B0.H0;\n", j2c16 => "02:-:-:-:1 \@P0 STS [writeAs + 4x<7*16>], load0A7;\n", j2c18 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<6*16>], load0A6;\n", j2c20 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<5*16>], load0A5;\n", j2c22 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<4*16>], load0A4;\n", j2c24 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<3*16>], load0A3;\n", j2c26 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<2*16>], load0A2;\n", j2c28 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<1*16>], load0A1;\n", j2c30 => "--:2:-:-:1 \@P0 STS [writeAs + 4x<0*16>], load0A0;\n", j3c16 => "04:-:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 0*16>], load0B7;\n", j3c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 0*16>], load0B6;\n", j3c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 0*16>], load0B5;\n", j3c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 0*16>], load0B4;\n", j3c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 0*16>], load0B3;\n", j3c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 0*16>], load0B2;\n", j3c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 0*16>], load0B1;\n", j3c30 => "--:3:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 0*16>], load0B0;\n", j4c16 => "08:-:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 1*16>], load1B7;\n", j4c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 1*16>], load1B6;\n", j4c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 1*16>], load1B5;\n", j4c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 1*16>], load1B4;\n", j4c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 1*16>], load1B3;\n", j4c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 1*16>], load1B2;\n", j4c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 1*16>], load1B1;\n", j4c30 => "--:4:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 1*16>], load1B0;\n", j5c16 => "10:-:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 2*16>], load2B7;\n", j5c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 2*16>], load2B6;\n", j5c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 2*16>], load2B5;\n", j5c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 2*16>], load2B4;\n", j5c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 2*16>], load2B3;\n", j5c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 2*16>], load2B2;\n", j5c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 2*16>], load2B1;\n", j5c30 => "--:5:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 2*16>], load2B0;\n", j6c16 => "20:-:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 3*16>], load3B7;\n", j6c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 3*16>], load3B6;\n", j6c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 3*16>], load3B5;\n", j6c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 3*16>], load3B4;\n", j6c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 3*16>], load3B3;\n", j6c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 3*16>], load3B2;\n", j6c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 3*16>], load3B1;\n", j6c30 => "--:6:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 3*16>], load3B0;\n", j2c61 => "02:-:2:-:1 \@P2 LDG.E.CI.128 load0A, [track0A];\n", j3c61 => "04:-:3:-:1 \@P3 LDG.E.CI.128 load0B, [track0B];\n", j4c61 => "08:-:4:-:1 \@P4 LDG.E.CI.128 load1B, [track1B];\n", j5c61 => "10:-:5:-:1 \@P5 LDG.E.CI.128 load2B, [track2B];\n", j6c61 => "20:-:6:-:1 \@P6 LDG.E.CI.128 load3B, [track3B];\n", ) : ( j1c35 => "02:-:-:-:1 \@P0 F2F.F32.F16 load0A0, load0A0;\n", j1c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A1, load0A1;\n", j1c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A2, load0A2;\n", j1c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A3, load0A3;\n", j1c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A4, load0A4;\n", j1c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A5, load0A5;\n", j1c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0A6, load0A6;\n", j1c63 => "--:-:2:-:1 \@P0 F2F.F32.F16 load0A7, load0A7;\n", j2c36 => "04:-:-:-:1 \@P0 F2F.F32.F16 load0B0, load0B0;\n", j2c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B1, load0B1;\n", j2c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B2, load0B2;\n", j2c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B3, load0B3;\n", j2c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B4, load0B4;\n", j2c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B5, load0B5;\n", j2c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load0B6, load0B6;\n", j2c63 => "--:-:3:-:1 \@P0 F2F.F32.F16 load0B7, load0B7;\n", j3c36 => "08:-:-:-:1 \@P0 F2F.F32.F16 load1B0, load1B0;\n", j3c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B1, load1B1;\n", j3c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B2, load1B2;\n", j3c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B3, load1B3;\n", j3c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B4, load1B4;\n", j3c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B5, load1B5;\n", j3c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load1B6, load1B6;\n", j3c63 => "--:-:4:-:1 \@P0 F2F.F32.F16 load1B7, load1B7;\n", j4c36 => "10:-:-:-:1 \@P0 F2F.F32.F16 load2B0, load2B0;\n", j4c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B1, load2B1;\n", j4c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B2, load2B2;\n", j4c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B3, load2B3;\n", j4c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B4, load2B4;\n", j4c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B5, load2B5;\n", j4c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load2B6, load2B6;\n", j4c63 => "--:-:5:-:1 \@P0 F2F.F32.F16 load2B7, load2B7;\n", j5c36 => "20:-:-:-:1 \@P0 F2F.F32.F16 load3B0, load3B0;\n", j5c39 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B1, load3B1;\n", j5c43 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B2, load3B2;\n", j5c47 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B3, load3B3;\n", j5c51 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B4, load3B4;\n", j5c55 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B5, load3B5;\n", j5c59 => "--:-:-:-:1 \@P0 F2F.F32.F16 load3B6, load3B6;\n", j5c63 => "--:-:6:-:1 \@P0 F2F.F32.F16 load3B7, load3B7;\n", j2c16 => "02:-:-:-:1 \@P0 STS [writeAs + 4x<0*16>], load0A0;\n", j2c18 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<1*16>], load0A1;\n", j2c20 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<2*16>], load0A2;\n", j2c22 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<3*16>], load0A3;\n", j2c24 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<4*16>], load0A4;\n", j2c26 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<5*16>], load0A5;\n", j2c28 => "--:-:-:-:1 \@P0 STS [writeAs + 4x<6*16>], load0A6;\n", j2c30 => "--:2:-:-:1 \@P0 STS [writeAs + 4x<7*16>], load0A7;\n", j3c16 => "04:-:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 0*16>], load0B0;\n", j3c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 0*16>], load0B1;\n", j3c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 0*16>], load0B2;\n", j3c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 0*16>], load0B3;\n", j3c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 0*16>], load0B4;\n", j3c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 0*16>], load0B5;\n", j3c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 0*16>], load0B6;\n", j3c30 => "--:3:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 0*16>], load0B7;\n", j4c16 => "08:-:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 1*16>], load1B0;\n", j4c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 1*16>], load1B1;\n", j4c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 1*16>], load1B2;\n", j4c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 1*16>], load1B3;\n", j4c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 1*16>], load1B4;\n", j4c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 1*16>], load1B5;\n", j4c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 1*16>], load1B6;\n", j4c30 => "--:4:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 1*16>], load1B7;\n", j5c16 => "10:-:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 2*16>], load2B0;\n", j5c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 2*16>], load2B1;\n", j5c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 2*16>], load2B2;\n", j5c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 2*16>], load2B3;\n", j5c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 2*16>], load2B4;\n", j5c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 2*16>], load2B5;\n", j5c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 2*16>], load2B6;\n", j5c30 => "--:5:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 2*16>], load2B7;\n", j6c16 => "20:-:-:-:1 \@P0 STS [writeBs + 4x<0*64 + 3*16>], load3B0;\n", j6c18 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<1*64 + 3*16>], load3B1;\n", j6c20 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<2*64 + 3*16>], load3B2;\n", j6c22 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<3*64 + 3*16>], load3B3;\n", j6c24 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<4*64 + 3*16>], load3B4;\n", j6c26 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<5*64 + 3*16>], load3B5;\n", j6c28 => "--:-:-:-:1 \@P0 STS [writeBs + 4x<6*64 + 3*16>], load3B6;\n", j6c30 => "--:6:-:-:1 \@P0 STS [writeBs + 4x<7*64 + 3*16>], load3B7;\n", j2c48 => "02:-:-:-:1 \@P2 LDG.E.CI.U16 load0A0, [track0A + 2x<0>];\n", j2c50 => "--:-:-:-:1 \@P2 LDG.E.CI.U16 load0A1, [track0A + 2x<1>];\n", j2c52 => "--:-:-:-:1 \@P2 LDG.E.CI.U16 load0A2, [track0A + 2x<2>];\n", j2c54 => "--:-:-:-:1 \@P2 LDG.E.CI.U16 load0A3, [track0A + 2x<3>];\n", j2c56 => "--:-:-:-:1 \@P2 LDG.E.CI.U16 load0A4, [track0A + 2x<4>];\n", j2c58 => "--:-:-:-:1 \@P2 LDG.E.CI.U16 load0A5, [track0A + 2x<5>];\n", j2c60 => "--:-:-:-:1 \@P2 LDG.E.CI.U16 load0A6, [track0A + 2x<6>];\n", j2c62 => "--:-:2:-:1 \@P2 LDG.E.CI.U16 load0A7, [track0A + 2x<7>];\n", j3c48 => "04:-:-:-:1 \@P3 LDG.E.CI.U16 load0B0, [track0B + 2x<0>];\n", j3c50 => "--:-:-:-:1 \@P3 LDG.E.CI.U16 load0B1, [track0B + 2x<1>];\n", j3c52 => "--:-:-:-:1 \@P3 LDG.E.CI.U16 load0B2, [track0B + 2x<2>];\n", j3c54 => "--:-:-:-:1 \@P3 LDG.E.CI.U16 load0B3, [track0B + 2x<3>];\n", j3c56 => "--:-:-:-:1 \@P3 LDG.E.CI.U16 load0B4, [track0B + 2x<4>];\n", j3c58 => "--:-:-:-:1 \@P3 LDG.E.CI.U16 load0B5, [track0B + 2x<5>];\n", j3c60 => "--:-:-:-:1 \@P3 LDG.E.CI.U16 load0B6, [track0B + 2x<6>];\n", j3c62 => "--:-:3:-:1 \@P3 LDG.E.CI.U16 load0B7, [track0B + 2x<7>];\n", j4c48 => "08:-:-:-:1 \@P4 LDG.E.CI.U16 load1B0, [track1B + 2x<0>];\n", j4c50 => "--:-:-:-:1 \@P4 LDG.E.CI.U16 load1B1, [track1B + 2x<1>];\n", j4c52 => "--:-:-:-:1 \@P4 LDG.E.CI.U16 load1B2, [track1B + 2x<2>];\n", j4c54 => "--:-:-:-:1 \@P4 LDG.E.CI.U16 load1B3, [track1B + 2x<3>];\n", j4c56 => "--:-:-:-:1 \@P4 LDG.E.CI.U16 load1B4, [track1B + 2x<4>];\n", j4c58 => "--:-:-:-:1 \@P4 LDG.E.CI.U16 load1B5, [track1B + 2x<5>];\n", j4c60 => "--:-:-:-:1 \@P4 LDG.E.CI.U16 load1B6, [track1B + 2x<6>];\n", j4c62 => "--:-:4:-:1 \@P4 LDG.E.CI.U16 load1B7, [track1B + 2x<7>];\n", j5c48 => "10:-:-:-:1 \@P5 LDG.E.CI.U16 load2B0, [track2B + 2x<0>];\n", j5c50 => "--:-:-:-:1 \@P5 LDG.E.CI.U16 load2B1, [track2B + 2x<1>];\n", j5c52 => "--:-:-:-:1 \@P5 LDG.E.CI.U16 load2B2, [track2B + 2x<2>];\n", j5c54 => "--:-:-:-:1 \@P5 LDG.E.CI.U16 load2B3, [track2B + 2x<3>];\n", j5c56 => "--:-:-:-:1 \@P5 LDG.E.CI.U16 load2B4, [track2B + 2x<4>];\n", j5c58 => "--:-:-:-:1 \@P5 LDG.E.CI.U16 load2B5, [track2B + 2x<5>];\n", j5c60 => "--:-:-:-:1 \@P5 LDG.E.CI.U16 load2B6, [track2B + 2x<6>];\n", j5c62 => "--:-:5:-:1 \@P5 LDG.E.CI.U16 load2B7, [track2B + 2x<7>];\n", j6c48 => "20:-:-:-:1 \@P6 LDG.E.CI.U16 load3B0, [track3B + 2x<0>];\n", j6c50 => "--:-:-:-:1 \@P6 LDG.E.CI.U16 load3B1, [track3B + 2x<1>];\n", j6c52 => "--:-:-:-:1 \@P6 LDG.E.CI.U16 load3B2, [track3B + 2x<2>];\n", j6c54 => "--:-:-:-:1 \@P6 LDG.E.CI.U16 load3B3, [track3B + 2x<3>];\n", j6c56 => "--:-:-:-:1 \@P6 LDG.E.CI.U16 load3B4, [track3B + 2x<4>];\n", j6c58 => "--:-:-:-:1 \@P6 LDG.E.CI.U16 load3B5, [track3B + 2x<5>];\n", j6c60 => "--:-:-:-:1 \@P6 LDG.E.CI.U16 load3B6, [track3B + 2x<6>];\n", j6c62 => "--:-:6:-:1 \@P6 LDG.E.CI.U16 load3B7, [track3B + 2x<7>];\n", ) ), j7c63 => "--:-:-:Y:5 \@P0 BRA.U LOOP;\n", ); my @cOrder; my @swirl = ([0,2],[1,2],[1,0],[0,0]); my @y = (0,1,4,5); foreach my $x (0,2,4,6) { foreach my $y (@y) { push @cOrder, [$x + $_->[0], $y + $_->[1]] foreach @swirl; } @y = reverse @y; } my $out = ''; foreach my $j (0 .. 7) { my $odd = $j & 1; my $nOdd = !$odd + 0; my $rsOffset = ($j + 1) % 8; my $rsPred = $j == 7 ? '@P0' : ' '; $insert{"j${j}c0"} = sprintf "--:-:-:-:1 %s LDS.U.128 j%dAy0, [readAs + 4x<%d*16 + 00>];\n", $rsPred, $nOdd, $rsOffset; $insert{"j${j}c2"} = sprintf "--:-:-:-:1 %s LDS.U.128 j%dBx0, [readBs + 4x<%d*64 + 00>];\n", $rsPred, $nOdd, $rsOffset; $insert{"j${j}c4"} = sprintf "--:-:-:-:1 %s LDS.U.128 j%dAy4, [readAs + 4x<%d*16 + 08>];\n", $rsPred, $nOdd, $rsOffset; $insert{"j${j}c6"} = sprintf "--:-:1:-:1 %s LDS.U.128 j%dBx4, [readBs + 4x<%d*64 + 32>];\n", $rsPred, $nOdd, $rsOffset; foreach my $c (0 .. 63) { my ($x,$y) = @{$cOrder[$c]}; my $ins = $insert{"j${j}c$c"} || ''; my $stall = $ins =~ /LDS|I2I|I2F|F2I|F2F|LDG|STS|BAR|BRA/ ? 0 : 1; my $yield = $c == 32 && $stall ? 'Y' : '-'; my $wait = $c == 0 ? '01' : '--'; my $ctrl = "$wait:-:-:$yield:$stall"; $out .= sprintf "%s FFMA cx%dy%d, j%dBx%d, j%dAy%d, cx%dy%d;\n%s", $ctrl, $x,$y, $odd,$x, $odd,$y, $x,$y, $ins; } } return $out; +] <SCHEDULE_BLOCK> --:-:-:-:1 MOV alpha, param_alpha; --:-:-:-:1 MOV beta, param_beta; // readCs = ((tid & 15) * 4 + (tid / 16) * 64) * 4 --:-:-:-:1 LOP.AND tid15, tid, 15; --:-:-:-:1 SHR.U32 tid16, tid, 4; --:-:-:-:1 SHL tid15, tid15, 2; --:-:-:-:1 ISCADD readCs, tid16, tid15, 6; --:-:-:-:1 SHL readCs, readCs, 2; // cx = idx_B*64 + tid15; --:-:-:-:1 ISCADD cx, idx_B, tid15, 6; --:-:-:-:1 IADD cx1, cx, 1; --:-:-:-:1 IADD cx2, cx, 2; --:-:-:-:1 IADD cx3, cx, 3; // cy = idx_A*16 + tid16 --:-:-:-:1 ISCADD cy, idx_A, tid16, 4; // C += (cy*cdc + cx) * 2; --:-:-:-:1 MOV cdc, param_cdc; --:-:-:-:1 SHL cdc8, cdc, 4; --:-:-:-:1 XMAD.LO ci, cy, cdc, cx, xmad_c; --:-:-:-:1 LEA C0.CC, ci, param_C[0], 1; --:-:-:-:1 LEA.HI.X C1, ci, param_C[1], RZ, 1; // P0 = cx < n --:-:-:-:1 ISETP.LT.AND P0, PT, cx, param_n, PT; --:-:-:-:1 ISETP.LT.AND P1, PT, cx1, param_n, PT; --:-:-:-:1 ISETP.LT.AND P2, PT, cx2, param_n, PT; --:-:-:-:1 ISETP.LT.AND P3, PT, cx3, param_n, PT; --:-:-:-:1 P2R preds, PR, RZ, 0x0f; // P4 = cy < m --:-:-:-:1 ISETP.LT.AND P4, PT, cy, param_m, PT; // P5 = beta != 0 && P4 --:-:-:-:1 ISETP.NE.AND P5, PT, beta, RZ, P4; // Init beta preds --:-:-:-:1 @P5 R2P PR, preds, 0x0f; --:-:-:-:1 @!P5 R2P PR, RZ, 0x0f; </SCHEDULE_BLOCK> --:-:-:-:1 FMUL shuffle_x0y0, cx0y0, alpha; --:-:-:-:1 FMUL shuffle_x1y0, cx1y0, alpha; --:-:-:-:1 FMUL shuffle_x2y0, cx2y0, alpha; --:-:-:-:1 FMUL shuffle_x3y0, cx3y0, alpha; --:-:-:-:1 FMUL shuffle_x4y0, cx4y0, alpha; --:-:-:-:1 FMUL shuffle_x5y0, cx5y0, alpha; --:-:-:-:1 FMUL shuffle_x6y0, cx6y0, alpha; --:-:-:-:0 FMUL shuffle_x7y0, cx7y0, alpha; --:-:-:-:1 STS.128 [writeCs+4x<0*64 + 00>], shuffle_x0y0; --:-:-:-:1 FMUL shuffle_x0y1, cx0y1, alpha; --:-:-:-:1 FMUL shuffle_x1y1, cx1y1, alpha; --:-:-:-:1 FMUL shuffle_x2y1, cx2y1, alpha; --:-:-:-:0 FMUL shuffle_x3y1, cx3y1, alpha; --:-:-:-:1 STS.128 [writeCs+4x<0*64 + 32>], shuffle_x4y0; --:-:-:-:1 FMUL shuffle_x4y1, cx4y1, alpha; --:-:-:-:1 FMUL shuffle_x5y1, cx5y1, alpha; --:-:-:-:1 FMUL shuffle_x6y1, cx6y1, alpha; --:-:-:-:0 FMUL shuffle_x7y1, cx7y1, alpha; --:-:-:-:1 STS.128 [writeCs+4x<1*64 + 00>], shuffle_x0y1; --:-:-:-:1 FMUL shuffle_x0y2, cx0y2, alpha; --:-:-:-:1 FMUL shuffle_x1y2, cx1y2, alpha; --:-:-:-:1 FMUL shuffle_x2y2, cx2y2, alpha; --:-:-:-:0 FMUL shuffle_x3y2, cx3y2, alpha; --:-:-:-:1 STS.128 [writeCs+4x<1*64 + 32>], shuffle_x4y1; --:-:-:-:1 FMUL shuffle_x4y2, cx4y2, alpha; --:-:-:-:1 FMUL shuffle_x5y2, cx5y2, alpha; --:-:-:-:1 FMUL shuffle_x6y2, cx6y2, alpha; --:-:-:-:0 FMUL shuffle_x7y2, cx7y2, alpha; --:-:-:-:1 STS.128 [writeCs+4x<2*64 + 00>], shuffle_x0y2; --:-:-:-:1 FMUL shuffle_x0y3, cx0y3, alpha; --:-:-:-:1 FMUL shuffle_x1y3, cx1y3, alpha; --:-:-:-:1 FMUL shuffle_x2y3, cx2y3, alpha; --:-:-:-:0 FMUL shuffle_x3y3, cx3y3, alpha; --:-:-:-:1 STS.128 [writeCs+4x<2*64 + 32>], shuffle_x4y2; --:-:-:-:1 FMUL shuffle_x4y3, cx4y3, alpha; --:-:-:-:1 FMUL shuffle_x5y3, cx5y3, alpha; --:-:-:-:1 FMUL shuffle_x6y3, cx6y3, alpha; --:-:-:-:0 FMUL shuffle_x7y3, cx7y3, alpha; --:-:-:-:4 STS.128 [writeCs+4x<3*64 + 00>], shuffle_x0y3; --:-:-:-:1 STS.128 [writeCs+4x<3*64 + 32>], shuffle_x4y3; --:-:-:-:5 BAR.SYNC 0; --:-:-:-:5 CAL STORE_C; --:-:-:-:1 FMUL shuffle_x0y4, cx0y4, alpha; --:-:-:-:1 FMUL shuffle_x1y4, cx1y4, alpha; --:-:-:-:1 FMUL shuffle_x2y4, cx2y4, alpha; --:-:-:-:1 FMUL shuffle_x3y4, cx3y4, alpha; --:-:-:-:1 FMUL shuffle_x4y4, cx4y4, alpha; --:-:-:-:1 FMUL shuffle_x5y4, cx5y4, alpha; --:-:-:-:0 FMUL shuffle_x6y4, cx6y4, alpha; --:-:-:-:5 BAR.SYNC 0; --:-:-:-:0 FMUL shuffle_x7y4, cx7y4, alpha; --:-:-:-:1 STS.128 [writeCs+4x<0*64 + 00>], shuffle_x0y4; --:-:-:-:1 FMUL shuffle_x0y5, cx0y5, alpha; --:-:-:-:1 FMUL shuffle_x1y5, cx1y5, alpha; --:-:-:-:1 FMUL shuffle_x2y5, cx2y5, alpha; --:-:-:-:0 FMUL shuffle_x3y5, cx3y5, alpha; --:-:-:-:1 STS.128 [writeCs+4x<0*64 + 32>], shuffle_x4y4; --:-:-:-:1 FMUL shuffle_x4y5, cx4y5, alpha; --:-:-:-:1 FMUL shuffle_x5y5, cx5y5, alpha; --:-:-:-:1 FMUL shuffle_x6y5, cx6y5, alpha; --:-:-:-:0 FMUL shuffle_x7y5, cx7y5, alpha; --:-:-:-:1 STS.128 [writeCs+4x<1*64 + 00>], shuffle_x0y5; --:-:-:-:1 FMUL shuffle_x0y6, cx0y6, alpha; --:-:-:-:1 FMUL shuffle_x1y6, cx1y6, alpha; --:-:-:-:1 FMUL shuffle_x2y6, cx2y6, alpha; --:-:-:-:0 FMUL shuffle_x3y6, cx3y6, alpha; --:-:-:-:1 STS.128 [writeCs+4x<1*64 + 32>], shuffle_x4y5; --:-:-:-:1 FMUL shuffle_x4y6, cx4y6, alpha; --:-:-:-:1 FMUL shuffle_x5y6, cx5y6, alpha; --:-:-:-:1 FMUL shuffle_x6y6, cx6y6, alpha; --:-:-:-:0 FMUL shuffle_x7y6, cx7y6, alpha; --:-:-:-:1 STS.128 [writeCs+4x<2*64 + 00>], shuffle_x0y6; --:-:-:-:1 FMUL shuffle_x0y7, cx0y7, alpha; --:-:-:-:1 FMUL shuffle_x1y7, cx1y7, alpha; --:-:-:-:1 FMUL shuffle_x2y7, cx2y7, alpha; --:-:-:-:0 FMUL shuffle_x3y7, cx3y7, alpha; --:-:-:-:1 STS.128 [writeCs+4x<2*64 + 32>], shuffle_x4y6; --:-:-:-:1 FMUL shuffle_x4y7, cx4y7, alpha; --:-:-:-:1 FMUL shuffle_x5y7, cx5y7, alpha; --:-:-:-:1 FMUL shuffle_x6y7, cx6y7, alpha; --:-:-:-:0 FMUL shuffle_x7y7, cx7y7, alpha; --:-:-:-:4 STS.128 [writeCs+4x<3*64 + 00>], shuffle_x0y7; --:-:-:-:1 STS.128 [writeCs+4x<3*64 + 32>], shuffle_x4y7; --:-:-:-:5 BAR.SYNC 0; --:-:-:-:5 CAL STORE_C; --:-:-:-:5 EXIT; STORE_C: [+ return vector() ? q{ --:-:1:-:1 @P0 LDG.E.64 loadC, [C]; } : q{ --:-:-:-:0 @!P0 MOV loadC0, RZ; --:-:-:-:1 @P0 LDG.E.CI.U16 loadC0, [C + 2x<0>]; --:-:-:-:0 @!P1 MOV loadC1, RZ; --:-:-:-:1 @P1 LDG.E.CI.U16 loadC1, [C + 2x<1>]; --:-:-:-:0 @!P2 MOV loadC2, RZ; --:-:-:-:1 @P2 LDG.E.CI.U16 loadC2, [C + 2x<2>]; --:-:-:-:0 @!P3 MOV loadC3, RZ; --:-:1:-:1 @P3 LDG.E.CI.U16 loadC3, [C + 2x<3>]; }; +] // Restore output preds --:-:-:-:1 @P4 R2P PR, preds, 0x0f; --:-:-:-:1 @!P4 R2P PR, RZ, 0x0f; --:-:-:-:1 LDS.U.128 part0C, [readCs + 4x<0*8*64>]; --:-:2:-:1 LDS.U.128 part1C, [readCs + 4x<1*8*64>]; --:-:-:-:1 LDS.U.128 part2C, [readCs + 4x<2*8*64>]; --:-:3:-:1 LDS.U.128 part3C, [readCs + 4x<3*8*64>]; --:-:-:-:1 LDS.U.128 part4C, [readCs + 4x<4*8*64>]; --:-:4:-:1 LDS.U.128 part5C, [readCs + 4x<5*8*64>]; --:-:-:-:1 LDS.U.128 part6C, [readCs + 4x<6*8*64>]; --:-:5:-:1 LDS.U.128 part7C, [readCs + 4x<7*8*64>]; <SCHEDULE_BLOCK> 02:-:-:-:1 @P0 FADD part0C0, part0C0, part1C0; --:-:-:-:1 @P1 FADD part0C1, part0C1, part1C1; --:-:-:-:1 @P2 FADD part0C2, part0C2, part1C2; --:-:-:-:1 @P3 FADD part0C3, part0C3, part1C3; 04:-:-:-:1 @P0 FADD part2C0, part2C0, part3C0; --:-:-:-:1 @P1 FADD part2C1, part2C1, part3C1; --:-:-:-:1 @P2 FADD part2C2, part2C2, part3C2; --:-:-:-:1 @P3 FADD part2C3, part2C3, part3C3; 08:-:-:-:1 @P0 FADD part4C0, part4C0, part5C0; --:-:-:-:1 @P1 FADD part4C1, part4C1, part5C1; --:-:-:-:1 @P2 FADD part4C2, part4C2, part5C2; --:-:-:-:1 @P3 FADD part4C3, part4C3, part5C3; 10:-:-:-:1 @P0 FADD part6C0, part6C0, part7C0; --:-:-:-:1 @P1 FADD part6C1, part6C1, part7C1; --:-:-:-:1 @P2 FADD part6C2, part6C2, part7C2; --:-:-:-:1 @P3 FADD part6C3, part6C3, part7C3; --:-:-:-:1 @P0 FADD part0C0, part0C0, part2C0; --:-:-:-:1 @P1 FADD part0C1, part0C1, part2C1; --:-:-:-:1 @P2 FADD part0C2, part0C2, part2C2; --:-:-:-:1 @P3 FADD part0C3, part0C3, part2C3; --:-:-:-:1 @P0 FADD part4C0, part4C0, part6C0; --:-:-:-:1 @P1 FADD part4C1, part4C1, part6C1; --:-:-:-:1 @P2 FADD part4C2, part4C2, part6C2; --:-:-:-:1 @P3 FADD part4C3, part4C3, part6C3; --:-:-:-:1 @P0 FADD c0, part0C0, part4C0; --:-:-:-:1 @P1 FADD c1, part0C1, part4C1; --:-:-:-:1 @P2 FADD c2, part0C2, part4C2; --:-:-:-:1 @P3 FADD c3, part0C3, part4C3; </SCHEDULE_BLOCK> --:-:-:-:0 IADD cy, cy, 8; [+ return vector() ? q{ 01:-:1:-:1 @P5 F2F.F32.F16 b0, loadC0.H0; --:-:2:-:1 @P5 F2F.F32.F16 b1, loadC0.H1; --:-:3:-:1 @P5 F2F.F32.F16 b2, loadC1.H0; --:-:4:-:1 @P5 F2F.F32.F16 b3, loadC1.H1; } : q{ 01:-:1:-:1 @P5 F2F.F32.F16 b0, loadC0; --:-:2:-:1 @P5 F2F.F32.F16 b1, loadC1; --:-:3:-:1 @P5 F2F.F32.F16 b2, loadC2; --:-:4:-:1 @P5 F2F.F32.F16 b3, loadC3; }; +] 01:-:-:-:1 @P5 FFMA c0, b0, beta, c0; 02:-:-:-:1 @P5 FFMA c1, b1, beta, c1; 04:-:-:-:1 @P5 FFMA c2, b2, beta, c2; 08:-:-:-:1 @P5 FFMA c3, b3, beta, c3; --:-:-:-:0 ISETP.LT.AND P5, PT, cy, param_m, P5; --:-:1:-:1 @P0 F2F.F16.F32 c0, c0; --:-:2:-:1 @P1 F2F.F16.F32 c1, c1; --:-:-:-:0 ISETP.LT.AND P4, PT, cy, param_m, PT; --:-:3:-:1 @P2 F2F.F16.F32 c2, c2; --:-:4:-:1 @P3 F2F.F16.F32 c3, c3; [+ return vector() ? q{ 03:-:-:-:2 @P0 BFI c0, c1, 0x1010, c0; 0c:-:-:-:2 @P0 BFI c1, c3, 0x1010, c2; --:1:-:-:1 @P0 STG.E.CG.64 [C], c; } : q{ 01:-:-:-:1 @P0 STG.E.U16 [C + 2x<0>], c0; 02:-:-:-:1 @P1 STG.E.U16 [C + 2x<1>], c1; 04:-:-:-:1 @P2 STG.E.U16 [C + 2x<2>], c2; 08:1:-:-:1 @P3 STG.E.U16 [C + 2x<3>], c3; }; +] // Restore beta preds --:-:-:-:1 @P5 R2P PR, preds, 0x0f; --:-:-:-:1 @!P5 R2P PR, RZ, 0x0f; 01:-:-:-:6 IADD C0.CC, C0, cdc8; --:-:-:-:0 IADD.X C1, C1, RZ; --:-:-:-:5 RET;

sass/hgemm_16x64x64_NT.sass (995 lines of code) (raw):