Skip to content

Commit dfc11ef

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #4791 from ChipKerchner/vectorizeSBGEMMincopy
Vectorize SBGEMM incopy for Power10 - 4x faster.
2 parents b70227a + f708944 commit dfc11ef

2 files changed

Lines changed: 139 additions & 35 deletions

File tree

‎kernel/power/sbgemm_ncopy_16_power10.c‎

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,11 @@
3737
/*********************************************************************/
3838

3939
#include <stdio.h>
40+
#include <altivec.h>
4041
#include "common.h"
4142

43+
typedef uint32_t vec_bf16x2 __attribute__ ((vector_size (16)));
44+
4245
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
4346
BLASLONG i, j;
4447

@@ -82,7 +85,84 @@ int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
8285
aoffset16 = aoffset15 + lda;
8386
aoffset += 16 * lda;
8487

85-
i = (m >> 1);
88+
i = (m >> 3);
89+
if (i > 0) {
90+
do {
91+
vec_bf16x2 vtemp01 = *(vec_bf16x2 *)(aoffset1);
92+
vec_bf16x2 vtemp02 = *(vec_bf16x2 *)(aoffset2);
93+
vec_bf16x2 vtemp03 = *(vec_bf16x2 *)(aoffset3);
94+
vec_bf16x2 vtemp04 = *(vec_bf16x2 *)(aoffset4);
95+
vec_bf16x2 vtemp05 = *(vec_bf16x2 *)(aoffset5);
96+
vec_bf16x2 vtemp06 = *(vec_bf16x2 *)(aoffset6);
97+
vec_bf16x2 vtemp07 = *(vec_bf16x2 *)(aoffset7);
98+
vec_bf16x2 vtemp08 = *(vec_bf16x2 *)(aoffset8);
99+
vec_bf16x2 vtemp09 = *(vec_bf16x2 *)(aoffset9);
100+
vec_bf16x2 vtemp10 = *(vec_bf16x2 *)(aoffset10);
101+
vec_bf16x2 vtemp11 = *(vec_bf16x2 *)(aoffset11);
102+
vec_bf16x2 vtemp12 = *(vec_bf16x2 *)(aoffset12);
103+
vec_bf16x2 vtemp13 = *(vec_bf16x2 *)(aoffset13);
104+
vec_bf16x2 vtemp14 = *(vec_bf16x2 *)(aoffset14);
105+
vec_bf16x2 vtemp15 = *(vec_bf16x2 *)(aoffset15);
106+
vec_bf16x2 vtemp16 = *(vec_bf16x2 *)(aoffset16);
107+
108+
vec_bf16x2 vtemp17 = vec_mergeh(vtemp01, vtemp03);
109+
vec_bf16x2 vtemp18 = vec_mergel(vtemp01, vtemp03);
110+
vec_bf16x2 vtemp19 = vec_mergeh(vtemp02, vtemp04);
111+
vec_bf16x2 vtemp20 = vec_mergel(vtemp02, vtemp04);
112+
vec_bf16x2 vtemp21 = vec_mergeh(vtemp05, vtemp07);
113+
vec_bf16x2 vtemp22 = vec_mergel(vtemp05, vtemp07);
114+
vec_bf16x2 vtemp23 = vec_mergeh(vtemp06, vtemp08);
115+
vec_bf16x2 vtemp24 = vec_mergel(vtemp06, vtemp08);
116+
vec_bf16x2 vtemp25 = vec_mergeh(vtemp09, vtemp11);
117+
vec_bf16x2 vtemp26 = vec_mergel(vtemp09, vtemp11);
118+
vec_bf16x2 vtemp27 = vec_mergeh(vtemp10, vtemp12);
119+
vec_bf16x2 vtemp28 = vec_mergel(vtemp10, vtemp12);
120+
vec_bf16x2 vtemp29 = vec_mergeh(vtemp13, vtemp15);
121+
vec_bf16x2 vtemp30 = vec_mergel(vtemp13, vtemp15);
122+
vec_bf16x2 vtemp31 = vec_mergeh(vtemp14, vtemp16);
123+
vec_bf16x2 vtemp32 = vec_mergel(vtemp14, vtemp16);
124+
125+
*(vec_bf16x2 *)(boffset + 0) = vec_mergeh(vtemp17, vtemp19);
126+
*(vec_bf16x2 *)(boffset + 8) = vec_mergeh(vtemp21, vtemp23);
127+
*(vec_bf16x2 *)(boffset + 16) = vec_mergeh(vtemp25, vtemp27);
128+
*(vec_bf16x2 *)(boffset + 24) = vec_mergeh(vtemp29, vtemp31);
129+
*(vec_bf16x2 *)(boffset + 32) = vec_mergel(vtemp17, vtemp19);
130+
*(vec_bf16x2 *)(boffset + 40) = vec_mergel(vtemp21, vtemp23);
131+
*(vec_bf16x2 *)(boffset + 48) = vec_mergel(vtemp25, vtemp27);
132+
*(vec_bf16x2 *)(boffset + 56) = vec_mergel(vtemp29, vtemp31);
133+
*(vec_bf16x2 *)(boffset + 64) = vec_mergeh(vtemp18, vtemp20);
134+
*(vec_bf16x2 *)(boffset + 72) = vec_mergeh(vtemp22, vtemp24);
135+
*(vec_bf16x2 *)(boffset + 80) = vec_mergeh(vtemp26, vtemp28);
136+
*(vec_bf16x2 *)(boffset + 88) = vec_mergeh(vtemp30, vtemp32);
137+
*(vec_bf16x2 *)(boffset + 96) = vec_mergel(vtemp18, vtemp20);
138+
*(vec_bf16x2 *)(boffset + 104) = vec_mergel(vtemp22, vtemp24);
139+
*(vec_bf16x2 *)(boffset + 112) = vec_mergel(vtemp26, vtemp28);
140+
*(vec_bf16x2 *)(boffset + 120) = vec_mergel(vtemp30, vtemp32);
141+
142+
aoffset1 += 8;
143+
aoffset2 += 8;
144+
aoffset3 += 8;
145+
aoffset4 += 8;
146+
aoffset5 += 8;
147+
aoffset6 += 8;
148+
aoffset7 += 8;
149+
aoffset8 += 8;
150+
aoffset9 += 8;
151+
aoffset10 += 8;
152+
aoffset11 += 8;
153+
aoffset12 += 8;
154+
aoffset13 += 8;
155+
aoffset14 += 8;
156+
aoffset15 += 8;
157+
aoffset16 += 8;
158+
159+
boffset += 128;
160+
161+
i--;
162+
} while (i > 0);
163+
}
164+
165+
i = (m & 7) >> 1;
86166
if (i > 0){
87167
do{
88168
ctemp01 = *(aoffset1 + 0);

‎test/compare_sgemm_sbgemm.c‎

Lines changed: 58 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,26 @@ main (int argc, char *argv[])
8686
{
8787
blasint m, n, k;
8888
int i, j, l;
89-
blasint x;
89+
blasint x, y;
9090
int ret = 0;
9191
int loop = 100;
9292
char transA = 'N', transB = 'N';
9393
float alpha = 1.0, beta = 0.0;
9494

9595
for (x = 0; x <= loop; x++)
96+
{
97+
for (y = 0; y < 4; y++)
9698
{
99+
if ((y == 0) || (y == 2)) {
100+
transA = 'N';
101+
} else {
102+
transA = 'T';
103+
}
104+
if ((y == 0) || (y == 1)) {
105+
transB = 'N';
106+
} else {
107+
transB = 'T';
108+
}
97109
m = k = n = x;
98110
float A[m * k];
99111
float B[k * n];
@@ -104,43 +116,55 @@ main (int argc, char *argv[])
104116
blasint one=1;
105117

106118
for (j = 0; j < m; j++)
107-
{
108-
for (i = 0; i < m; i++)
109-
{
110-
A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
111-
B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
112-
C[j * k + i] = 0;
113-
sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one);
114-
sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one);
115-
AA[j * k + i].v = atmp;
116-
BB[j * k + i].v = btmp;
117-
CC[j * k + i] = 0;
118-
DD[j * k + i] = 0;
119-
}
120-
}
119+
{
120+
for (i = 0; i < m; i++)
121+
{
122+
A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
123+
B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
124+
C[j * k + i] = 0;
125+
sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one);
126+
sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one);
127+
AA[j * k + i].v = atmp;
128+
BB[j * k + i].v = btmp;
129+
CC[j * k + i] = 0;
130+
DD[j * k + i] = 0;
131+
}
132+
}
121133
SGEMM (&transA, &transB, &m, &n, &k, &alpha, A,
122-
&m, B, &k, &beta, C, &m);
134+
&m, B, &k, &beta, C, &m);
123135
SBGEMM (&transA, &transB, &m, &n, &k, &alpha, (bfloat16*) AA,
124-
&m, (bfloat16*)BB, &k, &beta, CC, &m);
136+
&m, (bfloat16*)BB, &k, &beta, CC, &m);
137+
for (i = 0; i < n; i++)
138+
for (j = 0; j < m; j++)
139+
if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0)
140+
ret++;
125141
for (i = 0; i < n; i++)
126-
for (j = 0; j < m; j++)
127-
if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0)
128-
ret++;
129-
if (transA == 'N' && transB == 'N')
130-
{
131-
for (i = 0; i < n; i++)
132-
for (j = 0; j < m; j++)
133-
for (l = 0; l < k; l++)
134-
{
135-
DD[i * m + j] +=
136-
float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]);
137-
}
138-
for (i = 0; i < n; i++)
139-
for (j = 0; j < m; j++)
140-
if (CC[i * m + j] != DD[i * m + j])
141-
ret++;
142-
}
142+
for (j = 0; j < m; j++)
143+
for (l = 0; l < k; l++)
144+
if (transA == 'N' && transB == 'N')
145+
{
146+
DD[i * m + j] +=
147+
float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]);
148+
} else if (transA == 'T' && transB == 'N')
149+
{
150+
DD[i * m + j] +=
151+
float16to32 (AA[k * j + l]) * float16to32 (BB[l + k * i]);
152+
} else if (transA == 'N' && transB == 'T')
153+
{
154+
DD[i * m + j] +=
155+
float16to32 (AA[l * m + j]) * float16to32 (BB[i + l * n]);
156+
} else if (transA == 'T' && transB == 'T')
157+
{
158+
DD[i * m + j] +=
159+
float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]);
160+
}
161+
for (i = 0; i < n; i++)
162+
for (j = 0; j < m; j++)
163+
if (CC[i * m + j] != DD[i * m + j])
164+
ret++;
143165
}
166+
}
167+
144168
if (ret != 0)
145169
fprintf (stderr, "FATAL ERROR SBGEMM - Return code: %d\n", ret);
146170
return ret;

0 commit comments

Comments
 (0)