Merge pull request #4806 from Mousius/small-gemm

Small GEMM for AArch64 with SVE

Author: martin-frbg

Makefile.system

@@ -268,6 +268,8 @@ SMALL_MATRIX_OPT = 1
 else ifeq ($(ARCH), power)
 SMALL_MATRIX_OPT = 1
 BUILD_BFLOAT16 = 1
+else ifeq ($(ARCH), arm64)
+SMALL_MATRIX_OPT = 1
 endif
 ifeq ($(ARCH), loongarch64)
 SMALL_MATRIX_OPT = 1

kernel/arm64/KERNEL.ARMV8SVE

@@ -131,6 +131,16 @@ SGEMMITCOPYOBJ =  sgemm_itcopy$(TSUFFIX).$(SUFFIX)
 SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
+SGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c
+SGEMM_SMALL_K_NT = sgemm_small_kernel_nt_sve.c
+SGEMM_SMALL_K_B0_NT = sgemm_small_kernel_nt_sve.c
+SGEMM_SMALL_K_NN = sgemm_small_kernel_nn_sve.c
+SGEMM_SMALL_K_B0_NN = sgemm_small_kernel_nn_sve.c
+SGEMM_SMALL_K_TT = sgemm_small_kernel_tt_sve.c
+SGEMM_SMALL_K_B0_TT = sgemm_small_kernel_tt_sve.c
+SGEMM_SMALL_K_TN = sgemm_small_kernel_tn_sve.c
+SGEMM_SMALL_K_B0_TN = sgemm_small_kernel_tn_sve.c
+
 STRMMUNCOPY_M  =  trmm_uncopy_sve_v1.c
 STRMMLNCOPY_M  =  trmm_lncopy_sve_v1.c
 STRMMUTCOPY_M  =  trmm_utcopy_sve_v1.c
@@ -152,6 +162,16 @@ DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
 DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
+DGEMM_SMALL_M_PERMIT = gemm_small_kernel_permit_sve.c
+DGEMM_SMALL_K_NT = dgemm_small_kernel_nt_sve.c
+DGEMM_SMALL_K_B0_NT = dgemm_small_kernel_nt_sve.c
+DGEMM_SMALL_K_NN = dgemm_small_kernel_nn_sve.c
+DGEMM_SMALL_K_B0_NN = dgemm_small_kernel_nn_sve.c
+DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_sve.c
+DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_sve.c
+DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_sve.c
+DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_sve.c
+
 DTRMMUNCOPY_M  =  trmm_uncopy_sve_v1.c
 DTRMMLNCOPY_M  =  trmm_lncopy_sve_v1.c
 DTRMMUTCOPY_M  =  trmm_utcopy_sve_v1.c