Move headers for unittest with llm.c to unittest_llmc directory

Author: junjihashimoto

experimental/kernels/Makefile

@@ -11,8 +11,8 @@ else
     STDLIB := -stdlib=libc++
 endif
 
-CXXFLAGS=-std=c++17 -I$(GPUCPP) -I$(GPUCPP)/third_party/headers
-CFLAGS=-Ofast -march=native -I.
+CXXFLAGS=-std=c++17 -I$(GPUCPP) -I$(GPUCPP)/third_party/headers -I. -Iunittest_llmc
+CFLAGS=-Ofast -march=native -I. -Iunittest_llmc
 
 LDFLAGS=$(STDLIB) -L$(GPUCPP)/third_party/lib -ldl -ldawn
 FLAGS=$(CXXFLAGS) $(LDFLAGS)
@@ -39,7 +39,7 @@ gpt2_124M.bin: llm.c
           ln -s ./llm.c/gpt2_124M_debug_state.bin ; \
         fi
 
-build/test_gpt2: llm.c build/kernels_c.o gpt2_124M.bin
+build/test_gpt2: llm.c build/unittest_kernels.o gpt2_124M.bin
 	mkdir -p build
 	sed -i -e 's/void encoder_forward(/void ENCODER_FORWARD_CPU(/g' llm.c/train_gpt2.c
 	sed -i -e 's/void layernorm_forward(/void LAYERNORM_FORWARD_CPU(/g' llm.c/train_gpt2.c
@@ -56,10 +56,10 @@ build/test_gpt2: llm.c build/kernels_c.o gpt2_124M.bin
 	sed -i -e 's/void gelu_backward(/void GELU_BACKWARD_CPU(/g' llm.c/train_gpt2.c
 	sed -i -e 's/void residual_backward(/void RESIDUAL_BACKWARD_CPU(/g' llm.c/train_gpt2.c
 	sed -i -e 's/void crossentropy_softmax_backward(/void CROSSENTROPY_SOFTMAX_BACKWARD_CPU(/g' llm.c/train_gpt2.c
-	grep -q "^#include \"kernels_c.h\"" llm.c/train_gpt2.c || sed -i '1i#include \"kernels_c.h\"' llm.c/train_gpt2.c
-	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ llm.c/test_gpt2.c build/kernels_c.o
+	grep -q "^#include \"unittest_kernels.h\"" llm.c/train_gpt2.c || sed -i '1i#include \"unittest_kernels.h\"' llm.c/train_gpt2.c
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ llm.c/test_gpt2.c build/unittest_kernels.o
 
-build/kernels_c.o: kernels_c.cpp kernels_c.h kernels.h
+build/unittest_kernels.o: unittest_llmc/unittest_kernels.cpp unittest_llmc/unittest_kernels.h kernels.h
 	mkdir -p build && $(CXX) $(CXXFLAGS) -DNDEBUG -c -o $@ $<
 
 dawnlib: $(if $(wildcard $(GPUCPP)/third_party/lib/libdawn.so $(GPUCPP)/third_party/lib/libdawn.dylib),,run_setup)

experimental/kernels/kernels_c.cpp …rnels/unittest_llmc/unittest_kernels.cppexperimental/kernels/kernels_c.cpp renamed to experimental/kernels/unittest_llmc/unittest_kernels.cpp experimental/kernels/kernels_c.cpp renamed to experimental/kernels/unittest_llmc/unittest_kernels.cpp

@@ -4,7 +4,7 @@
 #include <future>
 
 #include "kernels.h"
-#include "kernels_c.h"
+#include "unittest_llmc/unittest_kernels.h"
 
 using namespace gpu; // createContext, createTensor, createKernel,
                      // createShader, dispatchKernel, wait, toCPU

experimental/kernels/kernels_c.h …kernels/unittest_llmc/unittest_kernels.hexperimental/kernels/kernels_c.h renamed to experimental/kernels/unittest_llmc/unittest_kernels.h experimental/kernels/kernels_c.h renamed to experimental/kernels/unittest_llmc/unittest_kernels.h

@@ -9,6 +9,7 @@ extern "C" {
 #define USE_GPU_FOR_LAYERNORM_FORWARD 1
 // --  Note: atomicAdd should be used with i32 or u32 not f32.
 // #define USE_GPU_FOR_LAYERNORM_BACKWARD 1
+// --  Note: matmul_forward kernel works, but it is too slow.
 #define USE_GPU_FOR_MATMUL_FORWARD 1
 // #define USE_GPU_FOR_MATMUL_BACKWARD 1
 #define USE_GPU_FOR_ATTENTION_FORWARD 1