Skip to content

Commit e96d1a6

Browse files
EatgrapesEatgrapes
committed
feat: implement interaction, eye tracking and motion callbacks
1 parent e6d3ff7 commit e96d1a6

2 files changed

Lines changed: 105 additions & 99 deletions

File tree

‎example/src/main/java/dev/eatgrapes/live2d/example/Main.java‎

Lines changed: 49 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import java.io.InputStream;
1010
import java.nio.ByteBuffer;
11+
import java.nio.DoubleBuffer;
1112
import java.nio.IntBuffer;
1213
import java.nio.file.Files;
1314
import java.nio.file.Path;
@@ -19,54 +20,67 @@
1920
public class Main {
2021
private long window;
2122
private CubismUserModel model;
22-
private final float[] mvpMatrix = new float[]{
23-
1.0f, 0.0f, 0.0f, 0.0f,
24-
0.0f, 1.0f, 0.0f, 0.0f,
25-
0.0f, 0.0f, 1.0f, 0.0f,
26-
0.0f, 0.0f, 0.0f, 1.0f
23+
private final float[] mvp = new float[]{
24+
1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1
2725
};
2826

2927
public void run() throws Exception {
30-
initWindow();
31-
initLive2D();
28+
init();
29+
setup();
3230
loop();
3331
cleanup();
3432
}
3533

36-
private void initWindow() {
37-
if (!glfwInit()) throw new IllegalStateException("GLFW init failed");
38-
window = glfwCreateWindow(800, 800, "Live2D Example", 0, 0);
34+
private void init() {
35+
if (!glfwInit()) throw new RuntimeException("GLFW failed");
36+
window = glfwCreateWindow(800, 800, "Live2D Interaction", 0, 0);
3937
glfwMakeContextCurrent(window);
4038
GL.createCapabilities();
4139
glfwSwapInterval(1);
40+
41+
glfwSetCursorPosCallback(window, (win, x, y) -> {
42+
float nx = (float) (x / 400.0 - 1.0);
43+
float ny = (float) (1.0 - y / 400.0);
44+
if (model != null) model.setDragging(nx, ny);
45+
});
46+
47+
glfwSetMouseButtonCallback(window, (win, button, action, mods) -> {
48+
if (button == GLFW_MOUSE_BUTTON_LEFT && action == GLFW_PRESS) {
49+
try (MemoryStack stack = MemoryStack.stackPush()) {
50+
DoubleBuffer x = stack.mallocDouble(1), y = stack.mallocDouble(1);
51+
glfwGetCursorPos(window, x, y);
52+
float nx = (float) (x.get(0) / 400.0 - 1.0);
53+
float ny = (float) (1.0 - y.get(0) / 400.0);
54+
55+
if (model.isHit("HitArea", nx, ny)) {
56+
System.out.println("Hit!");
57+
byte[] motion = load("/model/Hiyori/motions/Hiyori_m01.motion3.json");
58+
model.startMotion(motion, 2, name -> System.out.println("Motion finished: " + name));
59+
}
60+
} catch (Exception e) { e.printStackTrace(); }
61+
}
62+
});
4263
}
4364

44-
private void initLive2D() throws Exception {
65+
private void setup() throws Exception {
4566
CubismFramework.startUp();
4667
CubismFramework.initialize();
4768

4869
model = new CubismUserModel();
49-
model.loadModel(loadResource("/model/Hiyori/Hiyori.moc3"));
50-
model.loadPose(loadResource("/model/Hiyori/Hiyori.pose3.json"));
51-
model.loadPhysics(loadResource("/model/Hiyori/Hiyori.physics3.json"));
70+
model.loadModel(load("/model/Hiyori/Hiyori.moc3"));
71+
model.loadPose(load("/model/Hiyori/Hiyori.pose3.json"));
72+
model.loadPhysics(load("/model/Hiyori/Hiyori.physics3.json"));
5273
model.createRenderer();
5374

54-
model.registerTexture(0, loadTexture("/model/Hiyori/Hiyori.2048/texture_00.png"));
55-
model.registerTexture(1, loadTexture("/model/Hiyori/Hiyori.2048/texture_01.png"));
75+
model.registerTexture(0, loadTex("/model/Hiyori/Hiyori.2048/texture_00.png"));
76+
model.registerTexture(1, loadTex("/model/Hiyori/Hiyori.2048/texture_01.png"));
5677
}
5778

5879
private void loop() {
59-
long startTime = System.currentTimeMillis();
6080
while (!glfwWindowShouldClose(window)) {
61-
glClearColor(0.2f, 0.2f, 0.2f, 1.0f);
6281
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
63-
64-
float time = (System.currentTimeMillis() - startTime) / 1000.0f;
65-
model.setParameterValue("ParamAngleX", (float) Math.sin(time) * 30.0f);
66-
6782
model.update(0.016f);
68-
model.draw(mvpMatrix);
69-
83+
model.draw(mvp);
7084
glfwSwapBuffers(window);
7185
glfwPollEvents();
7286
}
@@ -78,34 +92,27 @@ private void cleanup() {
7892
glfwTerminate();
7993
}
8094

81-
private byte[] loadResource(String path) throws Exception {
82-
try (InputStream is = getClass().getResourceAsStream(path)) {
83-
return is.readAllBytes();
84-
}
95+
private byte[] load(String p) throws Exception {
96+
try (InputStream is = getClass().getResourceAsStream(p)) { return is.readAllBytes(); }
8597
}
8698

87-
private int loadTexture(String path) throws Exception {
88-
Path temp = Files.createTempFile("l2d", ".png");
89-
try (InputStream is = getClass().getResourceAsStream(path)) {
90-
Files.copy(is, temp, StandardCopyOption.REPLACE_EXISTING);
91-
}
99+
private int loadTex(String p) throws Exception {
100+
Path tmp = Files.createTempFile("l2d", ".png");
101+
try (InputStream is = getClass().getResourceAsStream(p)) { Files.copy(is, tmp, StandardCopyOption.REPLACE_EXISTING); }
92102
int w, h, tex;
93103
ByteBuffer img;
94-
try (MemoryStack stack = MemoryStack.stackPush()) {
95-
IntBuffer wb = stack.mallocInt(1), hb = stack.mallocInt(1), cb = stack.mallocInt(1);
96-
img = STBImage.stbi_load(temp.toString(), wb, hb, cb, 4);
104+
try (MemoryStack s = MemoryStack.stackPush()) {
105+
IntBuffer wb = s.mallocInt(1), hb = s.mallocInt(1), cb = s.mallocInt(1);
106+
img = STBImage.stbi_load(tmp.toString(), wb, hb, cb, 4);
97107
w = wb.get(); h = hb.get();
98108
}
99109
tex = glGenTextures();
100110
glBindTexture(GL_TEXTURE_2D, tex);
101-
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
102111
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0, GL_RGBA, GL_UNSIGNED_BYTE, img);
103112
STBImage.stbi_image_free(img);
104-
Files.delete(temp);
113+
Files.delete(tmp);
105114
return tex;
106115
}
107116

108-
public static void main(String[] args) throws Exception {
109-
new Main().run();
110-
}
111-
}
117+
public static void main(String[] args) throws Exception { new Main().run(); }
118+
}

‎native/src/CubismUserModel_JNI.cpp‎

Lines changed: 56 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -17,65 +17,64 @@ class JniUserModel : public CubismUserModel {
1717
}
1818

1919
~JniUserModel() {
20-
JNIEnv* env = GetEnv();
20+
JNIEnv* env = getEnv();
2121
if (env) env->DeleteGlobalRef(_javaObj);
2222
}
2323

24-
void LoadModelWithCopy(const csmByte* buffer, csmSizeInt size) {
24+
void loadModelCopy(const csmByte* buffer, csmSizeInt size) {
2525
_mocData.assign(buffer, buffer + size);
26-
this->LoadModel(_mocData.data(), static_cast<csmSizeInt>(_mocData.size()));
26+
LoadModel(_mocData.data(), (csmSizeInt)_mocData.size());
2727
}
2828

29-
void StartMotionWithCallback(const csmByte* buffer, csmSizeInt size, int priority) {
29+
void startMotion(const csmByte* buffer, csmSizeInt size, int priority) {
3030
auto* motion = CubismMotion::Create(buffer, size);
3131
if (!motion) return;
3232

33-
motion->SetFinishedMotionHandler([](ACubismMotion* self, void* customData) {
34-
auto* model = static_cast<JniUserModel*>(customData);
35-
model->NotifyMotionFinished("motion");
33+
motion->SetFinishedMotionHandlerAndMotionCustomData([](ACubismMotion* self, void* data) {
34+
auto* model = static_cast<JniUserModel*>(data);
35+
model->notifyFinished();
3636
CubismMotion::Delete(static_cast<CubismMotion*>(self));
3737
}, this);
3838

3939
_motionManager->StartMotionPriority(motion, true, priority);
4040
}
4141

42-
void NotifyMotionFinished(const std::string& name) {
43-
JNIEnv* env = GetEnv();
42+
void notifyFinished() {
43+
JNIEnv* env = getEnv();
4444
if (!env) return;
45-
jclass clazz = env->GetObjectClass(_javaObj);
46-
jmethodID mid = env->GetMethodID(clazz, "onMotionFinished", "(Ljava/lang/String;)V");
47-
jstring jname = env->NewStringUTF(name.c_str());
48-
env->CallVoidMethod(_javaObj, mid, jname);
49-
env->DeleteLocalRef(jname);
45+
jclass cls = env->GetObjectClass(_javaObj);
46+
jmethodID mid = env->GetMethodID(cls, "onMotionFinished", "(Ljava/lang/String;)V");
47+
jstring name = env->NewStringUTF("motion");
48+
env->CallVoidMethod(_javaObj, mid, name);
49+
env->DeleteLocalRef(name);
5050
}
5151

52-
void UpdateFramework(float deltaTime) {
52+
void update(float dt) {
5353
if (!_model) return;
5454

5555
_model->LoadParameters();
56-
if (_motionManager->IsFinished()) {
57-
// Idle logic could go here
58-
} else {
59-
_motionManager->UpdateMotion(_model, deltaTime);
56+
if (!_motionManager->IsFinished()) {
57+
_motionManager->UpdateMotion(_model, dt);
6058
}
6159
_model->SaveParameters();
6260

6361
if (_dragManager) {
64-
_dragManager->Update(deltaTime);
65-
_model->AddParameterValue(CubismFramework::GetIdManager()->GetId("ParamAngleX"), _dragManager->GetX() * 30.0f);
66-
_model->AddParameterValue(CubismFramework::GetIdManager()->GetId("ParamAngleY"), _dragManager->GetY() * 30.0f);
67-
_model->AddParameterValue(CubismFramework::GetIdManager()->GetId("ParamBodyAngleX"), _dragManager->GetX() * 10.0f);
68-
_model->AddParameterValue(CubismFramework::GetIdManager()->GetId("ParamEyeBallX"), _dragManager->GetX());
69-
_model->AddParameterValue(CubismFramework::GetIdManager()->GetId("ParamEyeBallY"), _dragManager->GetY());
62+
_dragManager->Update(dt);
63+
auto* idm = CubismFramework::GetIdManager();
64+
_model->AddParameterValue(idm->GetId("ParamAngleX"), _dragManager->GetX() * 30.0f);
65+
_model->AddParameterValue(idm->GetId("ParamAngleY"), _dragManager->GetY() * 30.0f);
66+
_model->AddParameterValue(idm->GetId("ParamBodyAngleX"), _dragManager->GetX() * 10.0f);
67+
_model->AddParameterValue(idm->GetId("ParamEyeBallX"), _dragManager->GetX());
68+
_model->AddParameterValue(idm->GetId("ParamEyeBallY"), _dragManager->GetY());
7069
}
7170

72-
if (_pose) _pose->UpdateParameters(_model, deltaTime);
73-
if (_physics) _physics->Evaluate(_model, deltaTime);
71+
if (_pose) _pose->UpdateParameters(_model, dt);
72+
if (_physics) _physics->Evaluate(_model, dt);
7473
_model->Update();
7574
}
7675

7776
private:
78-
JNIEnv* GetEnv() {
77+
JNIEnv* getEnv() {
7978
JNIEnv* env;
8079
if (_jvm->GetEnv((void**)&env, JNI_VERSION_1_6) == JNI_EDETACHED) {
8180
_jvm->AttachCurrentThread((void**)&env, nullptr);
@@ -94,32 +93,32 @@ JNIEXPORT jlong JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_createNative(J
9493
return (jlong) new JniUserModel(env, thiz);
9594
}
9695

97-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_deleteNative(JNIEnv* env, jclass clazz, jlong ptr) {
98-
delete (JniUserModel*) ptr;
96+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_deleteNative(JNIEnv* env, jclass, jlong ptr) {
97+
delete (JniUserModel*)ptr;
9998
}
10099

101-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadModelNative(JNIEnv* env, jclass clazz, jlong ptr, jbyteArray buffer) {
100+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadModelNative(JNIEnv* env, jclass, jlong ptr, jbyteArray buffer) {
102101
jsize len = env->GetArrayLength(buffer);
103102
jbyte* data = env->GetByteArrayElements(buffer, nullptr);
104-
((JniUserModel*)ptr)->LoadModelWithCopy((const csmByte*)data, len);
103+
((JniUserModel*)ptr)->loadModelCopy((const csmByte*)data, len);
105104
env->ReleaseByteArrayElements(buffer, data, JNI_ABORT);
106105
}
107106

108-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadPhysicsNative(JNIEnv* env, jclass clazz, jlong ptr, jbyteArray buffer) {
107+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadPhysicsNative(JNIEnv* env, jclass, jlong ptr, jbyteArray buffer) {
109108
jsize len = env->GetArrayLength(buffer);
110109
jbyte* data = env->GetByteArrayElements(buffer, nullptr);
111110
((JniUserModel*)ptr)->LoadPhysics((const csmByte*)data, len);
112111
env->ReleaseByteArrayElements(buffer, data, JNI_ABORT);
113112
}
114113

115-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadPoseNative(JNIEnv* env, jclass clazz, jlong ptr, jbyteArray buffer) {
114+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadPoseNative(JNIEnv* env, jclass, jlong ptr, jbyteArray buffer) {
116115
jsize len = env->GetArrayLength(buffer);
117116
jbyte* data = env->GetByteArrayElements(buffer, nullptr);
118117
((JniUserModel*)ptr)->LoadPose((const csmByte*)data, len);
119118
env->ReleaseByteArrayElements(buffer, data, JNI_ABORT);
120119
}
121120

122-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadExpressionNative(JNIEnv* env, jclass clazz, jlong ptr, jbyteArray buffer, jstring name) {
121+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadExpressionNative(JNIEnv* env, jclass, jlong ptr, jbyteArray buffer, jstring name) {
123122
jsize len = env->GetArrayLength(buffer);
124123
jbyte* data = env->GetByteArrayElements(buffer, nullptr);
125124
const char* n = env->GetStringUTFChars(name, nullptr);
@@ -128,61 +127,61 @@ JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_loadExpressionN
128127
env->ReleaseByteArrayElements(buffer, data, JNI_ABORT);
129128
}
130129

131-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_createRendererNative(JNIEnv* env, jclass clazz, jlong ptr) {
130+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_createRendererNative(JNIEnv*, jclass, jlong ptr) {
132131
((JniUserModel*)ptr)->CreateRenderer();
133132
}
134133

135-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_registerTextureNative(JNIEnv* env, jclass clazz, jlong ptr, jint index, jint textureId) {
134+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_registerTextureNative(JNIEnv*, jclass, jlong ptr, jint index, jint textureId) {
136135
auto* renderer = ((JniUserModel*)ptr)->GetRenderer<CubismRenderer_OpenGLES2>();
137136
if (renderer) renderer->BindTexture(index, (GLuint)textureId);
138137
}
139138

140-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_setDraggingNative(JNIEnv* env, jclass clazz, jlong ptr, jfloat x, jfloat y) {
139+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_setDraggingNative(JNIEnv*, jclass, jlong ptr, jfloat x, jfloat y) {
141140
((JniUserModel*)ptr)->SetDragging(x, y);
142141
}
143142

144-
JNIEXPORT jboolean JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_isHitNative(JNIEnv* env, jclass clazz, jlong ptr, jstring drawableId, jfloat x, jfloat y) {
145-
const char* idStr = env->GetStringUTFChars(drawableId, nullptr);
146-
bool hit = ((JniUserModel*)ptr)->IsHit(CubismFramework::GetIdManager()->GetId(idStr), x, y);
147-
env->ReleaseStringUTFChars(drawableId, idStr);
143+
JNIEXPORT jboolean JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_isHitNative(JNIEnv* env, jclass, jlong ptr, jstring id, jfloat x, jfloat y) {
144+
const char* s = env->GetStringUTFChars(id, nullptr);
145+
bool hit = ((JniUserModel*)ptr)->IsHit(CubismFramework::GetIdManager()->GetId(s), x, y);
146+
env->ReleaseStringUTFChars(id, s);
148147
return hit;
149148
}
150149

151-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_startMotionNative(JNIEnv* env, jclass clazz, jlong ptr, jbyteArray buffer, jint priority) {
150+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_startMotionNative(JNIEnv* env, jclass, jlong ptr, jbyteArray buffer, jint priority) {
152151
jsize len = env->GetArrayLength(buffer);
153152
jbyte* data = env->GetByteArrayElements(buffer, nullptr);
154-
((JniUserModel*)ptr)->StartMotionWithCallback((const csmByte*)data, len, priority);
153+
((JniUserModel*)ptr)->startMotion((const csmByte*)data, len, priority);
155154
env->ReleaseByteArrayElements(buffer, data, JNI_ABORT);
156155
}
157156

158-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_updateNative(JNIEnv* env, jclass clazz, jlong ptr, jfloat deltaTime) {
159-
((JniUserModel*)ptr)->UpdateFramework(deltaTime);
157+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_updateNative(JNIEnv*, jclass, jlong ptr, jfloat dt) {
158+
((JniUserModel*)ptr)->update(dt);
160159
}
161160

162-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_setParameterValueNative(JNIEnv* env, jclass clazz, jlong ptr, jstring id, jfloat value) {
163-
const char* idStr = env->GetStringUTFChars(id, nullptr);
164-
((JniUserModel*)ptr)->GetModel()->SetParameterValue(CubismFramework::GetIdManager()->GetId(idStr), value);
165-
env->ReleaseStringUTFChars(id, idStr);
161+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_setParameterValueNative(JNIEnv* env, jclass, jlong ptr, jstring id, jfloat value) {
162+
const char* s = env->GetStringUTFChars(id, nullptr);
163+
((JniUserModel*)ptr)->GetModel()->SetParameterValue(CubismFramework::GetIdManager()->GetId(s), value);
164+
env->ReleaseStringUTFChars(id, s);
166165
}
167166

168-
JNIEXPORT jfloat JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_getCanvasWidthNative(JNIEnv* env, jclass clazz, jlong ptr) {
167+
JNIEXPORT jfloat JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_getCanvasWidthNative(JNIEnv*, jclass, jlong ptr) {
169168
return ((JniUserModel*)ptr)->GetModel()->GetCanvasWidth();
170169
}
171170

172-
JNIEXPORT jfloat JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_getCanvasHeightNative(JNIEnv* env, jclass clazz, jlong ptr) {
171+
JNIEXPORT jfloat JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_getCanvasHeightNative(JNIEnv*, jclass, jlong ptr) {
173172
return ((JniUserModel*)ptr)->GetModel()->GetCanvasHeight();
174173
}
175174

176-
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_drawNative(JNIEnv* env, jclass clazz, jlong ptr, jfloatArray mvpMatrix) {
177-
auto* model = (JniUserModel*) ptr;
175+
JNIEXPORT void JNICALL Java_dev_eatgrapes_live2d_CubismUserModel_drawNative(JNIEnv* env, jclass, jlong ptr, jfloatArray matrix) {
176+
auto* model = (JniUserModel*)ptr;
178177
auto* renderer = model->GetRenderer<CubismRenderer_OpenGLES2>();
179178
if (renderer) {
180-
jfloat* matrix = env->GetFloatArrayElements(mvpMatrix, nullptr);
179+
jfloat* m_ptr = env->GetFloatArrayElements(matrix, nullptr);
181180
CubismMatrix44 m;
182-
m.SetMatrix(matrix);
181+
m.SetMatrix(m_ptr);
183182
renderer->SetMvpMatrix(&m);
184183
renderer->DrawModel();
185-
env->ReleaseFloatArrayElements(mvpMatrix, matrix, JNI_ABORT);
184+
env->ReleaseFloatArrayElements(matrix, m_ptr, JNI_ABORT);
186185
}
187186
}
188187

0 commit comments

Comments
 (0)