Skip to content

Commit 9d3ab09

Browse files
pratikgujjarpratikgujjar
committed
Training classifier on broader images
1 parent cfa569f commit 9d3ab09

26 files changed

Lines changed: 985 additions & 356 deletions

‎code/autoencoder_model/gifs/vid_17.gif‎

-540 KB
Binary file not shown.

‎code/autoencoder_model/gifs/vid_18.gif‎

-551 KB
Binary file not shown.

‎code/autoencoder_model/scripts/attention_autoencoder.py‎

Lines changed: 42 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -51,90 +51,45 @@
5151
import os
5252

5353

54-
# def encoder_model():
55-
# model = Sequential()
56-
#
57-
# # 10x128x128
58-
# model.add(Conv3D(filters=128,
59-
# strides=(1, 4, 4),
60-
# kernel_size=(3, 11, 11),
61-
# padding='same',
62-
# input_shape=(int(VIDEO_LENGTH/2), 128, 128, 3)))
63-
# model.add(TimeDistributed(BatchNormalization()))
64-
# model.add(TimeDistributed(LeakyReLU(alpha=0.2)))
65-
# model.add(TimeDistributed(Dropout(0.5)))
66-
#
67-
# # 10x32x32
68-
# model.add(Conv3D(filters=64,
69-
# strides=(1, 2, 2),
70-
# kernel_size=(3, 5, 5),
71-
# padding='same'))
72-
# model.add(TimeDistributed(BatchNormalization()))
73-
# model.add(TimeDistributed(LeakyReLU(alpha=0.2)))
74-
# model.add(TimeDistributed(Dropout(0.5)))
75-
#
76-
# # 10x16x16
77-
# model.add(Conv3D(filters=64,
78-
# strides=(1, 1, 1),
79-
# kernel_size=(3, 3, 3),
80-
# padding='same'))
81-
# model.add(TimeDistributed(BatchNormalization()))
82-
# model.add(TimeDistributed(LeakyReLU(alpha=0.2)))
83-
# model.add(TimeDistributed(Dropout(0.5)))
84-
#
85-
# return model
86-
87-
8854
def encoder_model():
89-
inputs = Input(shape=(int(VIDEO_LENGTH / 2), 128, 128, 3))
55+
model = Sequential()
9056

9157
# 10x128x128
92-
conv_1 = TimeDistributed(Conv2D(filters=64,
93-
strides=(4, 4),
94-
kernel_size=(11, 11),
95-
padding='same'))(inputs)
96-
x = TimeDistributed(BatchNormalization())(conv_1)
97-
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
98-
x = TimeDistributed(Dropout(0.4))(x)
58+
model.add(Conv3D(filters=128,
59+
strides=(1, 4, 4),
60+
kernel_size=(3, 11, 11),
61+
padding='same',
62+
input_shape=(int(VIDEO_LENGTH/2), 128, 128, 3)))
63+
model.add(TimeDistributed(BatchNormalization()))
64+
model.add(TimeDistributed(LeakyReLU(alpha=0.2)))
65+
model.add(TimeDistributed(Dropout(0.5)))
9966

10067
# 10x32x32
101-
conv_2 = TimeDistributed(Conv2D(filters=128,
102-
strides=(2, 2),
103-
kernel_size=(5, 5),
104-
padding='same'))(x)
105-
x = TimeDistributed(BatchNormalization())(conv_2)
106-
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
107-
out_2 = TimeDistributed(Dropout(0.5))(x)
68+
model.add(Conv3D(filters=64,
69+
strides=(1, 2, 2),
70+
kernel_size=(3, 5, 5),
71+
padding='same'))
72+
model.add(TimeDistributed(BatchNormalization()))
73+
model.add(TimeDistributed(LeakyReLU(alpha=0.2)))
74+
model.add(TimeDistributed(Dropout(0.5)))
10875

10976
# 10x16x16
110-
conv_3 = TimeDistributed(Conv2D(filters=128,
111-
strides=(1, 1),
112-
kernel_size=(3, 3),
113-
padding='same'))(out_2)
114-
x = TimeDistributed(BatchNormalization())(conv_3)
115-
x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
116-
in_rep = TimeDistributed(Dropout(0.5))(x)
117-
118-
# res_1 = concatenate([out_2, in_rep])
119-
#
120-
# clstm_1 = ConvLSTM2D(filters=1,
121-
# kernel_size=(3, 3),
122-
# strides=(1, 1),
123-
# padding='same',
124-
# return_sequences=False,
125-
# activation='relu',
126-
# recurrent_dropout=0.2)(res_1)
127-
# tr_in = add([in_rep, clstm_1])
128-
model = Model(inputs=inputs, outputs=in_rep)
77+
model.add(Conv3D(filters=64,
78+
strides=(1, 1, 1),
79+
kernel_size=(3, 3, 3),
80+
padding='same'))
81+
model.add(TimeDistributed(BatchNormalization()))
82+
model.add(TimeDistributed(LeakyReLU(alpha=0.2)))
83+
model.add(TimeDistributed(Dropout(0.5)))
12984

13085
return model
13186

13287

13388
def decoder_model():
134-
inputs = Input(shape=(int(VIDEO_LENGTH/2), 16, 16, 128))
89+
inputs = Input(shape=(int(VIDEO_LENGTH/2), 16, 16, 64))
13590

13691
# 10x16x16
137-
convlstm_1 = ConvLSTM2D(filters=128,
92+
convlstm_1 = ConvLSTM2D(filters=64,
13893
kernel_size=(3, 3),
13994
strides=(1, 1),
14095
padding='same',
@@ -192,13 +147,29 @@ def decoder_model():
192147
h_4 = TimeDistributed(LeakyReLU(alpha=0.2))(x)
193148
out_4 = UpSampling3D(size=(1, 2, 2))(h_4)
194149

150+
aclstm_1 = ConvLSTM2D(filters=1,
151+
kernel_size=(3, 3),
152+
strides=(1, 1),
153+
padding='same',
154+
return_sequences=True,
155+
recurrent_dropout=0.2)(h_4)
156+
x = TimeDistributed(BatchNormalization())(aclstm_1)
157+
flat_1 = TimeDistributed(Flatten())(x)
158+
dense_1 = TimeDistributed(Dense(units=128 * 128, activation='softmax'))(flat_1)
159+
x = TimeDistributed(BatchNormalization())(dense_1)
160+
x = TimeDistributed(Dropout(0.2))(x)
161+
print (x.shape)
162+
a1_reshape = Reshape(target_shape=(int(VIDEO_LENGTH/2), 128, 128, 1))(x)
163+
a1 = AttnLossLayer()(a1_reshape)
164+
dot_1 = multiply([out_4, a1])
165+
195166
# 10x128x128
196167
convlstm_5 = ConvLSTM2D(filters=3,
197168
kernel_size=(3, 3),
198169
strides=(1, 1),
199170
padding='same',
200171
return_sequences=True,
201-
recurrent_dropout=0.2)(out_4)
172+
recurrent_dropout=0.2)(dot_1)
202173
predictions = TimeDistributed(Activation('tanh'))(convlstm_5)
203174

204175
model = Model(inputs=inputs, outputs=predictions)

‎code/autoencoder_model/scripts/config_aa.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929

3030
VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/val/'
3131

32-
TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/test/'
33-
# TEST_DATA_DIR= '/local_home/JAAD_Dataset/fun_experiments/resized/'
32+
# TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/test/'
33+
TEST_DATA_DIR= '/local_home/JAAD_Dataset/fun_experiments/resized/'
3434

3535
MODEL_DIR = './../' + path_var + 'models'
3636
if not os.path.exists(MODEL_DIR):

‎code/autoencoder_model/scripts/config_aa.pyc‎

-3 Bytes
Binary file not shown.

‎code/autoencoder_model/scripts/config_oc.py‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
else:
2222
path_var = 'zhora/'
2323

24-
DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/train/'
24+
DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/train/'
2525
# DATA_DIR= '/local_home/data/KITTI_data/'
2626

27-
TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/test/'
27+
TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/test/'
2828

29-
VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/val/'
29+
VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/val/'
3030

3131
PRETRAINED_C3D= '/home/pratik/git_projects/c3d-keras/models/sports1M_weights_tf.json'
3232
PRETRAINED_C3D_WEIGHTS= '/home/pratik/git_projects/c3d-keras/models/sports1M_weights_tf.h5'
@@ -73,35 +73,33 @@
7373
SAVE_GENERATED_IMAGES = True
7474
SHUFFLE = True
7575
VIDEO_LENGTH = 16
76-
IMG_SIZE = (128, 128, 3)
76+
IMG_SIZE = (128, 208, 3)
7777
VIS_ATTN = True
78-
ATTN_COEFF = 0
79-
# KL coeff damages learning
80-
KL_COEFF = 0
8178
CLASSIFIER = True
8279
BUF_SIZE = 10
8380
LOSS_WEIGHTS = [1, 1]
8481
A_TRAIN_RATIO = 1
8582
C_TRAIN_RATIO = 1
8683
RAM_DECIMATE = False
87-
RETRAIN_CLASSIFIER = True
84+
RETRAIN_CLASSIFIER = False
8885
CLASS_TARGET_INDEX = 8
8986

90-
ped_actions = ['slow down', 'moving slow', 'standing', 'stopped',
91-
'speed up', 'moving fast', 'look', 'looking', 'clear path',
92-
'crossing', 'nod', 'handwave', 'unknown']
87+
88+
# ped_actions = ['standing 0', 'moving slow 1', 'moving fast 2', 'look 3', 'looking 4',
89+
# 'slow down 5', 'speed up 6', 'crossing 7', 'stopped 8', 'clear path 9',
90+
# 'nod 10', 'handwave 11', 'unknown 12']
91+
ped_actions = ['standing', 'moving slow', 'moving fast', 'look', 'looking',
92+
'slow down', 'speed up', 'crossing', 'stopped', 'clear path',
93+
'nod', 'handwave', 'unknown']
9394
# simple_ped_set = ['moving slow', 'stopped', 'moving fast', 'looking', 'clear path', 'crossing',
9495
# 'handwave', 'unknown']
9596

96-
simple_ped_set = ['crossing', 'stopped', 'looking', 'clear path', 'unknown']
97+
# simple_ped_set = ['crossing', 'stopped', 'looking', 'clear path', 'unknown']
98+
99+
# simple_ped_set = ['standing 0', 'approaching 1', 'looking 2', 'crossing 3', 'stopped 4', 'clear path 5', 'unknown 6']
100+
simple_ped_set = ['standing', 'approaching', 'looking', 'crossing', 'stopped', 'clear path', 'unknown']
101+
97102

98-
# ped_actions = ['slow down 0', 'moving slow 1', 'standing 2', 'stopped 3',
99-
# 'speed up 4', 'moving fast 5', 'look 6', 'looking 7', 'clear path 8',
100-
# 'crossing 9', 'nod 10', 'handwave 11', 'unknown 12']
101-
#
102-
#
103-
# simple_ped_set = ['moving slow 0', 'stopped 1', 'moving fast 2', 'looking 3', 'clear path 4', 'crossing 5',
104-
# 'handwave 6', 'unknown 7']
105103

106104

107105

@@ -128,8 +126,8 @@
128126
BATCH_SIZE = 25
129127
NB_EPOCHS_CLASS = 100
130128

131-
# OPTIM_C = Adam(lr=0.0000002, beta_1=0.5)
132-
OPTIM_C = SGD(lr=0.0001, momentum=0.9, nesterov=True)
129+
OPTIM_C = Adam(lr=0.0000002, beta_1=0.5)
130+
# OPTIM_C = SGD(lr=0.0001, momentum=0.9, nesterov=True)
133131

134132
# lr_schedule = [10, 20, 30] # epoch_step
135133

‎code/autoencoder_model/scripts/config_r10.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
from __future__ import absolute_import
2+
from __future__ import division
3+
from __future__ import print_function
4+
5+
from keras.optimizers import SGD
6+
from keras.optimizers import Adam
7+
from keras.optimizers import adadelta
8+
from keras.optimizers import rmsprop
9+
from keras.layers import Layer
10+
from keras import backend as K
11+
K.set_image_dim_ordering('tf')
12+
import socket
13+
import os
14+
15+
# -------------------------------------------------
16+
# Background config:
17+
hostname = socket.gethostname()
18+
if hostname == 'baymax':
19+
path_var = 'baymax/'
20+
elif hostname == 'walle':
21+
path_var = 'walle/'
22+
elif hostname == 'bender':
23+
path_var = 'bender/'
24+
else:
25+
path_var = 'zhora/'
26+
27+
DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/train/'
28+
# DATA_DIR= '/local_home/data/KITTI_data/'
29+
30+
VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/val/'
31+
32+
TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/test/'
33+
# TEST_DATA_DIR= '/local_home/JAAD_Dataset/fun_experiments/resized/'
34+
35+
MODEL_DIR = './../' + path_var + 'models'
36+
if not os.path.exists(MODEL_DIR):
37+
os.mkdir(MODEL_DIR)
38+
39+
CHECKPOINT_DIR = './../' + path_var + 'checkpoints'
40+
if not os.path.exists(CHECKPOINT_DIR):
41+
os.mkdir(CHECKPOINT_DIR)
42+
43+
ATTN_WEIGHTS_DIR = './../' + path_var + 'attn_weights'
44+
if not os.path.exists(ATTN_WEIGHTS_DIR):
45+
os.mkdir(ATTN_WEIGHTS_DIR)
46+
47+
GEN_IMAGES_DIR = './../' + path_var + 'generated_images'
48+
if not os.path.exists(GEN_IMAGES_DIR):
49+
os.mkdir(GEN_IMAGES_DIR)
50+
51+
LOG_DIR = './../' + path_var + 'logs'
52+
if not os.path.exists(LOG_DIR):
53+
os.mkdir(LOG_DIR)
54+
55+
TF_LOG_DIR = './../' + path_var + 'tf_logs'
56+
if not os.path.exists(TF_LOG_DIR):
57+
os.mkdir(TF_LOG_DIR)
58+
59+
TEST_RESULTS_DIR = './../' + path_var + 'test_results'
60+
if not os.path.exists(TEST_RESULTS_DIR):
61+
os.mkdir(TEST_RESULTS_DIR)
62+
63+
PRINT_MODEL_SUMMARY = True
64+
SAVE_MODEL = True
65+
PLOT_MODEL = True
66+
SAVE_GENERATED_IMAGES = True
67+
SHUFFLE = True
68+
VIDEO_LENGTH = 20
69+
IMG_SIZE = (128, 208, 3)
70+
ATTN_COEFF = 0
71+
KL_COEFF = 0
72+
RAM_DECIMATE = True
73+
74+
# -------------------------------------------------
75+
# Network configuration:
76+
print ("Loading network/training configuration.")
77+
print ("Config file: " + str(__name__))
78+
79+
BATCH_SIZE = 10
80+
NB_EPOCHS_AUTOENCODER = 40
81+
82+
OPTIM_A = Adam(lr=0.0001, beta_1=0.5)
83+
# OPTIM_A = SGD(lr=0.000001, momentum=0.5, nesterov=True)
84+
# OPTIM_A = rmsprop(lr=0.00001)
85+
86+
lr_schedule = [10, 20, 30] # epoch_step
87+
88+
def schedule(epoch_idx):
89+
if (epoch_idx + 1) < lr_schedule[0]:
90+
return 0.0001
91+
elif (epoch_idx + 1) < lr_schedule[1]:
92+
return 0.0001 # lr_decay_ratio = 10
93+
elif (epoch_idx + 1) < lr_schedule[2]:
94+
return 0.0001
95+
return 0.0001
96+
97+
# aclstm_1 = ConvLSTM2D(filters=1,
98+
# kernel_size=(3, 3),
99+
# strides=(1, 1),
100+
# padding='same',
101+
# return_sequences=True,
102+
# recurrent_dropout=0.2)(h_4)
103+
# x = TimeDistributed(BatchNormalization())(aclstm_1)
104+
#
105+
# flat_1 = TimeDistributed(Flatten())(x)
106+
# dense_1 = TimeDistributed(Dense(units=128 * 208, activation='softmax'))(flat_1)
107+
# x = TimeDistributed(BatchNormalization())(dense_1)
108+
# x = TimeDistributed(Dropout(0.2))(x)
109+
# print (x.shape)
110+
# a1_reshape = Reshape(target_shape=(int(VIDEO_LENGTH/2), 128, 208, 1))(x)
111+
# a1 = AttnLossLayer()(a1_reshape)
112+
# dot_1 = multiply([out_4, a1])
113+
114+

‎code/autoencoder_model/scripts/config_vm.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,13 @@
2424
else:
2525
path_var = 'zhora/'
2626

27-
DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/train/'
27+
DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/train/'
2828
# DATA_DIR= '/local_home/data/KITTI_data/'
2929

30-
VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/val/'
30+
VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/val/'
3131

32-
TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/test/'
33-
# TEST_DATA_DIR= '/local_home/JAAD_Dataset/fun_experiments/resized/'
32+
# TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_208/test/'
33+
TEST_DATA_DIR= '/local_home/JAAD_Dataset/fun_experiments/resized/'
3434

3535
MODEL_DIR = './../' + path_var + 'models'
3636
if not os.path.exists(MODEL_DIR):
@@ -66,7 +66,7 @@
6666
SAVE_GENERATED_IMAGES = True
6767
SHUFFLE = True
6868
VIDEO_LENGTH = 32
69-
IMG_SIZE = (128, 128, 3)
69+
IMG_SIZE = (128, 208, 3)
7070
ATTN_COEFF = 0
7171
KL_COEFF = 0
7272
RAM_DECIMATE = False
@@ -94,4 +94,21 @@ def schedule(epoch_idx):
9494
return 0.0001
9595
return 0.0001
9696

97+
# aclstm_1 = ConvLSTM2D(filters=1,
98+
# kernel_size=(3, 3),
99+
# strides=(1, 1),
100+
# padding='same',
101+
# return_sequences=True,
102+
# recurrent_dropout=0.2)(h_4)
103+
# x = TimeDistributed(BatchNormalization())(aclstm_1)
104+
#
105+
# flat_1 = TimeDistributed(Flatten())(x)
106+
# dense_1 = TimeDistributed(Dense(units=128 * 208, activation='softmax'))(flat_1)
107+
# x = TimeDistributed(BatchNormalization())(dense_1)
108+
# x = TimeDistributed(Dropout(0.2))(x)
109+
# print (x.shape)
110+
# a1_reshape = Reshape(target_shape=(int(VIDEO_LENGTH/2), 128, 208, 1))(x)
111+
# a1 = AttnLossLayer()(a1_reshape)
112+
# dot_1 = multiply([out_4, a1])
113+
97114

0 commit comments

Comments
 (0)