Reversed input to eliminate frame backlash

Author: pratikgujjar

code/autoencoder_model/scripts/aa10.py

@@ -99,22 +99,22 @@ def decoder_model():
     x = TimeDistributed(LeakyReLU(alpha=0.2))(x)
     out_1 = TimeDistributed(Dropout(0.5))(x)
 
-    # flat_1 = TimeDistributed(Flatten())(out_1)
-    # aclstm_1 = GRU(units=16 * 16,
-    #                recurrent_dropout=0.2,
-    #                return_sequences=True)(flat_1)
-    # x = TimeDistributed(BatchNormalization())(aclstm_1)
-    # dense_1 = TimeDistributed(Dense(units=16 * 16, activation='softmax'))(x)
-    # a1_reshape = Reshape(target_shape=(int(VIDEO_LENGTH/2), 16, 16, 1))(dense_1)
-    # a1 = AttnLossLayer()(a1_reshape)
-    # dot_1 = multiply([out_1, a1])
+    flat_1 = TimeDistributed(Flatten())(out_1)
+    aclstm_1 = GRU(units=16 * 16,
+                   recurrent_dropout=0.2,
+                   return_sequences=True)(flat_1)
+    x = TimeDistributed(BatchNormalization())(aclstm_1)
+    dense_1 = TimeDistributed(Dense(units=16 * 16, activation='softmax'))(x)
+    a1_reshape = Reshape(target_shape=(int(VIDEO_LENGTH/2), 16, 16, 1))(dense_1)
+    a1 = AttnLossLayer()(a1_reshape)
+    dot_1 = multiply([out_1, a1])
 
     convlstm_2 = ConvLSTM2D(filters=64,
                             kernel_size=(3, 3),
                             strides=(1, 1),
                             padding='same',
                             return_sequences=True,
-                            recurrent_dropout=0.2)(out_1)
+                            recurrent_dropout=0.2)(dot_1)
     x = TimeDistributed(BatchNormalization())(convlstm_2)
     h_2 = TimeDistributed(LeakyReLU(alpha=0.2))(x)
     out_2 = UpSampling3D(size=(1, 2, 2))(h_2)
@@ -147,29 +147,13 @@ def decoder_model():
     h_4 = TimeDistributed(LeakyReLU(alpha=0.2))(x)
     out_4 = UpSampling3D(size=(1, 2, 2))(h_4)
 
-    aclstm_1 = ConvLSTM2D(filters=1,
-                          kernel_size=(3, 3),
-                          strides=(1, 1),
-                          padding='same',
-                          return_sequences=True,
-                          recurrent_dropout=0.2)(h_4)
-    x = TimeDistributed(BatchNormalization())(aclstm_1)
-    flat_1 = TimeDistributed(Flatten())(x)
-    dense_1 = TimeDistributed(Dense(units=128 * 128, activation='softmax'))(flat_1)
-    x = TimeDistributed(BatchNormalization())(dense_1)
-    x = TimeDistributed(Dropout(0.2))(x)
-    print (x.shape)
-    a1_reshape = Reshape(target_shape=(int(VIDEO_LENGTH/2), 128, 128, 1))(x)
-    a1 = AttnLossLayer()(a1_reshape)
-    dot_1 = multiply([out_4, a1])
-
     # 10x128x128
     convlstm_5 = ConvLSTM2D(filters=3,
                             kernel_size=(3, 3),
                             strides=(1, 1),
                             padding='same',
                             return_sequences=True,
-                            recurrent_dropout=0.2)(dot_1)
+                            recurrent_dropout=0.2)(out_4)
     predictions = TimeDistributed(Activation('tanh'))(convlstm_5)
 
     model = Model(inputs=inputs, outputs=predictions)
@@ -479,8 +463,6 @@ def test(ENC_WEIGHTS, DEC_WEIGHTS):
     decoder = decoder_model()
     autoencoder = autoencoder_model(encoder, decoder)
 
-    if not os.path.exists(TEST_RESULTS_DIR + '/orig/'):
-            os.mkdir(TEST_RESULTS_DIR + '/orig/')
     if not os.path.exists(TEST_RESULTS_DIR + '/truth/'):
         os.mkdir(TEST_RESULTS_DIR + '/truth/')
     if not os.path.exists(TEST_RESULTS_DIR + '/pred/'):
@@ -501,7 +483,10 @@ def l1_l2_loss(y_true, y_pred):
     # n_test_videos = test_videos_list.shape[0]
 
     # NB_TEST_ITERATIONS = int(n_test_videos / BATCH_SIZE)
-    NB_TEST_ITERATIONS = 30
+    path, dirs, files = os.walk(TEST_DATA_DIR).next()
+    file_count = len(files)
+    NB_TEST_ITERATIONS = int((file_count/int(VIDEO_LENGTH/2))/BATCH_SIZE)
+    # NB_TEST_ITERATIONS = 30
 
     test_loss = []
     print (TEST_DATA_DIR)

code/autoencoder_model/scripts/attention_autoencoder.py

@@ -487,14 +487,38 @@ def get_action_classes(action_labels):
 
 
         simple_ped_action_per_frame = set(simple_ped_action_per_frame)
-        for action in simple_ped_action_per_frame:
-            count[action] = count[action] + 1
-            # Add all unique categorical one-hot vectors
-            encoded_ped_action = encoded_ped_action + to_categorical(action, len(simple_ped_set))
+
+        if 7 in simple_ped_action_per_frame:
+            action = 7
+        if 6 in simple_ped_action_per_frame:
+            action = 6
+        if 2 in simple_ped_action_per_frame:
+            action = 2
+        if 0 in simple_ped_action_per_frame:
+            action = 0
+        if 1 in simple_ped_action_per_frame:
+            action = 1
+        if 5 in simple_ped_action_per_frame:
+            action = 5
+        if 3 in simple_ped_action_per_frame:
+            action = 3
+        if 4 in simple_ped_action_per_frame:
+            action = 4
+
+        encoded_ped_action = to_categorical(action, len(simple_ped_set))
+        count[action] = count[action] + 1
+
+        # for action in simple_ped_action_per_frame:
+        #     count[action] = count[action] + 1
+        #     # Add all unique categorical one-hot vectors
+        #     encoded_ped_action = encoded_ped_action + to_categorical(action, len(simple_ped_set))
 
         if (sum(encoded_ped_action) == 0):
             print (simple_ped_action_per_frame)
             print (a_clean)
+        if (sum(encoded_ped_action) > 1):
+            print (simple_ped_action_per_frame)
+            print (a_clean)
         ped_action_class.append(encoded_ped_action.T)
         # action_class.append((encoded_driver_action + encoded_ped_action).T)
 

code/autoencoder_model/scripts/classifier.py

@@ -0,0 +1,97 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from keras.optimizers import SGD
+from keras.optimizers import Adam
+from keras.optimizers import adadelta
+from keras.optimizers import rmsprop
+from keras.layers import Layer
+from keras import backend as K
+K.set_image_dim_ordering('tf')
+import socket
+import os
+
+# -------------------------------------------------
+# Background config:
+hostname = socket.gethostname()
+if hostname == 'baymax':
+    path_var = 'baymax/'
+elif hostname == 'walle':
+    path_var = 'walle/'
+elif hostname == 'bender':
+    path_var = 'bender/'
+else:
+    path_var = 'zhora/'
+
+DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/train/'
+# DATA_DIR= '/local_home/data/KITTI_data/'
+
+VAL_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/val/'
+
+# TEST_DATA_DIR= '/local_home/JAAD_Dataset/iros/resized_imgs_128/test/'
+TEST_DATA_DIR= '/local_home/JAAD_Dataset/fun_experiments/resized/'
+
+MODEL_DIR = './../' + path_var + 'models'
+if not os.path.exists(MODEL_DIR):
+    os.mkdir(MODEL_DIR)
+
+CHECKPOINT_DIR = './../' + path_var + 'checkpoints'
+if not os.path.exists(CHECKPOINT_DIR):
+    os.mkdir(CHECKPOINT_DIR)
+
+ATTN_WEIGHTS_DIR = './../' + path_var + 'attn_weights'
+if not os.path.exists(ATTN_WEIGHTS_DIR):
+    os.mkdir(ATTN_WEIGHTS_DIR)
+
+GEN_IMAGES_DIR = './../' + path_var + 'generated_images'
+if not os.path.exists(GEN_IMAGES_DIR):
+    os.mkdir(GEN_IMAGES_DIR)
+
+LOG_DIR = './../' + path_var + 'logs'
+if not os.path.exists(LOG_DIR):
+    os.mkdir(LOG_DIR)
+
+TF_LOG_DIR = './../' + path_var + 'tf_logs'
+if not os.path.exists(TF_LOG_DIR):
+    os.mkdir(TF_LOG_DIR)
+
+TEST_RESULTS_DIR = './../' + path_var + 'test_results'
+if not os.path.exists(TEST_RESULTS_DIR):
+    os.mkdir(TEST_RESULTS_DIR)
+
+PRINT_MODEL_SUMMARY = True
+SAVE_MODEL = True
+PLOT_MODEL = True
+SAVE_GENERATED_IMAGES = True
+SHUFFLE = True
+VIDEO_LENGTH = 20
+IMG_SIZE = (128, 128, 3)
+ATTN_COEFF = 0
+KL_COEFF = 0
+RAM_DECIMATE = False
+
+# -------------------------------------------------
+# Network configuration:
+print ("Loading network/training configuration.")
+print ("Config file: " + str(__name__))
+
+BATCH_SIZE = 10
+NB_EPOCHS_AUTOENCODER = 40
+
+OPTIM_A = Adam(lr=0.0001, beta_1=0.5)
+# OPTIM_A = SGD(lr=0.000001, momentum=0.5, nesterov=True)
+# OPTIM_A = rmsprop(lr=0.00001)
+
+lr_schedule = [10, 20, 30]  # epoch_step
+
+def schedule(epoch_idx):
+    if (epoch_idx + 1) < lr_schedule[0]:
+        return 0.0001
+    elif (epoch_idx + 1) < lr_schedule[1]:
+        return 0.0001  # lr_decay_ratio = 10
+    elif (epoch_idx + 1) < lr_schedule[2]:
+        return 0.0001
+    return 0.0001
+
+