Fix benchmark Caffe2 model (#147)

* Fix benchmark Caffe2 model

*onnx_graph_to_caffe2_net takes onnx_model as argument and not the graph
*Modify bechmark_caffe2_model function to match the same method used in the already built in benchmark_pytorch_model

* Update tutorials/CorrectnessVerificationAndPerformanceComparison.ipynb

* Update tutorials/CorrectnessVerificationAndPerformanceComparison.ipynb

* Update tutorials/CorrectnessVerificationAndPerformanceComparison.ipynb

Co-authored-by: Vinitra Swamy <vinitras@gmail.com>

Author: marouenez

tutorials/CorrectnessVerificationAndPerformanceComparison.ipynb

@@ -22,23 +22,15 @@
    "cell_type": "code",
    "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "WARNING:root:This caffe2 python run does not have GPU support. Will run in CPU only mode.\n",
-      "WARNING:root:Debug message: No module named caffe2_pybind11_state_gpu\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from __future__ import absolute_import\n",
     "from __future__ import division\n",
     "from __future__ import print_function\n",
     "from __future__ import unicode_literals\n",
     "\n",
     "import io\n",
+    "import time\n",
     "import numpy as np\n",
     "import torch\n",
     "import onnx\n",
@@ -49,8 +41,7 @@
     "from caffe2.python import core\n",
     "from torch.autograd import Variable\n",
     "from caffe2.python.onnx.backend import Caffe2Backend\n",
-    "from caffe2.python.onnx.helper import c2_native_run_net, save_caffe2_net, load_caffe2_net, \\\n",
-    "    benchmark_caffe2_model, benchmark_pytorch_model\n",
+    "from caffe2.python.onnx.helper import c2_native_run_net, save_caffe2_net, load_caffe2_net,benchmark_pytorch_model\n",
     "\n",
     "\n",
     "class MNIST(nn.Module):\n",
@@ -97,34 +88,28 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "graph(%0 : Float(3, 1, 28, 28)\n",
-      "      %1 : Float(10, 1, 5, 5)\n",
-      "      %2 : Float(10)\n",
-      "      %3 : Float(20, 10, 5, 5)\n",
-      "      %4 : Float(20)\n",
-      "      %5 : Float(50, 320)\n",
-      "      %6 : Float(50)\n",
-      "      %7 : Float(10, 50)\n",
-      "      %8 : Float(10)) {\n",
-      "  %9 : UNKNOWN_TYPE = Conv[kernel_shape=[5, 5], strides=[1, 1], pads=[0, 0, 0, 0], dilations=[1, 1], group=1](%0, %1)\n",
-      "  %10 : Float(3, 10, 24, 24) = Add[broadcast=1, axis=1](%9, %2)\n",
-      "  %11 : Float(3, 10, 12, 12) = MaxPool[kernel_shape=[2, 2], pads=[0, 0], strides=[2, 2]](%10)\n",
-      "  %12 : Float(3, 10, 12, 12) = Relu(%11)\n",
-      "  %13 : UNKNOWN_TYPE = Conv[kernel_shape=[5, 5], strides=[1, 1], pads=[0, 0, 0, 0], dilations=[1, 1], group=1](%12, %3)\n",
-      "  %14 : Float(3, 20, 8, 8) = Add[broadcast=1, axis=1](%13, %4)\n",
-      "  %15 : Float(3, 20, 4, 4) = MaxPool[kernel_shape=[2, 2], pads=[0, 0], strides=[2, 2]](%14)\n",
-      "  %16 : Float(3, 20, 4, 4) = Relu(%15)\n",
-      "  %17 : Float(3, 320) = Reshape[shape=[-1, 320]](%16)\n",
-      "  %18 : Float(320!, 50!) = Transpose[perm=[1, 0]](%5)\n",
-      "  %20 : Float(3, 50) = Gemm[alpha=1, beta=1, broadcast=1](%17, %18, %6)\n",
-      "  %21 : Float(3, 50) = Relu(%20)\n",
-      "  %22 : Float(3, 50), %23 : UNKNOWN_TYPE = Dropout[is_test=1, ratio=0.5](%21)\n",
-      "  %24 : Float(50!, 10!) = Transpose[perm=[1, 0]](%7)\n",
-      "  %26 : Float(3, 10) = Gemm[alpha=1, beta=1, broadcast=1](%22, %24, %8)\n",
-      "  %27 : Float(3, 10) = Softmax[axis=1](%26)\n",
-      "  %28 : Float(3, 10) = Log(%27)\n",
-      "  return (%28);\n",
-      "}\n",
+      "graph(%input.1 : Float(3, 1, 28, 28),\n",
+      "      %conv1.weight : Float(10, 1, 5, 5),\n",
+      "      %conv1.bias : Float(10),\n",
+      "      %conv2.weight : Float(20, 10, 5, 5),\n",
+      "      %conv2.bias : Float(20),\n",
+      "      %fc1.weight : Float(50, 320),\n",
+      "      %fc1.bias : Float(50),\n",
+      "      %fc2.weight : Float(10, 50),\n",
+      "      %fc2.bias : Float(10)):\n",
+      "  %9 : Float(3, 10, 24, 24) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[5, 5], pads=[0, 0, 0, 0], strides=[1, 1]](%input.1, %conv1.weight, %conv1.bias), scope: MNIST/Conv2d[conv1] # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/modules/conv.py:340:0\n",
+      "  %10 : Float(3, 10, 12, 12) = onnx::MaxPool[kernel_shape=[2, 2], pads=[0, 0, 0, 0], strides=[2, 2]](%9), scope: MNIST # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:487:0\n",
+      "  %11 : Float(3, 10, 12, 12) = onnx::Relu(%10), scope: MNIST # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:913:0\n",
+      "  %12 : Float(3, 20, 8, 8) = onnx::Conv[dilations=[1, 1], group=1, kernel_shape=[5, 5], pads=[0, 0, 0, 0], strides=[1, 1]](%11, %conv2.weight, %conv2.bias), scope: MNIST/Dropout2d[conv2_drop] # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:844:0\n",
+      "  %13 : Float(3, 20, 4, 4) = onnx::MaxPool[kernel_shape=[2, 2], pads=[0, 0, 0, 0], strides=[2, 2]](%12), scope: MNIST # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:487:0\n",
+      "  %14 : Float(3, 20, 4, 4) = onnx::Relu(%13), scope: MNIST # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:913:0\n",
+      "  %15 : Tensor = onnx::Constant[value=  -1  320 [ Variable[CPULongType]{2} ]](), scope: MNIST\n",
+      "  %16 : Float(3, 320) = onnx::Reshape(%14, %15), scope: MNIST # <ipython-input-1-c1a2123bd117>:34:0\n",
+      "  %17 : Float(3, 50) = onnx::Gemm[alpha=1, beta=1, transB=1](%16, %fc1.weight, %fc1.bias), scope: MNIST/Linear[fc1] # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:1369:0\n",
+      "  %18 : Float(3, 50) = onnx::Relu(%17), scope: MNIST # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:806:0\n",
+      "  %19 : Float(3, 10) = onnx::Gemm[alpha=1, beta=1, transB=1](%18, %fc2.weight, %fc2.bias), scope: MNIST/Linear[fc2] # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:1369:0\n",
+      "  %20 : Float(3, 10) = onnx::LogSoftmax[axis=1](%19), scope: MNIST # /home/marouenez/anaconda3/envs/masterthesis/lib/python3.7/site-packages/torch/nn/functional.py:1316:0\n",
+      "  return (%20)\n",
       "\n",
       "Check the ONNX model.\n"
      ]
@@ -164,7 +149,7 @@
    "source": [
     "# Convert the ONNX model to a Caffe2 model.\n",
     "print(\"Convert the model to a Caffe2 model.\")\n",
-    "init_net, predict_net = Caffe2Backend.onnx_graph_to_caffe2_net(onnx_model.graph, device=\"CPU\")"
+    "init_net, predict_net = Caffe2Backend.onnx_graph_to_caffe2_net(onnx_model, device=\"CPU\")"
    ]
   },
   {
@@ -177,9 +162,7 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# Prepare the inputs for Caffe2.\n",
@@ -196,9 +179,7 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# Save the converted Caffe2 model in the protobuf files. (Optional)\n",
@@ -263,6 +244,30 @@
     "print(\"The exported model achieves {}-decimal precision.\".format(expected_decimal))"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def benchmark_caffe2_model(init_net, predict_net,inputs, warmup_iters=3, main_iters=10):\n",
+    "    '''\n",
+    "     Run the model several times, and measure the execution time.\n",
+    "     Print the execution time per iteration (millisecond) and the number of iterations per second.\n",
+    "    '''\n",
+    "    for _i in range(warmup_iters):\n",
+    "        ws, caffe2_results = c2_native_run_net(init_net, predict_net, inputs)    \n",
+    "\n",
+    "    total_time = 0.0\n",
+    "    for _i in range(main_iters):\n",
+    "        ts = time.time()\n",
+    "        ws, caffe2_results = c2_native_run_net(init_net, predict_net, inputs)\n",
+    "        te = time.time()\n",
+    "        total_time += te - ts\n",
+    "        \n",
+    "    return total_time / main_iters * 1000"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -275,46 +280,53 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "PyTorch model's execution time is 0.580716133118 milliseconds/ iteration, 1722.011742 iterations per second.\n",
-      "Caffe2 model's execution time is 0.399529695511 milliseconds / iteration, 2502.94286316 iterations per second\n"
+      "PyTorch model's execution time is 0.6218433380126953 milliseconds/ iteration, 1608.1220765278736 iterations per second.\n",
+      "Caffe2 model's execution time is 3.189969062805176 milliseconds / iteration, 313.48266403581545 iterations per second\n"
      ]
     }
    ],
    "source": [
     "pytorch_time = benchmark_pytorch_model(pytorch_model, inputs)\n",
-    "caffe2_time = benchmark_caffe2_model(init_net, predict_net)\n",
+    "caffe2_time = benchmark_caffe2_model(init_net, predict_net, caffe2_inputs)\n",
     "\n",
     "print(\"PyTorch model's execution time is {} milliseconds/ iteration, {} iterations per second.\".format(\n",
     "    pytorch_time, 1000 / pytorch_time))\n",
     "print(\"Caffe2 model's execution time is {} milliseconds / iteration, {} iterations per second\".format(\n",
     "    caffe2_time, 1000 / caffe2_time))"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 2",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "python2"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.13"
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
   }
  },
  "nbformat": 4,