Boosted inference speed

Author: antoinecharbel-inmind

src/main/deep_learning_service.py

@@ -1,7 +1,7 @@
 import os
+import re
 import json
 import uuid
-import re
 from inference.inference_engines_factory import InferenceEngineFactory
 from inference.exceptions import ModelNotFound, InvalidModelConfiguration, ModelNotLoaded, InferenceEngineNotFound, \
     InvalidInputData, ApplicationError
@@ -75,14 +75,14 @@ def load_models(self, model_names):
         for model in model_names:
             self.load_model(model)
 
-    async def run_model(self, model_name, input_data, draw_boxes, predict_batch):
+    async def run_model(self, model_name, input_data, draw, predict_batch):
         """
         Loads the model in case it was never loaded and calls the inference engine class to get a prediction.
         :param model_name: Model name
         :param input_data: Batch of images or a single image
-        :param draw_boxes: Boolean to specify if we need to draw the response on the input image
+        :param draw: Boolean to specify if we need to draw the response on the input image
         :param predict_batch: Boolean to specify if there is a batch of images in a request or not
-        :return: Model response in case draw_boxes was set to False, else an actual image
+        :return: Model response in case draw was set to False, else an actual image
         """
         if re.match(r'[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}', model_name,
                     flags=0):
@@ -92,18 +92,18 @@ async def run_model(self, model_name, input_data, draw_boxes, predict_batch):
         if self.model_loaded(model_name):
             try:
                 if predict_batch:
-                    return await self.models_dict[model_name].run_batch(input_data, draw_boxes, predict_batch)
+                    return await self.models_dict[model_name].run_batch(input_data, draw, predict_batch)
                 else:
-                    if not draw_boxes:
-                        return await self.models_dict[model_name].run(input_data, draw_boxes, predict_batch)
+                    if not draw:
+                        return await self.models_dict[model_name].infer(input_data, draw, predict_batch)
                     else:
-                        await self.models_dict[model_name].run(input_data, draw_boxes, predict_batch)
+                        await self.models_dict[model_name].infer(input_data, draw, predict_batch)
             except ApplicationError as e:
                 raise e
         else:
             try:
                 self.load_model(model_name)
-                return await self.run_model(model_name, input_data, draw_boxes, predict_batch)
+                return await self.run_model(model_name, input_data, draw, predict_batch)
             except ApplicationError as e:
                 raise e
 

src/main/inference/__init__.py

@@ -12,7 +12,7 @@ def __init__(self):
         """
         self.logger = logging.getLogger('logger')
         date = datetime.now().strftime('%Y-%m-%d')
-        file_path = 'logs/tensorflow_inference_engine_' + date + '.log'
+        file_path = 'logs/' + date + '.log'
         self.handler = logging.FileHandler(file_path)
         self.handler.setLevel(logging.INFO)
         self.handler.setFormatter(logging.Formatter("%(levelname)s;%(asctime)s;%(message)s"))

src/main/inference/base_error.py

@@ -4,89 +4,90 @@
 
 class AbstractInferenceEngine(ABC):
 
-    def __init__(self, model_path):
-        """
-        Takes a model path and calls the load function.
-        :param model_path: The model's path
-        :return:
-        """
-        self.labels = []
-        self.configuration = {}
-        self.model_path = model_path
-        try:
-            self.validate_configuration()
-        except ApplicationError as e:
-            raise e
-        try:
-            self.load()
-        except ApplicationError as e:
-            raise e
-        except Exception as e:
-            raise ModelNotLoaded()
+	def __init__(self, model_path):
+		"""
+		Takes a model path and calls the load function.
+		:param model_path: The model's path
+		:return:
+		"""
+		self.labels = []
+		self.configuration = {}
+		self.model_path = model_path
+		try:
+			self.validate_configuration()
+		except ApplicationError as e:
+			raise e
+		try:
+			self.load()
+		except ApplicationError as e:
+			raise e
+		except Exception as e:
+			print(e)
+			raise ModelNotLoaded()
 
-    @abstractmethod
-    def load(self):
-        """
-        Loads the model based on the underlying implementation.
-        """
-        pass
+	@abstractmethod
+	def load(self):
+		"""
+		Loads the model based on the underlying implementation.
+		"""
+		pass
 
-    @abstractmethod
-    def free(self):
-        """
-        Performs any manual memory implementation required to when unloading a model.
-        Will be called when the class's destructor is called.
-        """
-        pass
+	@abstractmethod
+	def free(self):
+		"""
+		Performs any manual memory implementation required to when unloading a model.
+		Will be called when the class's destructor is called.
+		"""
+		pass
 
-    @abstractmethod
-    async def run(self, input_data, draw_boxes, predict_batch):
-        """
-        Performs the required inference based on the underlying implementation of this class.
-        Could be used to return classification predictions, object detection coordinates...
-        :param predict_batch: Boolean
-        :param input_data: A single image
-        :param draw_boxes: Used to draw bounding boxes on image instead of returning them
-        :return: A bounding-box
-        """
-        pass
+	@abstractmethod
+	async def infer(self, input_data, draw, predict_batch):
+		"""
+		Performs the required inference based on the underlying implementation of this class.
+		Could be used to return classification predictions, object detection coordinates...
+		:param predict_batch: Boolean
+		:param input_data: A single image
+		:param draw: Used to draw bounding boxes on image instead of returning them
+		:return: A bounding-box
+		"""
+		pass
 
-    @abstractmethod
-    async def run_batch(self, input_data, draw_boxes, predict_batch):
-        """
-        Iterates over images and returns a prediction for each one.
-        :param predict_batch: Boolean
-        :param input_data: List of images
-        :param draw_boxes: Used to draw bounding boxes on image instead of returning them
-        :return: List of bounding-boxes
-        """
-        pass
+	@abstractmethod
+	async def run_batch(self, input_data, draw, predict_batch):
+		"""
+		Iterates over images and returns a prediction for each one.
+		:param predict_batch: Boolean
+		:param input_data: List of images
+		:param draw: Used to draw bounding boxes on image instead of returning them
+		:return: List of bounding-boxes
+		"""
+		pass
 
-    @abstractmethod
-    def validate_configuration(self):
-        """
-        Validates that the model and its files are valid based on the underlying implementation's requirements.
-        Can check for configuration values, folder structure...
-        """
-        pass
+	@abstractmethod
+	def validate_configuration(self):
+		"""
+		Validates that the model and its files are valid based on the underlying implementation's requirements.
+		Can check for configuration values, folder structure...
+		"""
+		pass
 
-    @abstractmethod
-    def set_configuration(self, data):
-        """
-        Takes the configuration from the config.json file
-        :param data: Json data
-        :return:
-        """
-        pass
+	@abstractmethod
+	def set_model_configuration(self, data):
+		"""
+		Takes the configuration from the config.json file
+		:param data: Json data
+		:return:
+		"""
+		pass
 
-    @abstractmethod
-    def validate_json_configuration(self, data):
-        """
-        Validates the configuration of the config.json file.
-        :param data: Json data
-        :return:
-        """
-        pass
+	@abstractmethod
+	def validate_json_configuration(self, data):
+		"""
+		Validates the configuration of the config.json file.
+		:param data: Json data
+		:return:
+		"""
+		pass
 
-    def __del__(self):
-        self.free()
+	def __del__(self):
+		self.free()

src/main/inference/base_inference_engine.py

@@ -1,6 +1,6 @@
 import os
 import logging
-from datetime import datetime, date
+from datetime import datetime
 from inference.base_error import AbstractError
 
 
@@ -31,15 +31,15 @@ def check_date(self):
         :return:
         """
         self.date = datetime.now().strftime('%Y-%m-%d')
-        file_path = 'tensorflow_inference_engine_' + self.date + '.log'
+        file_path = self.date + '.log'
         if file_path not in os.listdir('logs'):
             self.logger.removeHandler(self.handler)
             self.handler = logging.FileHandler('logs/' + file_path)
             self.handler.setLevel(logging.INFO)
             self.handler.setFormatter(logging.Formatter("%(levelname)s;%(asctime)s;%(message)s"))
             self.logger.addHandler(self.handler)
         oldest_log_file = os.listdir('logs')[0]
-        oldest_date = oldest_log_file.split("_")[3].split('.')[0]
+        oldest_date = oldest_log_file.split('.')[0]
         a = datetime.strptime(datetime.now().strftime('%Y-%m-%d'), '%Y-%m-%d')
         b = datetime.strptime(oldest_date, '%Y-%m-%d')
         delta = a - b

src/main/inference/errors.py

@@ -27,6 +27,8 @@ def get_engine(path_to_model):
             # model instance
             return getattr(__import__(inference_engine_name), 'InferenceEngine')(path_to_model)
         except ApplicationError as e:
+            print(e)
             raise e
         except Exception as e:
+            print(e)
             raise InferenceEngineNotFound(inference_engine_name)