Build scripts for TFLite (x64, Linux & macOS) (#251)

Author: rafie

get_deps.sh

@@ -11,7 +11,7 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 
 if [[ $1 == --help || $1 == help ]]; then
 	cat <<-END
-		get_deps.sh [cpu|gpu] [--help|help]
+		[ARGVARS...] get_deps.sh [cpu|gpu] [--help|help]
 		
 		Argument variables:
 		VERBOSE=1       Print commands
@@ -145,6 +145,7 @@ if [[ $WITH_TFLITE != 0 ]]; then
 	if [[ ! -d $LIBTFLITE ]]; then
 		echo "Installing TensorFlow Lite ..."
 
+		LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow
 		if [[ $OS == linux ]]; then
 			TFLITE_OS="linux"
 			# if [[ $GPU == no ]]; then
@@ -153,7 +154,6 @@ if [[ $WITH_TFLITE != 0 ]]; then
 			# 	TFLITE_BUILD="gpu"
 			# fi
 
-			LIBTF_URL_BASE=https://s3.amazonaws.com/redismodules/tensorflow
 			if [[ $ARCH == x64 ]]; then
 				TFLITE_ARCH=x86_64
 			elif [[ $ARCH == arm64v8 ]]; then

opt/Makefile

@@ -2,9 +2,6 @@
 ROOT:=..
 include readies/mk/main
 
-# temporarily disabled
-# export WITH_TFLITE=0
-
 MK_CMAKE:=1
 MK_CMAKE_INSTALL:=1
 

opt/build/tflite/Dockerfile.x64

@@ -21,5 +21,9 @@ ADD ./opt/build/tflite/build /build/
 ADD ./opt/readies/ /build/readies/
 ADD ./opt/build/tflite/collect.py /build/
 
-RUN ./build
+RUN set -e ;\
+	cd tensorflow/tensorflow/lite/tools/make ;\
+	./download_dependencies.sh ;\
+	./build_lib.sh
+
 RUN ./collect.py --version ${FTLITE_VER} --dest /build/dest

opt/build/tflite/Makefile

@@ -1,18 +1,25 @@
 
 ROOT=../../..
 
-VERSION ?= 2.0.0
+export VERSION ?= 2.0.0
 OSNICK ?= buster
 
 #----------------------------------------------------------------------------------------------
 
 S3_URL=redismodules/tensorflow
 
 OS:=$(shell $(ROOT)/opt/readies/bin/platform --os)
-ifeq ($(OS),macosx)
-OS:=macos
+
+ifeq ($(OS),linux)
+OS.publish:=$(OS)
+ARCH.publish:=$(ARCH)
+
+else ifeq ($(OS),macosx)
+OS.publish:=darwin
+ARCH.publish:=x86_64
 endif
-STEM=libtensorflowlite-$(OS)
+
+STEM=libtensorflowlite-$(OS.publish)
 
 DOCKER_OS.bionic=ubuntu:bionic
 DOCKER_OS.stretch=debian:stretch-slim
@@ -21,6 +28,8 @@ DOCKER_OS=$(DOCKER_OS.$(OSNICK))
 
 #----------------------------------------------------------------------------------------------
 
+ifeq ($(OS),linux)
+
 define targets # (1=OP, 2=op)
 $(1)_TARGETS :=
 $(1)_TARGETS += $(if $(findstring $(X64),1),$(2)_x64)
@@ -30,6 +39,14 @@ $(1)_TARGETS += $(if $(findstring $(ARM8),1),$(2)_arm64v8)
 $(1)_TARGETS += $$(if $$(strip $$($(1)_TARGETS)),,$(2)_x64 $(2)_arm32v7 $(2)_arm64v8)
 endef
 
+else ifeq ($(OS),macosx)
+
+define targets # (1=OP, 2=op)
+$(1)_TARGETS := $(2)_x64
+endef
+
+endif
+
 $(eval $(call targets,BUILD,build))
 $(eval $(call targets,PUBLISH,publish))
 
@@ -95,12 +112,6 @@ $(error Please run 'aws configure' and provide it with access credentials)
 endif
 endif
 
-else ifeq ($(OS),macosx)
-
-build: ;
-
-endif # macosx
-
 publish: $(PUBLISH_TARGETS)
 
 $(eval $(call publish_x64,x64,x86_64))
@@ -110,4 +121,19 @@ $(eval $(call publish_arm,arm32v7,arm))
 help:
 	@echo "make [build|publish] [X64=1|ARM7=1|ARM8=1]"
 
+else ifeq ($(OS),macosx)
+
+build:
+	@VERSION=$(VERSION) ./build.macos
+	@mv macos/dest/$(STEM)-$(ARCH.publish)-$(VERSION).tar.gz .
+
+publish: $(PUBLISH_TARGETS)
+
+$(eval $(call publish_x64,x64,x86_64))
+
+help:
+	@echo "make [build|publish]"
+
+endif # macosx
+
 .PHONY: all build publish help

opt/build/tflite/build

@@ -12,8 +12,7 @@ if [[ $OS == linux ]]; then
 	elif [[ $ARCH == arm64v8 ]]; then
 		bash build_aarch64_lib.sh
 	elif [[ $ARCH == arm32v7 ]]; then
-		echo "Error: arm32 not supported by TFLite"
-		exit 1
+		bash build_rpi_lib.sh
 	fi
 elif [[ $OS == macosx ]]; then
 	TARGET=osx

opt/build/tflite/build.macos

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+cd $HERE
+
+set -e
+mkdir -p macos
+cd macos
+cp ../collect.py .
+ln -s ../../../readies/
+git clone --single-branch --branch v${VERSION} --depth 1 https://github.com/tensorflow/tensorflow.git
+./collect.py --version $VERSION --dest dest

opt/build/tflite/collect.py

@@ -32,8 +32,10 @@
 platform = paella.Platform()
 
 tf_os = platform.os
+tf_os_internal = tf_os
 if tf_os == 'macosx':
     tf_os = 'darwin'
+    tf_os_internal = 'osx'
 
 tf_arch = platform.arch
 if tf_arch == 'x64':
@@ -68,7 +70,7 @@ def collect_tflite():
             with cwd('downloads/flatbuffers/include'):
                 for f in Path('.').glob('**/*.h'):
                     copy_p(f, d_tensorflow/'include')
-            with cwd(f'gen/linux_{tf_arch}/lib'):
+            with cwd(f'gen/{tf_os_internal}_{tf_arch}/lib'):
                 for f in Path('.').glob('*.a'):
                     copy_p(f, d_tensorflow/'lib')
     create_tar(dest/f'libtensorflowlite-{tf_os}-{tf_arch}-{tf_ver}.tar.gz', dest)