Improve offline build experience for forge developers

pradeep · pradeep · commit c8eaf765547e · 2021-07-26T19:42:55.000+05:30
The following common scenarios(majority we think) are covered with this change. - Developer has cloud connection always. - Developer has cloud connection for initial cmake run, but not later. - Developer has lost cloud connection for a while after the initial successful cmake run but regained the connection later. - Developer is doing an completely disconnected build using the source tarball we generate and attach to our release assets. When the developer wants to do an offline build for a specific commit other than release tags, they would have to generate the relevant source tarball themselves. The commands required to do the same can be found from the following ci workflow file in our repository. .github/workflows/release_src_artifact.yml The source tarball generation CI job has also been changed to reflect the change in external dependencies location. (cherry picked from commit da1ffc3)
diff --git a/.github/workflows/release_src_artifact.yml b/.github/workflows/release_src_artifact.yml
@@ -52,6 +52,7 @@ jobs:
                   rm -rf forge-full-${FG_VER}/.github
                   rm forge-full-${FG_VER}/.gitmodules
                   cd forge-full-${FG_VER}/build/
+                  cp -r ./examples/third_party/cl2hpp ../extern/cl2hpp
                   shopt -s extglob
                   rm -r !(extern)
                   cd ./extern
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -68,12 +68,10 @@ if(Boost_FOUND AND NOT TARGET Boost::boost)
 endif(Boost_FOUND AND NOT TARGET Boost::boost)
 
 if(NOT TARGET glad::glad) # find_package(glad) failed
-    FetchContent_Declare(
-        ${glad_prefix}
-        GIT_REPOSITORY https://github.com/arrayfire/glad.git
-        GIT_TAG obj_lib
-        )
-    fg_check_and_populate(${glad_prefix})
+    fg_dep_check_and_populate(${glad_prefix}
+        URI https://github.com/arrayfire/glad.git
+        REF obj_lib
+    )
     add_subdirectory(${${glad_prefix}_SOURCE_DIR} ${${glad_prefix}_BINARY_DIR})
 
     add_library(forge_glad STATIC $<TARGET_OBJECTS:glad_obj_lib>)
@@ -86,17 +84,18 @@ else()
     add_library(forge_glad ALIAS glad::glad)
 endif()
 
-if(NOT TARGET glm::glm AND NOT TARGET glm) # find_package(glm) failed
-    FetchContent_Declare(
-        ${glm_prefix}
-        GIT_REPOSITORY https://github.com/g-truc/glm.git
-        GIT_TAG 0.9.9.8
-        )
-    fg_check_and_populate(${glm_prefix})
-    add_library(glm INTERFACE IMPORTED)
-    set_target_properties(glm PROPERTIES
-        INTERFACE_INCLUDE_DIRECTORIES "${${glm_prefix}_SOURCE_DIR}"
-        )
+add_library(forge_glm INTERFACE)
+if(TARGET glm::glm)
+    target_include_directories(forge_glm
+        SYSTEM INTERFACE
+        $<TARGET_PROPERTY:glm::glm,INTERFACE_INCLUDE_DIRECTORIES>
+    )
+else() # find_package(glm) failed
+    fg_dep_check_and_populate(${glm_prefix}
+        URI https://github.com/g-truc/glm.git
+        REF 0.9.9.8
+    )
+    target_include_directories(forge_glm INTERFACE "${${glm_prefix}_SOURCE_DIR}")
 endif()
 
 add_subdirectory(src/backend/common)
diff --git a/CMakeModules/ForgeConfigureDepsVars.cmake b/CMakeModules/ForgeConfigureDepsVars.cmake
@@ -5,15 +5,53 @@
 # The complete license agreement can be obtained at:
 # http://arrayfire.com/licenses/BSD-3-Clause
 
-option(FG_BUILD_OFFLINE "Build Forge assuming there is no network" OFF)
+file(DOWNLOAD
+  "https://github.com/arrayfire/forge/blob/v1.0.0/CMakeLists.txt"
+  "${Forge_BINARY_DIR}/download_copy_cmakelists.stamp"
+  STATUS fg_check_result
+  TIMEOUT 4
+)
+list(GET fg_check_result 0 fg_is_connected)
+if(${fg_is_connected})
+  set(BUILD_OFFLINE ON)
+  # Turn ON disconnected flag when connected to cloud
+  set(FETCHCONTENT_FULLY_DISCONNECTED ON CACHE BOOL
+      "Disable Download/Update stages of FetchContent workflow" FORCE)
+
+  message(STATUS "No cloud connection. Attempting offline build if dependencies are available")
+else()
+  set(BUILD_OFFLINE OFF)
+  # Turn OFF disconnected flag when connected to cloud
+  # This is required especially in the following scenario:
+  # - cmake run successfully first
+  # - lost connection, but development can still be done
+  # - Now, connection regained. Hence updates should be allowed
+  set(FETCHCONTENT_FULLY_DISCONNECTED OFF CACHE BOOL
+      "Disable Download/Update stages of FetchContent workflow" FORCE)
+endif()
+
+# Track dependencies download persistently across multiple
+# cmake configure runs. *_POPULATED variables are reset for each
+# cmake run to 0. Hence, this internal cache value is needed to
+# check for already (from previous cmake run's) populated data
+# during the current cmake run if it looses network connection.
+set(FG_INTERNAL_DOWNLOAD_FLAG OFF CACHE BOOL "Dependencies Download Tracker Flag")
 
 # Override fetch content base dir before including AFfetch_content
 set(FETCHCONTENT_BASE_DIR "${Forge_BINARY_DIR}/extern" CACHE PATH
     "Base directory where Forge dependencies are downloaded and/or built" FORCE)
 
 include(ForgeFetchContent)
 
-macro(set_and_mark_depname var name)
+mark_as_advanced(
+  FG_INTERNAL_DOWNLOAD_FLAG
+  FETCHCONTENT_BASE_DIR
+  FETCHCONTENT_QUIET
+  FETCHCONTENT_FULLY_DISCONNECTED
+  FETCHCONTENT_UPDATES_DISCONNECTED
+)
+
+macro(set_and_mark_depnames_advncd var name)
   string(TOLOWER ${name} ${var})
   string(TOUPPER ${name} ${var}_ucname)
   mark_as_advanced(
@@ -22,31 +60,79 @@ macro(set_and_mark_depname var name)
   )
 endmacro()
 
-mark_as_advanced(
-  FETCHCONTENT_BASE_DIR
-  FETCHCONTENT_QUIET
-  FETCHCONTENT_FULLY_DISCONNECTED
-  FETCHCONTENT_UPDATES_DISCONNECTED
-)
+set_and_mark_depnames_advncd(glad_prefix "fg_glad")
+set_and_mark_depnames_advncd(glm_prefix "fg_glm")
 
-set_and_mark_depname(glad_prefix "fg_glad")
-set_and_mark_depname(glm_prefix "fg_glm")
+macro(fg_dep_check_and_populate dep_prefix)
+  set(single_args URI REF)
+  cmake_parse_arguments(fdcp_args "" "${single_args}" "" ${ARGN})
 
-if(FG_BUILD_OFFLINE)
-  macro(set_fetchcontent_src_dir prefix_var dep_name)
-    set(FETCHCONTENT_SOURCE_DIR_${${prefix_var}_ucname}
-        "${FETCHCONTENT_BASE_DIR}/${${prefix_var}}-src" CACHE PATH
-        "Source directory for ${dep_name} dependency")
-    mark_as_advanced(FETCHCONTENT_SOURCE_DIR_${${prefix_var}_ucname})
-  endmacro()
+  if("${fdcp_args_URI}" STREQUAL "")
+    message(FATAL_ERROR [=[
+        Cannot check requested dependency source's availability.
+        Please provide a valid URI(almost always a URL to a github repo).
+        Note that the above error message if for developers of Forge.
+        ]=])
+  endif()
 
-  set_fetchcontent_src_dir(assets_prefix "glad")
-  set_fetchcontent_src_dir(testdata_prefix "glm")
-endif()
+  string(FIND "${fdcp_args_REF}" "=" fdcp_has_algo_id)
 
-macro(fg_check_and_populate prefix)
-  FetchContent_GetProperties(${prefix})
-  if(NOT ${prefix}_POPULATED)
-    FetchContent_Populate(${prefix})
+  if(${BUILD_OFFLINE} AND NOT ${FG_INTERNAL_DOWNLOAD_FLAG})
+    if(NOT ${fdcp_has_algo_id} EQUAL -1)
+      FetchContent_Populate(${dep_prefix}
+        QUIET
+        URL            ${fdcp_args_URI}
+        URL_HASH       ${fdcp_args_REF}
+        DOWNLOAD_COMMAND \"\"
+        UPDATE_DISCONNECTED ON
+        SOURCE_DIR     "${Forge_SOURCE_DIR}/extern/${dep_prefix}-src"
+        BINARY_DIR     "${Forge_BINARY_DIR}/extern/${dep_prefix}-build"
+        SUBBUILD_DIR   "${Forge_BINARY_DIR}/extern/${dep_prefix}-subbuild"
+      )
+    elseif("${fdcp_args_REF}" STREQUAL "")
+      FetchContent_Populate(${dep_prefix}
+        QUIET
+        URL            ${fdcp_args_URI}
+        DOWNLOAD_COMMAND \"\"
+        UPDATE_DISCONNECTED ON
+        SOURCE_DIR     "${Forge_SOURCE_DIR}/extern/${dep_prefix}-src"
+        BINARY_DIR     "${Forge_BINARY_DIR}/extern/${dep_prefix}-build"
+        SUBBUILD_DIR   "${Forge_BINARY_DIR}/extern/${dep_prefix}-subbuild"
+      )
+    else()
+      # The left over alternative is assumed to be a cloud hosted git repository
+      FetchContent_Populate(${dep_prefix}
+        QUIET
+        GIT_REPOSITORY ${fdcp_args_URI}
+        GIT_TAG        ${fdcp_args_REF}
+        DOWNLOAD_COMMAND \"\"
+        UPDATE_DISCONNECTED ON
+        SOURCE_DIR     "${Forge_SOURCE_DIR}/extern/${dep_prefix}-src"
+        BINARY_DIR     "${Forge_BINARY_DIR}/extern/${dep_prefix}-build"
+        SUBBUILD_DIR   "${Forge_BINARY_DIR}/extern/${dep_prefix}-subbuild"
+      )
+    endif()
+  else()
+    if(NOT ${fdcp_has_algo_id} EQUAL -1)
+      FetchContent_Declare(${dep_prefix}
+        URL            ${fdcp_args_URI}
+        URL_HASH       ${fdcp_args_REF}
+      )
+    elseif("${fdcp_args_REF}" STREQUAL "")
+      FetchContent_Declare(${dep_prefix}
+        URL            ${fdcp_args_URI}
+      )
+    else()
+      # The left over alternative is assumed to be a cloud hosted git repository
+      FetchContent_Declare(${dep_prefix}
+        GIT_REPOSITORY ${fdcp_args_URI}
+        GIT_TAG        ${fdcp_args_REF}
+      )
+    endif()
+    FetchContent_GetProperties(${dep_prefix})
+    if(NOT ${dep_prefix}_POPULATED)
+      FetchContent_Populate(${dep_prefix})
+    endif()
+    set(FG_INTERNAL_DOWNLOAD_FLAG ON CACHE BOOL "Dependencies Download Tracker Flag" FORCE)
   endif()
 endmacro()
diff --git a/CMakeModules/ForgeInternalUtils.cmake b/CMakeModules/ForgeInternalUtils.cmake
@@ -93,24 +93,8 @@ function(fg_set_target_compilation_props target)
         SYSTEM PRIVATE
         $<TARGET_PROPERTY:Boost::boost,INTERFACE_INCLUDE_DIRECTORIES>
         $<TARGET_PROPERTY:forge_glad,INTERFACE_INCLUDE_DIRECTORIES>
+        $<TARGET_PROPERTY:forge_glm,INTERFACE_INCLUDE_DIRECTORIES>
         )
-    # As imported targets can't be aliased until CMake 3.11, below is a temporary
-    # work around until we move to a CMake version >= 3.11
-    if(TARGET glm::glm)
-        target_include_directories(${target}
-            SYSTEM PRIVATE
-            $<TARGET_PROPERTY:glm::glm,INTERFACE_INCLUDE_DIRECTORIES>
-            )
-    elseif(TARGET glm)
-        target_include_directories(${target}
-            SYSTEM PRIVATE
-            $<TARGET_PROPERTY:glm,INTERFACE_INCLUDE_DIRECTORIES>
-            )
-    else()
-        # Ideally it wouldn't reach here, just in case of corner case
-        # I couldn't think of or someother cmake bug
-        message(FATAL_ERROR "Please install glm dependency")
-    endif()
     target_include_directories(${target}
         PUBLIC
         $<INSTALL_INTERFACE:${FG_INSTALL_INC_DIR}>
diff --git a/examples/CMakeModules/build_cl2hpp.cmake b/examples/CMakeModules/build_cl2hpp.cmake
@@ -1,23 +1,56 @@
-set(cl2hpp_header
+file(DOWNLOAD
+  "https://github.com/arrayfire/forge/blob/master/.github/LICENSE"
+  "${PROJECT_BINARY_DIR}/LICENSE.md"
+  STATUS fg_check_result
+  TIMEOUT 4
+)
+list(GET fg_check_result 0 fg_is_connected)
+
+set(cl2hpp_extern_header
+    "${Forge_SOURCE_DIR}/extern/cl2hpp/include/CL/cl2.hpp")
+set(cl2hpp_download_header
     "${PROJECT_BINARY_DIR}/third_party/cl2hpp/include/CL/cl2.hpp")
-if (OpenCL_FOUND)
-    if (NOT EXISTS ${cl2hpp_header})
-        set(file_url
-            "https://github.com/KhronosGroup/OpenCL-CLHPP/releases/download/v2.0.10/cl2.hpp")
-        file(DOWNLOAD ${file_url} ${cl2hpp_header}
-            EXPECTED_HASH MD5=c38d1b78cd98cc809fa2a49dbd1734a5
-            STATUS download_result)
-        list(GET download_result 0 download_code)
-        if (NOT ${download_code} EQUAL 0)
-            file(REMOVE ${cl2hpp_header}) #empty file have to be removed
-            message(FATAL_ERROR "Failed to download cl2hpp header")
+
+if(${fg_is_connected} AND NOT EXISTS ${cl2hpp_download_header})
+    if(EXISTS ${cl2hpp_extern_header})
+        get_filename_component(download_dir ${cl2hpp_extern_header} DIRECTORY)
+        message(STATUS "Found cl2.hpp header at: ${download_dir}")
+        if (NOT TARGET OpenCL::cl2hpp)
+            add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
+            set_target_properties(OpenCL::cl2hpp PROPERTIES
+                INTERFACE_INCLUDE_DIRECTORIES "${download_dir}/..")
+        endif ()
+    else()
+        message(FATAL_ERROR [=[
+            Offline builds require already available cl2hpp header. Please
+            download the file from and place it at the below path  path under
+            project root directory.
+
+            Download URL: https://github.com/KhronosGroup/OpenCL-CLHPP/releases/download/v2.0.10/cl2.hpp
+            Target Location: extern/cl2hpp/include/CL
+            ]=])
+    endif()
+else()
+    # Any CMakeLists.txt file including this file should call find_package(OpenCL)
+    if (OpenCL_FOUND)
+        if (NOT EXISTS ${cl2hpp_download_header})
+            set(file_url
+                "https://github.com/KhronosGroup/OpenCL-CLHPP/releases/download/v2.0.10/cl2.hpp")
+            file(DOWNLOAD ${file_url} ${cl2hpp_download_header}
+                EXPECTED_HASH MD5=c38d1b78cd98cc809fa2a49dbd1734a5
+                STATUS download_result)
+            list(GET download_result 0 download_code)
+            if (NOT ${download_code} EQUAL 0)
+                file(REMOVE ${cl2hpp_download_header}) #empty file have to be removed
+                message(FATAL_ERROR "Failed to download cl2hpp header")
+            endif ()
+        endif ()
+        get_filename_component(download_dir ${cl2hpp_download_header} DIRECTORY)
+        message(STATUS "Found cl2.hpp header at: ${download_dir}")
+        if (NOT TARGET OpenCL::cl2hpp)
+            add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
+            set_target_properties(OpenCL::cl2hpp PROPERTIES
+                INTERFACE_INCLUDE_DIRECTORIES "${download_dir}/..")
         endif ()
     endif ()
-    get_filename_component(download_dir ${cl2hpp_header} DIRECTORY)
-    message(STATUS "Found cl2.hpp header at: ${download_dir}")
-    if (NOT TARGET OpenCL::cl2hpp)
-        add_library(OpenCL::cl2hpp IMPORTED INTERFACE GLOBAL)
-        set_target_properties(OpenCL::cl2hpp PROPERTIES
-            INTERFACE_INCLUDE_DIRECTORIES "${download_dir}/..")
-    endif ()
-endif ()
+endif()
diff --git a/src/backend/CMakeLists.txt b/src/backend/CMakeLists.txt
@@ -56,15 +56,19 @@ if(FG_WITH_FREEIMAGE)
     endif()
 endif()
 
-
 target_link_libraries(${BkndTargetName}
     PRIVATE
     Boost::boost
-    forge_glad
     ${FREETYPE_LIBRARIES}
     ${CMAKE_DL_LIBS}
     )
 
+if(TARGET glad::glad)
+    target_link_libraries(${BkndTargetName} PRIVATE glad::glad)
+else()
+    target_link_libraries(${BkndTargetName} PRIVATE forge_glad)
+endif()
+
 if(UNIX)
     target_link_libraries(${BkndTargetName}
         PRIVATE
