rp2,esp32,extmod: Implement UPDATE_SUBMODULES in CMake.

Rather than having Make calling CMake to generate a list of submodules and
then run a Make target (which is complex and prone to masking other
errors), implement the submodule update logic in CMake itself.

Internal CMake-side changes are that GIT_SUBMODULES is now a CMake list,
and the trigger variable name is changed from ECHO_SUBMODULES to
UPDATE_SUBMODULES.

The run is otherwise 100% a normal CMake run now, so most of the other
special casing can be removed.

Signed-off-by: Angus Gratton <angus@redyak.com.au>

7 files changed, 105 insertions, 98 deletions

diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake
index 3643f1aee..7cacd0a43 100644
--- a/extmod/extmod.cmake
+++ b/extmod/extmod.cmake
@@ -107,62 +107,57 @@ set(MICROPY_SOURCE_LIB_LIBM_SQRT_HW ${MICROPY_DIR}/lib/libm/thumb_vfp_sqrtf.c)
 
 if(MICROPY_PY_BTREE)
     set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
-    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/berkeley-db-1.xx)
+    list(APPEND GIT_SUBMODULES lib/berkeley-db-1.xx)
 
-    if(ECHO_SUBMODULES)
-        # No-op, we're just doing submodule/variant discovery.
-        # Cannot run the add_library/target_include_directories rules (even though
-        # the build won't run) because IDF will attempt verify the files exist.
-    elseif(NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
+    if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
         # Regular build, submodule not initialised -- fail with a clear error.
         message(FATAL_ERROR " MICROPY_PY_BTREE is enabled but the berkeley-db submodule is not initialised.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
-    else()
-        # Regular build, we have the submodule.
-        add_library(micropy_extmod_btree OBJECT
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
-            ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
-            ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
-        )
+    endif()
 
-        target_include_directories(micropy_extmod_btree PRIVATE
-            ${MICROPY_LIB_BERKELEY_DIR}/include
-        )
+    add_library(micropy_extmod_btree OBJECT
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
+        ${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
+        ${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
+    )
 
-        if(NOT BERKELEY_DB_CONFIG_FILE)
-            set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
-        endif()
+    target_include_directories(micropy_extmod_btree PRIVATE
+        ${MICROPY_LIB_BERKELEY_DIR}/include
+    )
 
-        target_compile_definitions(micropy_extmod_btree PRIVATE
-            BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
-        )
+    if(NOT BERKELEY_DB_CONFIG_FILE)
+        set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
+    endif()
 
-        # The include directories and compile definitions below are needed to build
-        # modbtree.c and should be added to the main MicroPython target.
+    target_compile_definitions(micropy_extmod_btree PRIVATE
+        BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
+    )
 
-        list(APPEND MICROPY_INC_CORE
-            "${MICROPY_LIB_BERKELEY_DIR}/include"
-        )
+    # The include directories and compile definitions below are needed to build
+    # modbtree.c and should be added to the main MicroPython target.
 
-        list(APPEND MICROPY_DEF_CORE
-            MICROPY_PY_BTREE=1
-            BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
-        )
+    list(APPEND MICROPY_INC_CORE
+        "${MICROPY_LIB_BERKELEY_DIR}/include"
+    )
 
-        list(APPEND MICROPY_SOURCE_EXTMOD
-            ${MICROPY_EXTMOD_DIR}/modbtree.c
-        )
-    endif()
+    list(APPEND MICROPY_DEF_CORE
+        MICROPY_PY_BTREE=1
+        BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
+    )
+
+    list(APPEND MICROPY_SOURCE_EXTMOD
+        ${MICROPY_EXTMOD_DIR}/modbtree.c
+    )
 endif()
 
 # Library for mbedtls
@@ -350,5 +345,5 @@ if(MICROPY_PY_LWIP)
         ${MICROPY_LIB_LWIP_DIR}/include
     )
 
-    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/lwip)
+    list(APPEND GIT_SUBMODULES lib/lwip)
 endif()
diff --git a/ports/esp32/Makefile b/ports/esp32/Makefile
index 1ce4d9720..a4e553185 100644
--- a/ports/esp32/Makefile
+++ b/ports/esp32/Makefile
@@ -106,12 +106,10 @@ size-components:
 size-files:
 	$(call RUN_IDF_PY,size-files)
 
-# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
-# print out the value of the GIT_SUBMODULES variable, prefixed with
-# "GIT_SUBMODULES", and then abort. This extracts out that line from the idf.py
-# output and passes the list of submodules to py/mkrules.mk which does the
-# `git submodule init` on each.
+# Run idf.py with the UPDATE_SUBMODULES flag to update
+# necessary submodules for this board.
+#
+# This is done in a dedicated build directory as some CMake cache values are not
+# set correctly if not all submodules are loaded yet.
 submodules:
-	@GIT_SUBMODULES=$$(IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \
-	                  grep '^GIT_SUBMODULES=' | cut -d= -f2); \
-	$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
+	IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D UPDATE_SUBMODULES=1 reconfigure
diff --git a/ports/esp32/esp32_common.cmake b/ports/esp32/esp32_common.cmake
index 6473f04a5..f7b00900a 100644
--- a/ports/esp32/esp32_common.cmake
+++ b/ports/esp32/esp32_common.cmake
@@ -71,8 +71,8 @@ list(APPEND MICROPY_SOURCE_DRIVERS
     ${MICROPY_DIR}/drivers/dht/dht.c
 )
 
-string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
-if(MICROPY_PY_TINYUSB AND NOT ECHO_SUBMODULES)
+list(APPEND GIT_SUBMODULES lib/tinyusb)
+if(MICROPY_PY_TINYUSB)
     set(TINYUSB_SRC "${MICROPY_DIR}/lib/tinyusb/src")
     string(TOUPPER OPT_MCU_${IDF_TARGET} tusb_mcu)
 
@@ -195,6 +195,17 @@ if (MICROPY_USER_LDFRAGMENTS)
     set(MICROPY_LDFRAGMENTS ${MICROPY_USER_LDFRAGMENTS})
 endif()
 
+if (UPDATE_SUBMODULES)
+    # ESP-IDF checks if some paths exist before CMake does. Some paths don't
+    # yet exist if this is an UPDATE_SUBMODULES pass on a brand new checkout, so remove
+    # any path which might not exist yet. A "real" build will not set UPDATE_SUBMODULES.
+    unset(MICROPY_SOURCE_TINYUSB)
+    unset(MICROPY_SOURCE_EXTMOD)
+    unset(MICROPY_SOURCE_LIB)
+    unset(MICROPY_INC_TINYUSB)
+    unset(MICROPY_INC_CORE)
+endif()
+
 # Register the main IDF component.
 idf_component_register(
     SRCS
diff --git a/ports/rp2/CMakeLists.txt b/ports/rp2/CMakeLists.txt
index 7002ad876..f89e2792c 100644
--- a/ports/rp2/CMakeLists.txt
+++ b/ports/rp2/CMakeLists.txt
@@ -76,8 +76,8 @@ if (MICROPY_PY_NETWORK_CYW43)
 endif()
 
 # Necessary submodules for all boards.
-string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mbedtls)
-string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
+list(APPEND GIT_SUBMODULES lib/mbedtls)
+list(APPEND GIT_SUBMODULES lib/tinyusb)
 
 # Include component cmake fragments
 include(${MICROPY_DIR}/py/py.cmake)
@@ -340,7 +340,7 @@ if (MICROPY_PY_BLUETOOTH_CYW43)
 endif()
 
 if (MICROPY_BLUETOOTH_BTSTACK)
-    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/btstack)
+    list(APPEND GIT_SUBMODULES lib/btstack)
 
     list(APPEND MICROPY_SOURCE_PORT mpbtstackport.c)
 
@@ -358,8 +358,8 @@ if (MICROPY_BLUETOOTH_BTSTACK)
 endif()
 
 if(MICROPY_BLUETOOTH_NIMBLE)
-    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mynewt-nimble)
-    if(NOT (${ECHO_SUBMODULES}) AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
+    list(APPEND GIT_SUBMODULES lib/mynewt-nimble)
+    if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
         message(FATAL_ERROR " mynewt-nimble not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
     endif()
 
@@ -386,8 +386,8 @@ target_include_directories(${MICROPY_TARGET} PRIVATE
 )
 
 if (MICROPY_PY_NETWORK_CYW43)
-    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/cyw43-driver)
-    if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
+    list(APPEND GIT_SUBMODULES lib/cyw43-driver)
+    if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
         message(FATAL_ERROR " cyw43-driver not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
     endif()
 
@@ -433,8 +433,8 @@ if (MICROPY_PY_NETWORK_NINAW10)
 endif()
 
 if (MICROPY_PY_NETWORK_WIZNET5K)
-    string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/wiznet5k)
-    if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
+    list(APPEND GIT_SUBMODULES lib/wiznet5k)
+    if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
         message(FATAL_ERROR " wiznet5k not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
     endif()
 
diff --git a/ports/rp2/Makefile b/ports/rp2/Makefile
index 200899d33..bfc85f371 100644
--- a/ports/rp2/Makefile
+++ b/ports/rp2/Makefile
@@ -65,15 +65,11 @@ all:
 clean:
 	$(RM) -rf $(BUILD)
 
-# First ensure that pico-sdk is initialised, then use cmake to pick everything
-# else (including board-specific dependencies).
-# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
-# print out the value of the GIT_SUBMODULES variable, prefixed with
-# "GIT_SUBMODULES", and then abort. This extracts out that line from the cmake
-# output and passes the list of submodules to py/mkrules.mk which does the
-# `git submodule init` on each.
+# First ensure that pico-sdk is initialised, then run CMake with the
+# UPDATE_SUBMODULES flag to update necessary submodules for this board.
+#
+# This is done in a dedicated build directory as some CMake cache values are not
+# set correctly if not all submodules are loaded yet.
 submodules:
 	$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="lib/pico-sdk" submodules
-	@GIT_SUBMODULES=$$(cmake -B $(BUILD)/submodules -DECHO_SUBMODULES=1 ${CMAKE_ARGS} -S . 2>&1 | \
-	                  grep '^GIT_SUBMODULES=' | cut -d= -f2); \
-	$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
+	cmake -S . -B $(BUILD)/submodules -DUPDATE_SUBMODULES=1 ${CMAKE_ARGS}
diff --git a/py/mkrules.cmake b/py/mkrules.cmake
index cafcbce56..4374b8b4d 100644
--- a/py/mkrules.cmake
+++ b/py/mkrules.cmake
@@ -209,16 +209,11 @@ if(MICROPY_FROZEN_MANIFEST)
     # Note: target_compile_definitions already added earlier.
 
     if(NOT MICROPY_LIB_DIR)
-        string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/micropython-lib)
+        list(APPEND GIT_SUBMODULES lib/micropython-lib)
         set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib)
     endif()
 
-    if(ECHO_SUBMODULES)
-        # No-op, we're just doing submodule/variant discovery.
-        # Note: All the following rules are safe to run in discovery mode even
-        # though the submodule might not be available as they do not directly depend
-        # on anything from the submodule.
-    elseif(NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
+    if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
         message(FATAL_ERROR " micropython-lib not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
     endif()
 
@@ -272,12 +267,29 @@ if(MICROPY_FROZEN_MANIFEST)
     )
 endif()
 
-# Update submodules
-if(ECHO_SUBMODULES)
-    # If cmake is run with GIT_SUBMODULES defined on command line, process the port / board
-    # settings then print the final GIT_SUBMODULES variable and exit.
-    # Note: the GIT_SUBMODULES is done via echo rather than message, as message splits
-    # the output onto multiple lines
-    execute_process(COMMAND ${CMAKE_COMMAND} -E echo "GIT_SUBMODULES=${GIT_SUBMODULES}")
-    message(FATAL_ERROR "Done")
+# Update submodules, this is invoked on some ports via 'make submodules'.
+#
+# Note: This logic has a Makefile equivalent in py/mkrules.mk
+if(UPDATE_SUBMODULES AND GIT_SUBMODULES)
+    macro(run_git)
+      execute_process(COMMAND git ${ARGV} WORKING_DIRECTORY ${MICROPY_DIR}
+          RESULT_VARIABLE RES)
+    endmacro()
+
+    list(JOIN GIT_SUBMODULES " " GIT_SUBMODULES_MSG)
+    message("Updating submodules: ${GIT_SUBMODULES_MSG}")
+    run_git(submodule sync ${GIT_SUBMODULES})
+    if(RES EQUAL 0)
+        # If available, do blobless partial clones of submodules to save time and space.
+        # A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
+        run_git(submodule update --init --filter=blob:none ${GIT_SUBMODULES})
+        # Fallback to standard submodule update if blobless isn't available (earlier than git 2.36.0)
+        if (NOT RES EQUAL 0)
+            run_git(submodule update --init ${GIT_SUBMODULES})
+        endif()
+    endif()
+
+    if (NOT RES EQUAL 0)
+        message(FATAL_ERROR "Submodule update failed")
+    endif()
 endif()
diff --git a/py/mkrules.mk b/py/mkrules.mk
index ad855c0d7..495d8d48b 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -262,19 +262,14 @@ endif
 # If available, do blobless partial clones of submodules to save time and space.
 # A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
 # Fallback to standard submodule update if blobless isn't available (earlier than 2.36.0)
+#
+# Note: This target has a CMake equivalent in py/mkrules.cmake
 submodules:
 	$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
 ifneq ($(GIT_SUBMODULES),)
 	$(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES)
 	$(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \
 	  git submodule update --init $(GIT_SUBMODULES)
-else
-ifeq ($(GIT_SUBMODULES_FAIL_IF_EMPTY),1)
-	# If you see this error, it may mean the internal step run by the port's build
-	# system to find git submodules has failed. Double-check dependencies are set correctly.
-	$(ECHO) "Internal build error: The submodule list should not be empty."
-	exit 1
-endif
 endif
 .PHONY: submodules