esp32: Add IDF Component Lockfiles to git repo.

This is recommended by Espressif, and it's the only way to ensure
everyone builds the same set of component versions.

The awkward part is that updating the ESP-IDF version will churn a line
in each of these files (and possibly other changes).

Adds a build-time check for lock file changes, which is either a warning or
a hard error depending on the value of MICROPY_MAINTAINER_BUILD
flag (introduced in previous commit).

This work was funded through GitHub Sponsors.

Signed-off-by: Angus Gratton <angus@redyak.com.au>

11 files changed, 234 insertions, 5 deletions

diff --git a/ports/esp32/.gitignore b/ports/esp32/.gitignore
index a78ecd019..9c708bb85 100644
--- a/ports/esp32/.gitignore
+++ b/ports/esp32/.gitignore
@@ -1,2 +1 @@
-dependencies.lock
 managed_components/
diff --git a/ports/esp32/CMakeLists.txt b/ports/esp32/CMakeLists.txt
index 1db374b40..9d6be8010 100644
--- a/ports/esp32/CMakeLists.txt
+++ b/ports/esp32/CMakeLists.txt
@@ -61,5 +61,21 @@ set(SDKCONFIG_DEFAULTS ${CMAKE_BINARY_DIR}/sdkconfig.combined)
 # Include main IDF cmake file.
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 
+# Generate individual dependencies.lock files based on chip target
+idf_build_set_property(DEPENDENCIES_LOCK lockfiles/dependencies.lock.${IDF_TARGET})
+
 # Define the project.
 project(micropython)
+
+# Check for lockfile changes and either warn or error depending on build type
+message("Checking lockfile contents...")
+execute_process(COMMAND git diff --exit-code lockfiles/ WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+    RESULT_VARIABLE RES)
+if (RES)
+    # Maintainer builds (CI or autobuild runs) should fail if this has happened
+    if($ENV{MICROPY_MAINTAINER_BUILD})
+        message(FATAL_ERROR "Failing build as lockfiles are dirty (see above). Check that ESP-IDF versions match.")
+    else()
+        message(WARNING "Component lockfile contents have changed (see above). This may be due to building with a different ESP-IDF version. Please mention this output if reporting an issue with MicroPython.")
+    endif()
+endif()
diff --git a/ports/esp32/README.md b/ports/esp32/README.md
index dd4584772..2c09ebccd 100644
--- a/ports/esp32/README.md
+++ b/ports/esp32/README.md
@@ -30,8 +30,9 @@ framework, aka SDK).  The ESP-IDF includes the libraries and RTOS needed to
 manage the ESP32 microcontroller, as well as a way to manage the required
 build environment and toolchains needed to build the firmware.
 
-The ESP-IDF changes quickly and MicroPython only supports certain versions.
-Currently MicroPython supports v5.2, v5.2.2, v5.3, v5.4, v5.4.1 and v5.4.2.
+The ESP-IDF changes quickly and MicroPython only supports certain versions. The
+current recommended version of ESP-IDF for MicroPython is v5.4.2. MicroPython
+also supports v5.2, v5.2.2, v5.3, v5.4 and v5.4.1.
 
 To install the ESP-IDF the full instructions can be found at the
 [Espressif Getting Started guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/index.html#installation-step-by-step).
diff --git a/ports/esp32/lockfiles/README.md b/ports/esp32/lockfiles/README.md
new file mode 100644
index 000000000..a957b428f
--- /dev/null
+++ b/ports/esp32/lockfiles/README.md
@@ -0,0 +1,12 @@
+# ESP-IDF Component Lockfiles
+
+This directory contains the exact versions of ESP-IDF components that have been
+used to build MicroPython. It is updated by the [component version
+solver](https://docs.espressif.com/projects/idf-component-manager/en/latest/guides/version_solver.html).
+
+Unless you change the `main/idf_component.yml` file for MicroPython ESP32, files
+in this directory should only change contents if you build using a different
+ESP-IDF version to the last time the file was updated.
+
+*Please do not commit changes to these files and submit PRs unless the PR needs
+to change versions of components or ESP-IDF.*
diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32 b/ports/esp32/lockfiles/dependencies.lock.esp32
new file mode 100644
index 000000000..5c2b78275
--- /dev/null
+++ b/ports/esp32/lockfiles/dependencies.lock.esp32
@@ -0,0 +1,35 @@
+dependencies:
+  espressif/lan867x:
+    component_hash: 0ff9dae3affeff53811e7c8283e67c6d36dc0c03e3bc5102c0fba629e08bf6c4
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.3'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    targets:
+    - esp32
+    - esp32p4
+    version: 1.0.3
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/lan867x
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32
+version: 2.0.0
diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32c2 b/ports/esp32/lockfiles/dependencies.lock.esp32c2
new file mode 100644
index 000000000..df6ed2c1d
--- /dev/null
+++ b/ports/esp32/lockfiles/dependencies.lock.esp32c2
@@ -0,0 +1,21 @@
+dependencies:
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32c2
+version: 2.0.0
diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32c3 b/ports/esp32/lockfiles/dependencies.lock.esp32c3
new file mode 100644
index 000000000..a4c955d47
--- /dev/null
+++ b/ports/esp32/lockfiles/dependencies.lock.esp32c3
@@ -0,0 +1,21 @@
+dependencies:
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32c3
+version: 2.0.0
diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32c6 b/ports/esp32/lockfiles/dependencies.lock.esp32c6
new file mode 100644
index 000000000..fa81d8ad7
--- /dev/null
+++ b/ports/esp32/lockfiles/dependencies.lock.esp32c6
@@ -0,0 +1,21 @@
+dependencies:
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32c6
+version: 2.0.0
diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32s2 b/ports/esp32/lockfiles/dependencies.lock.esp32s2
new file mode 100644
index 000000000..12430428e
--- /dev/null
+++ b/ports/esp32/lockfiles/dependencies.lock.esp32s2
@@ -0,0 +1,50 @@
+dependencies:
+  espressif/esp_tinyusb:
+    component_hash: 96d232ced7afe1976119b62f7fbf1944a2a78b36228ff6f7b9318394ac1153cc
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    - name: espressif/tinyusb
+      registry_url: https://components.espressif.com
+      require: public
+      version: '>=0.14.2'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.7.6~1
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  espressif/tinyusb:
+    component_hash: aa65639878f27a44d349044afd9c3fc134a92bd560874fdac1d836019b5c07ca
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com
+      type: service
+    targets:
+    - esp32s2
+    - esp32s3
+    - esp32p4
+    version: 0.18.0~4
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/esp_tinyusb
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32s2
+version: 2.0.0
diff --git a/ports/esp32/lockfiles/dependencies.lock.esp32s3 b/ports/esp32/lockfiles/dependencies.lock.esp32s3
new file mode 100644
index 000000000..215346dbd
--- /dev/null
+++ b/ports/esp32/lockfiles/dependencies.lock.esp32s3
@@ -0,0 +1,50 @@
+dependencies:
+  espressif/esp_tinyusb:
+    component_hash: 96d232ced7afe1976119b62f7fbf1944a2a78b36228ff6f7b9318394ac1153cc
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    - name: espressif/tinyusb
+      registry_url: https://components.espressif.com
+      require: public
+      version: '>=0.14.2'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.7.6~1
+  espressif/mdns:
+    component_hash: 46ee81d32fbf850462d8af1e83303389602f6a6a9eddd2a55104cb4c063858ed
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com/
+      type: service
+    version: 1.1.0
+  espressif/tinyusb:
+    component_hash: aa65639878f27a44d349044afd9c3fc134a92bd560874fdac1d836019b5c07ca
+    dependencies:
+    - name: idf
+      require: private
+      version: '>=5.0'
+    source:
+      registry_url: https://components.espressif.com
+      type: service
+    targets:
+    - esp32s2
+    - esp32s3
+    - esp32p4
+    version: 0.18.0~4
+  idf:
+    source:
+      type: idf
+    version: 5.4.2
+direct_dependencies:
+- espressif/esp_tinyusb
+- espressif/mdns
+- idf
+manifest_hash: 3b18b5bbac91c9fe5098d3759a37c84ed0828546d8cbc92e26e4c1698e689c8a
+target: esp32s3
+version: 2.0.0
diff --git a/tools/ci.sh b/tools/ci.sh
index 115442ffd..69e595087 100755
--- a/tools/ci.sh
+++ b/tools/ci.sh
@@ -171,14 +171,17 @@ function ci_cc3200_build {
 ########################################################################################
 # ports/esp32
 
-# GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch)
-IDF_VER=v5.4.2
+# GitHub tag of ESP-IDF to use for CI, extracted from the esp32 dependency lockfile
+# This should end up as a tag name like vX.Y.Z
+# (note: This hacky parsing can be replaced with 'yq' once Ubuntu >=24.04 is in use)
+IDF_VER=v$(grep -A10 "idf:" ports/esp32/lockfiles/dependencies.lock.esp32 | grep "version:" | head -n1 | sed -E 's/ +version: //')
 PYTHON=$(command -v python3 2> /dev/null)
 PYTHON_VER=$(${PYTHON:-python} --version | cut -d' ' -f2)
 
 export IDF_CCACHE_ENABLE=1
 
 function ci_esp32_idf_setup {
+    echo "Using ESP-IDF version $IDF_VER"
     git clone --depth 1 --branch $IDF_VER https://github.com/espressif/esp-idf.git
     # doing a treeless clone isn't quite as good as --shallow-submodules, but it
     # is smaller than full clones and works when the submodule commit isn't a head.