[openwrt/openwrt] CI: push-containers: build and push container with external toolchain

[LEDE Commits]

ansuel pushed a commit to openwrt/openwrt.git, branch openwrt-23.05:
https://git.openwrt.org/e2780cbb2feac4a5e5e7f010202a5e6eb92d8db3

commit e2780cbb2feac4a5e5e7f010202a5e6eb92d8db3
Author: Christian Marangi <ansuelsmth at gmail.com>
AuthorDate: Sat Dec 17 00:21:31 2022 +0100

    CI: push-containers: build and push container with external toolchain
    
    Build and push container with external toolchain embedded in the
    container image.
    
    Signed-off-by: Christian Marangi <ansuelsmth at gmail.com>
    (cherry picked from commit e1370cdd497a07612413106d707973155ad3004b)
---
 .github/workflows/Dockerfile.toolchain |   8 ++
 .github/workflows/push-containers.yml  | 156 +++++++++++++++++++++++++++++----
 2 files changed, 146 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/Dockerfile.toolchain b/.github/workflows/Dockerfile.toolchain
new file mode 100644
index 0000000000..949bc2bb5c
--- /dev/null
+++ b/.github/workflows/Dockerfile.toolchain
@@ -0,0 +1,8 @@
+ARG OWNER_LC
+ARG CONTAINER_TAG
+
+FROM ghcr.io/$OWNER_LC/tools:$CONTAINER_TAG
+
+ARG TOOLCHAIN_NAME
+
+ADD $TOOLCHAIN_NAME /external-toolchain/
diff --git a/.github/workflows/push-containers.yml b/.github/workflows/push-containers.yml
index e74e7c2216..3cc08dbef8 100644
--- a/.github/workflows/push-containers.yml
+++ b/.github/workflows/push-containers.yml
@@ -9,6 +9,10 @@ on:
       - '.github/workflows/build-tools.yml'
       - '.github/workflows/push-containers.yml'
       - '.github/workflows/Dockerfile.tools'
+      - 'toolchain/**'
+      - '.github/workflows/build.yml'
+      - '.github/workflows/toolchain.yml'
+      - '.github/workflows/Dockerfile.toolchain'
 
 permissions:
   contents: read
@@ -18,29 +22,21 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-linux-buildbot:
-    name: Build tools with buildbot container
-    if: ${{ github.repository_owner  == 'openwrt' }}
-    uses: ./.github/workflows/build-tools.yml
-    with:
-      generate_prebuilt_artifacts: true
-
-  push-tools-container:
-    needs: build-linux-buildbot
-    name: Push prebuilt tools container
+  determine-container-info:
+    name: Determine needed info to push containers
     if: ${{ github.repository_owner  == 'openwrt' }}
     runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-      packages: write
+    outputs:
+      owner-lc: ${{ steps.generate-owner-lc.outputs.owner-lc }}
+      container-tag: ${{ steps.determine-container-tag.outputs.container-tag }}
 
     steps:
       - name: Set lower case owner name
+        id: generate-owner-lc
         env:
           OWNER: ${{ github.repository_owner }}
         run: |
-          echo "OWNER_LC=${OWNER,,}" >> "$GITHUB_ENV"
+          echo "owner-lc=${OWNER,,}" >> "$GITHUB_OUTPUT"
 
       # Per branch tools container tag
       # By default stick to latest
@@ -51,6 +47,7 @@ jobs:
       # (example branch openwrt-22.03 -> tools:openwrt-22.03)
       # (example branch openwrt-22.03-test -> tools:openwrt-22.03)
       - name: Determine tools container tag
+        id: determine-container-tag
         run: |
           CONTAINER_TAG=latest
 
@@ -64,9 +61,27 @@ jobs:
             fi
           fi
 
-          echo "Tools container to push tools:$CONTAINER_TAG"
-          echo "CONTAINER_TAG=$CONTAINER_TAG" >> "$GITHUB_ENV"
+          echo "Container tag to push for tools and toolchain is $CONTAINER_TAG"
+          echo "container-tag=$CONTAINER_TAG" >> "$GITHUB_OUTPUT"
+
+  build-linux-buildbot:
+    name: Build tools with buildbot container
+    if: ${{ github.repository_owner  == 'openwrt' }}
+    uses: ./.github/workflows/build-tools.yml
+    with:
+      generate_prebuilt_artifacts: true
 
+  push-tools-container:
+    needs: [ determine-container-info, build-linux-buildbot ]
+    if: ${{ github.repository_owner  == 'openwrt' }}
+    name: Push prebuilt tools container
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
       - name: Checkout
         uses: actions/checkout at v3
         with:
@@ -94,5 +109,110 @@ jobs:
         with:
           context: openwrt
           push: true
-          tags: ghcr.io/${{ env.OWNER_LC }}/tools:${{ env.CONTAINER_TAG }}
+          tags: ghcr.io/${{ needs.determine-container-info.outputs.owner-lc }}/tools:${{ needs.determine-container-info.outputs.container-tag }}
           file: openwrt/.github/workflows/Dockerfile.tools
+
+  determine-targets:
+    name: Set targets
+    if: ${{ github.repository_owner  == 'openwrt' }}
+    runs-on: ubuntu-latest
+    outputs:
+      target: ${{ steps.find_targets.outputs.target }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout at v3
+
+      - name: Set targets
+        id: find_targets
+        run: |
+          export TARGETS="$(perl ./scripts/dump-target-info.pl targets 2>/dev/null \
+            | awk '{ print $1 }')"
+
+          JSON='['
+          FIRST=1
+          for TARGET in $TARGETS; do
+            TUPLE='{"target":"'"$(echo $TARGET | cut -d "/" -f 1)"'","subtarget":"'"$(echo $TARGET | cut -d "/" -f 2)"'"}'
+            [[ $FIRST -ne 1 ]] && JSON="$JSON"','
+            JSON="$JSON""$TUPLE"
+            FIRST=0
+          done
+          JSON="$JSON"']'
+
+          echo -e "\n---- targets ----\n"
+          echo "$JSON"
+          echo -e "\n---- targets ----\n"
+
+          echo "target=$JSON" >> $GITHUB_OUTPUT
+
+  build:
+    name: Build Target Toolchain
+    if: ${{ github.repository_owner  == 'openwrt' }}
+    needs: [ determine-targets, push-tools-container ]
+    permissions:
+      contents: read
+      packages: read
+    strategy:
+      fail-fast: False
+      matrix:
+        include: ${{fromJson(needs.determine-targets.outputs.target)}}
+    uses: ./.github/workflows/build.yml
+    with:
+      target: ${{ matrix.target }}
+      subtarget: ${{ matrix.subtarget }}
+      build_toolchain: true
+      build_external_toolchain: true
+      upload_external_toolchain: true
+
+  push-toolchain-container:
+    name: Push Target Toolchain container
+    if: ${{ github.repository_owner  == 'openwrt' }}
+    needs: [ determine-container-info, determine-targets, build ]
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: False
+      matrix:
+        include: ${{fromJson(needs.determine-targets.outputs.target)}}
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout at v3
+        with:
+          path: 'openwrt'
+
+      - name: Download external toolchain from build job
+        uses: actions/download-artifact at v3
+        with:
+          name: ${{ matrix.target }}-${{ matrix.subtarget }}-external-toolchain
+          path: openwrt
+
+      - name: Find external toolchain name
+        id: get-toolchain-name
+        working-directory: openwrt
+        run: |
+          TOOLCHAIN_NAME=$(ls | grep toolchain-${{ matrix.target }}-${{ matrix.subtarget }})
+          echo "toolchain-name=$TOOLCHAIN_NAME" >> $GITHUB_OUTPUT
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action at v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action at v3
+        with:
+          context: openwrt
+          push: true
+          tags: ghcr.io/${{ needs.determine-container-info.outputs.owner-lc }}/toolchain:${{ matrix.target }}-${{ matrix.subtarget }}-${{ needs.determine-container-info.outputs.container-tag }}
+          file: openwrt/.github/workflows/Dockerfile.toolchain
+          build-args: |
+            OWNER_LC=${{ needs.determine-container-info.outputs.owner-lc }}
+            CONTAINER_TAG=${{ needs.determine-container-info.outputs.container-tag }}
+            TOOLCHAIN_NAME=${{ steps.get-toolchain-name.outputs.toolchain-name }}