[openwrt/openwrt] CI: add support for getting ccache cache from S3
LEDE Commits
lede-commits at lists.infradead.org
Tue Oct 24 08:14:44 PDT 2023
ansuel pushed a commit to openwrt/openwrt.git, branch openwrt-23.05:
https://git.openwrt.org/f7e4f8cbbf8c5ae8602a77112cb2aacb067fce05
commit f7e4f8cbbf8c5ae8602a77112cb2aacb067fce05
Author: Christian Marangi <ansuelsmth at gmail.com>
AuthorDate: Tue May 30 18:57:42 2023 +0200
CI: add support for getting ccache cache from S3
Add support for getting ccache cache from S3.
ccache is archieved in a tar and downloaded from S3 Cloud Storage.
For push events, ccache is then uplodaed back to S3 to refresh and have
a ccache cache always fresh.
An additional workflow is added to upload files to an S3 Cloud Storage
from artifacts uplodaed to github. The minio tool is used to upload
files to S3.
If the ccache can't be downloaded from s3, we fallback to github cache
system.
Also limit s3 upload to the openwrt repository since external fork won't
have (obviously) the required secrtes to upload data to the S3 Cloud
Storage.
Signed-off-by: Christian Marangi <ansuelsmth at gmail.com>
(cherry picked from commit ebbc806d30502ff003ae7a19098c6afaaf1295a5)
---
.github/workflows/build.yml | 40 ++++++++++++++++++++++++++++---
.github/workflows/kernel.yml | 18 ++++++++++++++
.github/workflows/packages.yml | 23 ++++++++++++++++++
.github/workflows/upload-file-s3.yml | 46 ++++++++++++++++++++++++++++++++++++
4 files changed, 124 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 95d19f4c4e..efaf759403 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -60,6 +60,8 @@ on:
ccache_type:
type: string
default: kernel
+ upload_ccache_cache:
+ type: boolean
permissions:
contents: read
@@ -232,6 +234,20 @@ jobs:
echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
echo "TOOLCHAIN_PATH=$TOOLCHAIN_PATH" >> "$GITHUB_ENV"
+ - name: Download and extract ccache cache from s3
+ id: restore-ccache-cache-s3
+ if: inputs.use_ccache_cache == true
+ working-directory: openwrt
+ run: |
+ ENDPOINT=https://storage.googleapis.com
+ BUCKET=openwrt-ci-cache
+ CCACHE_TAR=ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar
+
+ if curl -o /dev/null -s --head --fail $ENDPOINT/$BUCKET/$CCACHE_TAR; then
+ wget -O - $ENDPOINT/$BUCKET/$CCACHE_TAR | tar -xf -
+ echo "cache-hit=true" >> $GITHUB_OUTPUT
+ fi
+
- name: Fix permission
run: |
chown -R buildbot:buildbot openwrt
@@ -256,7 +272,7 @@ jobs:
- name: Restore ccache cache
id: restore-ccache-cache
- if: inputs.use_ccache_cache == true
+ if: inputs.use_ccache_cache == true && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
uses: actions/cache/restore at v3
with:
path: openwrt/.ccache
@@ -498,7 +514,8 @@ jobs:
path: "openwrt/logs"
- name: Delete already present ccache cache
- if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true && github.event_name == 'push'
+ if: steps.restore-ccache-cache.outputs.cache-hit == 'true' && inputs.use_ccache_cache == true &&
+ github.event_name == 'push' && steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
uses: octokit/request-action at v2.x
with:
route: DELETE /repos/{repository}/actions/caches?key={key}
@@ -508,12 +525,29 @@ jobs:
INPUT_KEY: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
- name: Save ccache cache
- if: inputs.use_ccache_cache == true && github.event_name == 'push'
+ if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
+ steps.restore-ccache-cache-s3.outputs.cache-hit != 'true'
uses: actions/cache/save at v3
with:
path: openwrt/.ccache
key: ${{ steps.restore-ccache-cache.outputs.cache-primary-key }}
+ - name: Archive ccache
+ if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
+ inputs.upload_ccache_cache == true
+ shell: su buildbot -c "sh -e {0}"
+ working-directory: openwrt
+ run: tar -cf ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar .ccache
+
+ - name: Upload ccache cache
+ if: inputs.use_ccache_cache == true && github.event_name == 'push' &&
+ inputs.upload_ccache_cache == true
+ uses: actions/upload-artifact at v3
+ with:
+ name: ${{ inputs.target }}-${{ inputs.subtarget }}-ccache-cache
+ path: openwrt/ccache-${{ inputs.ccache_type }}-${{ inputs.target }}-${{ inputs.subtarget }}.tar
+ retention-days: 1
+
- name: Find external toolchain name
id: get-toolchain-name
if: inputs.upload_external_toolchain == true
diff --git a/.github/workflows/kernel.yml b/.github/workflows/kernel.yml
index d9ab8581c9..5f18a39c55 100644
--- a/.github/workflows/kernel.yml
+++ b/.github/workflows/kernel.yml
@@ -117,6 +117,7 @@ jobs:
subtarget: ${{ matrix.subtarget }}
build_kernel: true
build_all_kmods: true
+ upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }}
check-kernel-patches:
name: Check Kernel patches
@@ -133,3 +134,20 @@ jobs:
target: ${{ matrix.target }}
subtarget: ${{ matrix.subtarget }}
+ upload-ccache-cache-in-s3:
+ if: github.event_name == 'push' && github.repository_owner == 'openwrt'
+ name: Upload ccache cache to s3
+ needs: [determine_targets, build]
+ strategy:
+ fail-fast: False
+ matrix:
+ include: ${{fromJson(needs.determine_targets.outputs.targets_subtargets)}}
+ secrets:
+ s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }}
+ s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }}
+ uses: ./.github/workflows/upload-file-s3.yml
+ with:
+ endpoint: https://storage.googleapis.com
+ bucket: openwrt-ci-cache
+ download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache
+ filename: ccache-kernel-${{ matrix.target }}-${{ matrix.subtarget }}.tar
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml
index 58bbcd0131..fe2898cacd 100644
--- a/.github/workflows/packages.yml
+++ b/.github/workflows/packages.yml
@@ -54,4 +54,27 @@ jobs:
build_all_modules: true
build_full: true
ccache_type: packages
+ upload_ccache_cache: ${{ github.repository_owner == 'openwrt' }}
+
+ upload-ccache-cache-in-s3:
+ if: github.event_name == 'push' && github.repository_owner == 'openwrt'
+ name: Upload ccache cache to s3
+ needs: build
+ strategy:
+ fail-fast: False
+ matrix:
+ include:
+ - target: malta
+ subtarget: be
+ - target: x86
+ subtarget: 64
+ secrets:
+ s3_access_key: ${{ secrets.GCS_S3_ACCESS_KEY }}
+ s3_secret_key: ${{ secrets.GCS_S3_SECRET_KEY }}
+ uses: ./.github/workflows/upload-file-s3.yml
+ with:
+ endpoint: https://storage.googleapis.com
+ bucket: openwrt-ci-cache
+ download_id: ${{ matrix.target }}-${{ matrix.subtarget }}-ccache-cache
+ filename: ccache-packages-${{ matrix.target }}-${{ matrix.subtarget }}.tar
diff --git a/.github/workflows/upload-file-s3.yml b/.github/workflows/upload-file-s3.yml
new file mode 100644
index 0000000000..6bcb172468
--- /dev/null
+++ b/.github/workflows/upload-file-s3.yml
@@ -0,0 +1,46 @@
+name: Upload File to S3
+
+on:
+ workflow_call:
+ secrets:
+ s3_access_key:
+ s3_secret_key:
+ inputs:
+ endpoint:
+ required: true
+ type: string
+ bucket:
+ required: true
+ type: string
+ download_id:
+ required: true
+ type: string
+ filename:
+ required: true
+ type: string
+
+jobs:
+ upload-file-in-s3:
+ name: Upload file in S3
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Install minio
+ run: |
+ curl https://dl.min.io/client/mc/release/linux-amd64/mc \
+ --create-dirs \
+ -o $GITHUB_WORKSPACE/minio-binaries/mc
+
+ chmod +x $GITHUB_WORKSPACE/minio-binaries/mc
+ echo $GITHUB_WORKSPACE/minio-binaries/ >> $GITHUB_PATH
+
+ - name: Setup minio
+ run: mc alias set s3 ${{ inputs.endpoint }} ${{ secrets.s3_access_key }} ${{ secrets.s3_secret_key }}
+
+ - name: Download file
+ uses: actions/download-artifact at v3
+ with:
+ name: ${{ inputs.download_id }}
+
+ - name: Upload file to s3
+ run: mc cp ${{ inputs.filename }} s3/${{ inputs.bucket }}/
More information about the lede-commits
mailing list