build: handle out of disk space on source cache (#44491)

* build: handle out of disk space on source cache

Co-authored-by: John Kleinschmidt <[email protected]>

* build: add cron job to free up source cache disk space

Co-authored-by: John Kleinschmidt <[email protected]>

---------

Co-authored-by: trop[bot] <37223003+trop[bot]@users.noreply.github.com>
Co-authored-by: John Kleinschmidt <[email protected]>

.github/actions/checkout/action.yml

@@ -57,13 +57,26 @@ runs:
       cache_path=/mnt/cross-instance-cache/$DEPSHASH.tar
       echo "Using cache key: $DEPSHASH"
       echo "Checking for cache in: $cache_path"
-      if [ ! -f "$cache_path" ]; then
+      if [ ! -f "$cache_path" ] || [ `du $cache_path | cut -f1` = "0" ]; then
         echo "cache_exists=false" >> $GITHUB_OUTPUT
         echo "Cache Does Not Exist for $DEPSHASH"
       else
         echo "cache_exists=false" >> $GITHUB_OUTPUT
         echo "Cache Already Exists for $DEPSHASH, Busting cache.."
       fi
+  - name: Check cross instance cache disk space
+    if: steps.check-cache.outputs.cache_exists == 'false'
+    shell: bash
+    run: |    
+      # if there is less than 20 GB free space then creating the cache might fail so exit early
+      freespace=`df -m /mnt/cross-instance-cache | grep -w /mnt/cross-instance-cache | awk '{print $4}'`
+      freespace_human=`df -h /mnt/cross-instance-cache | grep -w /mnt/cross-instance-cache | awk '{print $4}'`
+      if [ $freespace -le 20000 ]; then
+        echo "The cross mount cache has $freespace_human free space which is not enough - exiting"
+        exit 1
+      else
+        echo "The cross mount cache has $freespace_human free space - continuing"
+      fi
   - name: Gclient Sync
     if: steps.check-cache.outputs.cache_exists == 'false'
     shell: bash

.github/actions/restore-cache-aks/action.yml

@@ -17,6 +17,11 @@ runs:
       fi
 
       echo "Persisted cache is $(du -sh $cache_path | cut -f1)"
+      if [ `du  $cache_path | cut -f1` = "0" ]; then
+        echo "Cache is empty - exiting"
+        exit 1
+      fi
+
       mkdir temp-cache
       tar -xf $cache_path -C temp-cache
       echo "Unzipped cache is $(du -sh temp-cache/src | cut -f1)"

.github/actions/restore-cache-azcopy/action.yml

@@ -44,6 +44,11 @@ runs:
     shell: bash
     run: |
       echo "Downloaded cache is $(du -sh $DEPSHASH.tar | cut -f1)"
+      if [ `du $DEPSHASH.tar | cut -f1` = "0" ]; then
+        echo "Cache is empty - exiting"
+        exit 1
+      fi
+      
       mkdir temp-cache
       tar -xf $DEPSHASH.tar -C temp-cache
       echo "Unzipped cache is $(du -sh temp-cache/src | cut -f1)"

.github/workflows/clean-src-cache.yml

@@ -0,0 +1,21 @@
+name: Clean Source Cache
+
+on:
+  schedule:
+    - cron: "0 0 * * SUN"  # Run at midnight every Sunday
+
+jobs:
+  clean-src-cache:
+    runs-on: electron-arc-linux-amd64-32core
+    container:
+      image: ghcr.io/electron/build:bc2f48b2415a670de18d13605b1cf0eb5fdbaae1
+      options: --user root
+      volumes:
+        - /mnt/cross-instance-cache:/mnt/cross-instance-cache
+    steps:
+    - name: Cleanup Source Cache
+      shell: bash
+      run: |
+        df -h /mnt/cross-instance-cache
+        find /mnt/cross-instance-cache -type f -mtime +30 -delete
+        df -h /mnt/cross-instance-cache