diff --git a/.github/workflows/refresh.yml b/.github/workflows/refresh.yml
index bb8f2e6be2..bff6b44231 100644
--- a/.github/workflows/refresh.yml
+++ b/.github/workflows/refresh.yml
@@ -27,7 +27,8 @@ jobs:
         git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
         git remote add -f --tags remote https://git.ffmpeg.org/ffmpeg.git
         git fetch --all
-    - name: "Update Branches and Trigger Builds"
+    - name: "Synchronize with Remote and trigger Builds"
+      if: ${{ github.event_name == 'schedule' }}
       shell: bash
       run: |
         declare -a BRANCHES
@@ -85,3 +86,20 @@ jobs:
               "https://api.github.com/repos/${{ github.repository }}/actions/workflows/build.yml/dispatches"
           fi
         done
+    - name: "Only trigger build on manual push"
+      if: ${{ github.event_name == 'push' }}
+      shell: bash
+      run: |
+        declare -a BRANCHES
+        BRANCHES[${#BRANCHES[@]}]="master"
+        BRANCHES[${#BRANCHES[@]}]="release/4.4"
+        BRANCHES[${#BRANCHES[@]}]="release/3.4"
+
+        for d in ${BRANCHES[@]}; do
+          curl \
+            -X POST \
+            -H "Authorization: token ${{ secrets.WORKFLOW_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            -d "{\"ref\":\"${{ github.ref }}\",\"inputs\":{\"ref\":\"${d}\"}}" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/workflows/build.yml/dispatches"
+        done