From 02858b73935cf9eee4370b92b4a762d36e764b0e Mon Sep 17 00:00:00 2001 From: torlando-tech Date: Sat, 8 Nov 2025 19:08:35 -0500 Subject: [PATCH] feat(ci): Refactor deployment to use matrix strategy with per-Pi nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completely refactored the deployment workflow to create separate GitHub Actions nodes for each Pi, with independent deploy and validation steps. This provides much better visibility and control. New Architecture: 1. **setup** job: Parses PI_HOSTS into JSON matrix 2. **deploy** job: Matrix execution (one instance per Pi) 3. **validate** job: Matrix execution (one instance per Pi) 4. **summary** job: Aggregate results GitHub Actions Graph View (2 Pis): ``` setup ━┳━> deploy-pi-0 ━> validate-pi-0 ┗━> deploy-pi-1 ━> validate-pi-1 ``` Features: - **Parallel execution**: All Pis deploy simultaneously - **Independent nodes**: Each Pi has its own deploy + validate node - **fail-fast: false**: One Pi failure doesn't block others - **Per-Pi logs**: Clean, isolated logs for each device - **Comprehensive validation**: * Wait 5s for startup * Check rnsd process * Verify BLE interface online (retry 3x with 3s delay) * Check Bluetooth adapter powered * Display adapter MAC address - **Better error reporting**: Shows which specific Pi failed - **Granular status**: See each Pi's status independently Validation Checks: ✓ rnsd process running ✓ Log file exists ✓ No critical errors in logs ✓ "interface online" message found ✓ Bluetooth adapter powered ✓ Retry logic for startup delays Benefits: - Easier to identify which Pi has issues - Can re-run individual Pi jobs - Faster deployment (parallel vs sequential) - Clearer progression in GitHub UI - Each Pi's logs are isolated and clean Example UI with failure: ``` setup ✓ ├─ deploy-pi-0 ✓ │ └─ validate-pi-0 ✗ (BLE failed to start) └─ deploy-pi-1 ✓ └─ validate-pi-1 ✓ (BLE online) ``` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/deploy.yml | 369 ++++++++++++++++++++++++----------- 1 file changed, 252 insertions(+), 117 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 11b0f45..0fb8c5c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,92 +8,118 @@ on: workflow_dispatch: jobs: - deploy: - name: Deploy to Raspberry Pis - runs-on: self-hosted + # ============================================================================ + # JOB 1: Parse PI_HOSTS into matrix for parallel deployment + # ============================================================================ + setup: + name: Setup Deployment Matrix + runs-on: ubuntu-latest # Only run if tests passed (for workflow_run) or if manually triggered if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + branch: ${{ steps.get-branch.outputs.branch }} steps: - - name: Validate required secrets - run: | - if [ -z "${{ secrets.PI_HOSTS }}" ]; then - echo "Error: PI_HOSTS secret is not set" - echo "Please set PI_HOSTS secret with comma-separated hostnames (e.g., 'pi1.local,pi2.local')" - exit 1 - fi - if [ -z "${{ secrets.PI_REPO_PATH }}" ]; then - echo "Error: PI_REPO_PATH secret is not set" - echo "Please set PI_REPO_PATH secret with repository path (e.g., '/home/pi/ble-reticulum')" - exit 1 - fi - if [ -z "${{ secrets.PI_USER }}" ]; then - echo "Error: PI_USER secret is not set" - echo "Please set PI_USER secret with SSH username (e.g., 'pi')" - exit 1 - fi - if [ -z "${{ secrets.PI_SSH_KEY }}" ]; then - echo "Error: PI_SSH_KEY secret is not set" - echo "Please set PI_SSH_KEY secret with SSH private key for Pi access" - exit 1 - fi - echo "All required secrets are configured" + - name: Validate required secrets + run: | + if [ -z "${{ secrets.PI_HOSTS }}" ]; then + echo "Error: PI_HOSTS secret is not set" + echo "Please set PI_HOSTS secret with comma-separated hostnames (e.g., 'pi1.local,pi2.local')" + exit 1 + fi + if [ -z "${{ secrets.PI_REPO_PATH }}" ]; then + echo "Error: PI_REPO_PATH secret is not set" + echo "Please set PI_REPO_PATH secret with repository path (e.g., '/home/pi/ble-reticulum')" + exit 1 + fi + if [ -z "${{ secrets.PI_USER }}" ]; then + echo "Error: PI_USER secret is not set" + echo "Please set PI_USER secret with SSH username (e.g., 'pi')" + exit 1 + fi + if [ -z "${{ secrets.PI_SSH_KEY }}" ]; then + echo "Error: PI_SSH_KEY secret is not set" + echo "Please set PI_SSH_KEY secret with SSH private key for Pi access" + exit 1 + fi + echo "✓ All required secrets are configured" - - name: Setup SSH key - env: - PI_SSH_KEY: ${{ secrets.PI_SSH_KEY }} - run: | - # Create .ssh directory if it doesn't exist - mkdir -p ~/.ssh - chmod 700 ~/.ssh + - name: Get branch name + id: get-branch + run: | + BRANCH="${{ github.event.workflow_run.head_branch || github.ref_name }}" + echo "branch=$BRANCH" >> $GITHUB_OUTPUT + echo "Deployment branch: $BRANCH" - # Write SSH private key to file - echo "$PI_SSH_KEY" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 + - name: Parse PI_HOSTS into deployment matrix + id: set-matrix + env: + PI_HOSTS: ${{ secrets.PI_HOSTS }} + run: | + # Split comma-separated PI_HOSTS into array + IFS=',' read -ra HOSTS <<< "$PI_HOSTS" - # Disable strict host key checking for known local hosts - cat >> ~/.ssh/config <> $GITHUB_OUTPUT + echo "Deployment matrix created for ${#HOSTS[@]} Pi(s)" + echo "$JSON" | jq '.' - - name: Deploy to Raspberry Pis - env: - PI_HOSTS: ${{ secrets.PI_HOSTS }} - PI_REPO_PATH: ${{ secrets.PI_REPO_PATH }} - PI_USER: ${{ secrets.PI_USER }} - BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.ref_name }} - run: | - # Split comma-separated PI_HOSTS into array - IFS=',' read -ra HOSTS <<< "$PI_HOSTS" + # ============================================================================ + # JOB 2: Deploy to each Pi (parallel matrix execution) + # ============================================================================ + deploy: + name: Deploy to Pi ${{ matrix.pi.index }} (${{ matrix.pi.host }}) + runs-on: self-hosted + needs: setup + strategy: + matrix: + pi: ${{ fromJson(needs.setup.outputs.matrix) }} + fail-fast: false # Continue deploying to other Pis if one fails - echo "===================================" - echo "Deployment Configuration" - echo "===================================" - echo "Branch: $BRANCH_NAME" - echo "Target Pis: ${#HOSTS[@]}" - echo "Repository Path: $PI_REPO_PATH" - echo "User: $PI_USER" - echo "===================================" - echo "" + steps: + - name: Setup SSH key + env: + PI_SSH_KEY: ${{ secrets.PI_SSH_KEY }} + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "$PI_SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 - # Track deployment status - FAILED_HOSTS=() - SUCCESSFUL_HOSTS=() + cat >> ~/.ssh/config <>> Deploying to $HOST..." - - # Create deployment script + # Deployment script DEPLOY_SCRIPT="set -e echo ' [1/7] Navigating to repository...' cd '$PI_REPO_PATH' || exit 1 @@ -133,56 +159,165 @@ jobs: echo ' ✓ Deployment successful!'" - # Deploy with error handling - if echo "$DEPLOY_SCRIPT" | ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$PI_USER@$HOST" bash; then - echo "✓ Successfully deployed to $HOST" - SUCCESSFUL_HOSTS+=("$HOST") + # Execute deployment via SSH + if echo "$DEPLOY_SCRIPT" | ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$PI_USER@$PI_HOST" bash; then + echo "" + echo "✓ Successfully deployed to $PI_HOST" else - echo "✗ Failed to deploy to $HOST" - FAILED_HOSTS+=("$HOST") + echo "" + echo "✗ Failed to deploy to $PI_HOST" + exit 1 + fi + + - name: Cleanup SSH key + if: always() + run: rm -f ~/.ssh/id_ed25519 + + # ============================================================================ + # JOB 3: Validate BLE interface on each Pi (parallel matrix execution) + # ============================================================================ + validate: + name: Validate Pi ${{ matrix.pi.index }} (${{ matrix.pi.host }}) + runs-on: self-hosted + needs: [setup, deploy] + strategy: + matrix: + pi: ${{ fromJson(needs.setup.outputs.matrix) }} + fail-fast: false + + steps: + - name: Setup SSH key + env: + PI_SSH_KEY: ${{ secrets.PI_SSH_KEY }} + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "$PI_SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + + - name: Validate BLE interface on ${{ matrix.pi.host }} + env: + PI_HOST: ${{ matrix.pi.host }} + PI_USER: ${{ secrets.PI_USER }} + run: | + echo "===================================" + echo "Validating Pi ${{ matrix.pi.index }}" + echo "===================================" + echo "Host: $PI_HOST" + echo "===================================" + echo "" + + # Validation script + VALIDATION_SCRIPT='set -e + + echo " [1/4] Waiting for startup (5s)..." + sleep 5 + + echo " [2/4] Checking rnsd process..." + if ! pgrep -x rnsd > /dev/null; then + echo " ✗ rnsd process not running" + exit 1 + fi + echo " ✓ rnsd is running (PID: $(pgrep -x rnsd))" + + echo " [3/4] Checking BLE interface logs..." + LOG_FILE="$HOME/.reticulum/logfile" + + if [ ! -f "$LOG_FILE" ]; then + echo " ✗ Log file not found at $LOG_FILE" + exit 1 + fi + + # Retry 3 times with 3s delay + SUCCESS=false + for attempt in 1 2 3; do + RECENT_LOGS=$(tail -100 "$LOG_FILE" 2>/dev/null || echo "") + + # Check for critical errors + if echo "$RECENT_LOGS" | grep -qE "(failed to start driver|Timeout waiting for Transport)"; then + echo " ✗ BLE driver/identity error detected" + echo "" + echo " Recent error logs:" + tail -30 "$LOG_FILE" | grep -E "(BLE|ERROR)" + exit 1 + fi + + # Check for success + if echo "$RECENT_LOGS" | grep -q "interface online"; then + echo " ✓ BLE interface online" + SUCCESS=true + break + fi + + if [ $attempt -lt 3 ]; then + echo " Retry $attempt/3 (waiting 3s)..." + sleep 3 + fi + done + + if [ "$SUCCESS" = false ]; then + echo " ✗ Interface did not come online after 3 attempts" + echo "" + echo " Recent logs:" + tail -30 "$LOG_FILE" | grep -E "(BLE|ERROR|WARNING)" + exit 1 + fi + + echo " [4/4] Checking Bluetooth adapter..." + if bluetoothctl show 2>/dev/null | grep -q "Powered: yes"; then + ADAPTER_MAC=$(bluetoothctl show 2>/dev/null | grep "Address:" | awk "{print \$2}") + echo " ✓ Bluetooth adapter powered ($ADAPTER_MAC)" + else + echo " ⚠ Bluetooth adapter status unknown" fi echo "" - done + echo " ✓ Validation successful!" + ' - # Print summary - echo "===================================" - echo "Deployment Summary" - echo "===================================" - echo "Successful: ${#SUCCESSFUL_HOSTS[@]}/${#HOSTS[@]}" - if [ ${#SUCCESSFUL_HOSTS[@]} -gt 0 ]; then - printf ' ✓ %s\n' "${SUCCESSFUL_HOSTS[@]}" - fi + # Execute validation via SSH + if echo "$VALIDATION_SCRIPT" | ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$PI_USER@$PI_HOST" bash; then + echo "" + echo "✓ $PI_HOST validation passed" + else + echo "" + echo "✗ $PI_HOST validation failed" + exit 1 + fi - if [ ${#FAILED_HOSTS[@]} -gt 0 ]; then - echo "" - echo "Failed: ${#FAILED_HOSTS[@]}/${#HOSTS[@]}" - printf ' ✗ %s\n' "${FAILED_HOSTS[@]}" - echo "" - echo "===================================" - exit 1 - fi + - name: Cleanup SSH key + if: always() + run: rm -f ~/.ssh/id_ed25519 - echo "===================================" + # ============================================================================ + # JOB 4: Summary (runs after all deploy + validate jobs complete) + # ============================================================================ + summary: + name: Deployment Summary + runs-on: ubuntu-latest + needs: [setup, deploy, validate] + if: always() - - name: Cleanup SSH key - if: always() - run: | - # Remove SSH key for security - rm -f ~/.ssh/id_ed25519 - echo "SSH key cleaned up" + steps: + - name: Generate summary + run: | + echo "## 🎉 Deployment Complete" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Branch:** ${{ needs.setup.outputs.branch }}" >> $GITHUB_STEP_SUMMARY + echo "**Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY - - name: Deployment status - if: always() - run: | - echo "## Deployment Results" >> $GITHUB_STEP_SUMMARY - echo "- **Branch:** ${{ github.event.workflow_run.head_branch }}" >> $GITHUB_STEP_SUMMARY - echo "- **Commit:** ${{ github.event.workflow_run.head_sha }}" >> $GITHUB_STEP_SUMMARY - echo "- **Triggered by:** ${{ github.event.workflow_run.actor.login }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - if [ "${{ job.status }}" == "success" ]; then - echo "✓ All Raspberry Pis deployed successfully" >> $GITHUB_STEP_SUMMARY - else - echo "✗ Deployment failed on one or more Raspberry Pis" >> $GITHUB_STEP_SUMMARY - echo "Check the job logs for details" >> $GITHUB_STEP_SUMMARY - fi + if [ "${{ needs.deploy.result }}" == "success" ] && [ "${{ needs.validate.result }}" == "success" ]; then + echo "### ✅ All Pis Deployed and Validated Successfully" >> $GITHUB_STEP_SUMMARY + else + echo "### ⚠️ Some Pis Failed" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [ "${{ needs.deploy.result }}" != "success" ]; then + echo "- **Deploy:** ${{ needs.deploy.result }}" >> $GITHUB_STEP_SUMMARY + fi + if [ "${{ needs.validate.result }}" != "success" ]; then + echo "- **Validate:** ${{ needs.validate.result }}" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "Check individual job logs for details." >> $GITHUB_STEP_SUMMARY + fi