hacktricks-cloud/.github/workflows/build_master.yml

name: Build Master

on:
  push:
    branches:
      - master
    paths-ignore:
      - 'scripts/**'
      - '.gitignore'
      - '.github/**'
      - 'book/**'
  workflow_dispatch:

concurrency: build_master

permissions:
  packages: write
  id-token: write
  contents: write

jobs:
  run-translation:
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/hacktricks-wiki/hacktricks-cloud/translator-image:latest
    environment: prod

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 #Needed to download everything to be able to access the master & language branches
      
      # Build the mdBook
      - name: Build mdBook
        run: MDBOOK_BOOK__LANGUAGE=en mdbook build || (echo "Error logs" && cat hacktricks-preprocessor-error.log && echo "" && echo "" && echo "Debug logs" && (cat hacktricks-preprocessor.log | tail -n 20) && exit 1)
      
      - name: Push search index to hacktricks-searchindex repo
        shell: bash
        env:
          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
        run: |
          set -euo pipefail

          ASSET="book/searchindex.js"
          TARGET_REPO="HackTricks-wiki/hacktricks-searchindex"
          FILENAME="searchindex-cloud-en.js"

          if [ ! -f "$ASSET" ]; then
            echo "Expected $ASSET to exist after build" >&2
            exit 1
          fi

          TOKEN="${PAT_TOKEN}"
          if [ -z "$TOKEN" ]; then
            echo "No PAT_TOKEN available" >&2
            exit 1
          fi

          # Clone the searchindex repo
          git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
          
          # Copy and compress the searchindex file
          cp "$ASSET" "/tmp/searchindex-repo/${FILENAME}"
          gzip -9 -k -f "$ASSET"
          cp "${ASSET}.gz" "/tmp/searchindex-repo/${FILENAME}.gz"
          
          # Show compression stats
          ORIGINAL_SIZE=$(wc -c < "$ASSET")
          COMPRESSED_SIZE=$(wc -c < "${ASSET}.gz")
          RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
          echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
          
          # Commit and push with retry logic
          cd /tmp/searchindex-repo
          git config user.name "GitHub Actions"
          git config user.email "github-actions@github.com"
          git add "${FILENAME}" "${FILENAME}.gz"
          
          if git diff --staged --quiet; then
            echo "No changes to commit"
          else
            git commit -m "Update ${FILENAME} from hacktricks-cloud build"
            
            # Retry push up to 20 times with pull --rebase between attempts
            MAX_RETRIES=20
            RETRY_COUNT=0
            while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
              if git push origin main; then
                echo "Successfully pushed on attempt $((RETRY_COUNT + 1))"
                break
              else
                RETRY_COUNT=$((RETRY_COUNT + 1))
                if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
                  echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
                  git pull --rebase origin main
                  sleep $((RETRY_COUNT * 2))  # Exponential backoff
                else
                  echo "Failed to push after $MAX_RETRIES attempts"
                  exit 1
                fi
              fi
            done
          fi

      # Login in AWs
      - name: Configure AWS credentials using OIDC
        uses: aws-actions/configure-aws-credentials@v3
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1
      
      # Sync the build to S3
      - name: Sync to S3
        run: aws s3 sync ./book s3://hacktricks-cloud/en --delete