f

.github/workflows/build_master.yml

@@ -60,10 +60,20 @@ jobs:
           # Clone the searchindex repo
           git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git /tmp/searchindex-repo
           
+          cd /tmp/searchindex-repo
+          git config user.name "GitHub Actions"
+          git config user.email "github-actions@github.com"
+          
+          # Create a fresh orphan branch (no history)
+          git checkout --orphan new-main
+          
+          # Remove all existing files from git index
+          git rm -rf . 2>/dev/null || true
+          
           # Copy and compress the searchindex file
-          cp "$ASSET" "/tmp/searchindex-repo/${FILENAME}"
+          cp "$ASSET" "${FILENAME}"
           gzip -9 -k -f "$ASSET"
-          cp "${ASSET}.gz" "/tmp/searchindex-repo/${FILENAME}.gz"
+          cp "${ASSET}.gz" "${FILENAME}.gz"
           
           # Show compression stats
           ORIGINAL_SIZE=$(wc -c < "$ASSET")
@@ -71,37 +81,34 @@ jobs:
           RATIO=$(awk "BEGIN {printf \"%.1f\", ($COMPRESSED_SIZE / $ORIGINAL_SIZE) * 100}")
           echo "Compression: ${ORIGINAL_SIZE} bytes -> ${COMPRESSED_SIZE} bytes (${RATIO}%)"
           
-          # Commit and push with retry logic
+          # Add all files from other workflows (if they exist)
-          cd /tmp/searchindex-repo
+          git checkout main -- . 2>/dev/null || true
-          git config user.name "GitHub Actions"
-          git config user.email "github-actions@github.com"
-          git add "${FILENAME}" "${FILENAME}.gz"
           
-          if git diff --staged --quiet; then
+          # Add our new files (will overwrite if they existed)
-            echo "No changes to commit"
+          cp "$ASSET" "${FILENAME}"
-          else
+          cp "${ASSET}.gz" "${FILENAME}.gz"
-            git commit -m "Update ${FILENAME} from hacktricks-cloud build"
+          
-            
+          # Create README if it doesn't exist
-            # Retry push up to 20 times with pull --rebase between attempts
+          if [ ! -f "README.md" ]; then
-            MAX_RETRIES=20
+            echo "# HackTricks Search Index Repository" > README.md
-            RETRY_COUNT=0
+            echo "" >> README.md
-            while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+            echo "This repository contains searchindex files for HackTricks and HackTricks Cloud." >> README.md
-              if git push origin main; then
+            echo "Files are automatically generated and updated by GitHub Actions." >> README.md
-                echo "Successfully pushed on attempt $((RETRY_COUNT + 1))"
+            echo "" >> README.md
-                break
+            echo "⚠️ This repository is reset periodically to keep history clean." >> README.md
-              else
-                RETRY_COUNT=$((RETRY_COUNT + 1))
-                if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
-                  echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
-                  git pull --rebase origin main
-                  sleep $((RETRY_COUNT * 2))  # Exponential backoff
-                else
-                  echo "Failed to push after $MAX_RETRIES attempts"
-                  exit 1
-                fi
-              fi
-            done
           fi
+          
+          # Stage all files
+          git add -A
+          
+          # Commit with timestamp
+          TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
+          git commit -m "Update searchindex files - ${TIMESTAMP}"
+          
+          # Force push to replace main branch (deletes history)
+          git push -f origin new-main:main
+          
+          echo "Successfully reset repository and pushed searchindex files"
 
       # Login in AWs
       - name: Configure AWS credentials using OIDC

.github/workflows/translate_all.yml

@@ -202,7 +202,33 @@ jobs:
                 RETRY_COUNT=$((RETRY_COUNT + 1))
                 if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
                   echo "Push failed, attempt $RETRY_COUNT/$MAX_RETRIES. Pulling and retrying..."
-                  git pull --rebase origin main
+                  
+                  # Try normal rebase first
+                  if git pull --rebase origin main 2>&1 | tee /tmp/pull_output.txt; then
+                    echo "Rebase successful, retrying push..."
+                  else
+                    # If rebase fails due to divergent histories (orphan branch reset), re-clone
+                    if grep -q "unrelated histories\|refusing to merge\|fatal: invalid upstream" /tmp/pull_output.txt; then
+                      echo "Detected history rewrite, re-cloning repository..."
+                      cd /tmp
+                      rm -rf searchindex-repo
+                      git clone https://x-access-token:${TOKEN}@github.com/${TARGET_REPO}.git searchindex-repo
+                      cd searchindex-repo
+                      git config user.name "GitHub Actions"
+                      git config user.email "github-actions@github.com"
+                      
+                      # Re-copy and compress the searchindex file
+                      cp "$ASSET" "${FILENAME}"
+                      cp "${ASSET}.gz" "${FILENAME}.gz"
+                      
+                      git add "${FILENAME}" "${FILENAME}.gz"
+                      git commit -m "Update ${FILENAME} from hacktricks-cloud build"
+                      echo "Re-cloned and re-committed, will retry push..."
+                    else
+                      echo "Rebase failed for unknown reason, retrying anyway..."
+                    fi
+                  fi
+                  
                   sleep $((RETRY_COUNT * 2))  # Exponential backoff
                 else
                   echo "Failed to push after $MAX_RETRIES attempts"