scripts/validate-test-suite.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+
+# Test Suite Validation Script
+# Runs tests N times and collects failure metrics
+# Usage: ./scripts/validate-test-suite.sh [N] [test_path]
+#   N          - Number of times to run tests (default: 20)
+#   test_path  - Test path (default: ./...)
+
+set -e
+
+# Default values
+RUN_COUNT=${1:-20}
+TEST_PATH=${2:-./...}
+SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Temporary files
+FAILURE_LOG=$(mktemp)
+UNIQUE_FAILURES=$(mktemp)
+SUMMARY_REPORT=$(mktemp)
+
+# Cleanup temporary files on exit
+cleanup() {
+    rm -f "$FAILURE_LOG" "$UNIQUE_FAILURES" "$SUMMARY_REPORT"
+}
+trap cleanup EXIT
+
+echo "🧪 Test Suite Validation Script"
+echo "=============================="
+echo "Runs: $RUN_COUNT"
+echo "Tests: $TEST_PATH"
+echo "Date: $(date)"
+echo
+
+# Initialize counters
+SUCCESS_COUNT=0
+FAILURE_COUNT=0
+START_TIME=$(date +%s)
+
+echo "Starting validation runs..."
+echo
+
+# Main validation loop
+for (( run=1; run<=$RUN_COUNT; run++ )); do
+    echo "Run $run/$RUN_COUNT..."
+    
+    # Clean test cache for each run
+    go clean -testcache > /dev/null 2>&1
+    
+    # Run tests and capture output
+    set +e  # Temporarily disable exit on error
+    TEST_OUTPUT=$(go test $TEST_PATH -v 2>&1)
+    TEST_EXIT_CODE=$?
+    set -e  # Re-enable exit on error
+    
+    if [ $TEST_EXIT_CODE -eq 0 ]; then
+        echo "  ✅ Passed"
+        ((SUCCESS_COUNT++))
+    else
+        echo "  ❌ Failed"
+        ((FAILURE_COUNT++))
+        
+        # Extract failing test names and errors
+        echo "$TEST_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$FAILURE_LOG"
+        
+        # Extract specific test failures with errors
+        echo "$TEST_OUTPUT" | grep -A 5 "FAIL.*\.go" | head -6 >> "$FAILURE_LOG"
+        echo "---" >> "$FAILURE_LOG"
+    fi
+done
+
+echo
+END_TIME=$(date +%s)
+DURATION=$((END_TIME - START_TIME))
+
+echo "Validation Complete"
+echo "=================="
+echo "Total Runs: $RUN_COUNT"
+echo "Success: ${GREEN}$SUCCESS_COUNT${NC}"
+echo "Failures: ${RED}$FAILURE_COUNT${NC}"
+echo "Duration: $DURATION seconds"
+echo
+
+# Check if there were any failures
+if [ $FAILURE_COUNT -eq 0 ]; then
+    echo "${GREEN}✅ All tests passed successfully!${NC}"
+    echo "Test suite is stable and ready for production"
+    exit 0
+else
+    echo "${RED}❌ Some tests failed during validation${NC}"
+    echo
+    
+    # Process failure log to get unique failures with counts
+    if [ -s "$FAILURE_LOG" ]; then
+        echo "Failure Analysis"
+        echo "================"
+        
+        # Count occurrences of each failing test
+        echo "Failing Test Summary:"
+        grep "FAIL" "$FAILURE_LOG" | sort | uniq -c | sort -rn | while read count test; do
+            test_name=$(echo "$test" | sed 's/FAIL[[:space:]]*//')
+            echo "  $count × $test_name"
+        done
+        
+        echo
+        echo "Unique Failure Patterns:"
+        
+        # Extract unique failure patterns
+        grep -E "^(FAIL|---)" "$FAILURE_LOG" | sort | uniq | while read line; do
+            if [[ "$line" == FAIL* ]]; then
+                echo "  • $line"
+            fi
+        done
+        
+        echo
+        echo "Detailed Failure Log:"
+        echo "======================"
+        cat "$FAILURE_LOG"
+        
+        echo
+        echo "Recommendations:"
+        echo "  1. Mark flaky tests with @flaky tag"
+        echo "  2. Investigate and fix failing tests"
+        echo "  3. Run with FIXED_TEST_PORT=true to debug port issues"
+        echo "  4. Check for race conditions in failing tests"
+    fi
+    
+    exit 1
+fi