135 lines
3.6 KiB
Bash
Executable File
135 lines
3.6 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# Test Suite Validation Script
|
||
# Runs tests N times and collects failure metrics
|
||
# Usage: ./scripts/validate-test-suite.sh [N] [test_path]
|
||
# N - Number of times to run tests (default: 20)
|
||
# test_path - Test path (default: ./...)
|
||
|
||
set -e
|
||
|
||
# Default values
|
||
RUN_COUNT=${1:-20}
|
||
TEST_PATH=${2:-./...}
|
||
SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
|
||
|
||
# Colors for output
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# Temporary files
|
||
FAILURE_LOG=$(mktemp)
|
||
UNIQUE_FAILURES=$(mktemp)
|
||
SUMMARY_REPORT=$(mktemp)
|
||
|
||
# Cleanup temporary files on exit
|
||
cleanup() {
|
||
rm -f "$FAILURE_LOG" "$UNIQUE_FAILURES" "$SUMMARY_REPORT"
|
||
}
|
||
trap cleanup EXIT
|
||
|
||
echo "🧪 Test Suite Validation Script"
|
||
echo "=============================="
|
||
echo "Runs: $RUN_COUNT"
|
||
echo "Tests: $TEST_PATH"
|
||
echo "Date: $(date)"
|
||
echo
|
||
|
||
# Initialize counters
|
||
SUCCESS_COUNT=0
|
||
FAILURE_COUNT=0
|
||
START_TIME=$(date +%s)
|
||
|
||
echo "Starting validation runs..."
|
||
echo
|
||
|
||
# Main validation loop
|
||
for (( run=1; run<=$RUN_COUNT; run++ )); do
|
||
echo "Run $run/$RUN_COUNT..."
|
||
|
||
# Clean test cache for each run
|
||
go clean -testcache > /dev/null 2>&1
|
||
|
||
# Run tests and capture output
|
||
set +e # Temporarily disable exit on error
|
||
TEST_OUTPUT=$(go test $TEST_PATH -v 2>&1)
|
||
TEST_EXIT_CODE=$?
|
||
set -e # Re-enable exit on error
|
||
|
||
if [ $TEST_EXIT_CODE -eq 0 ]; then
|
||
echo " ✅ Passed"
|
||
((SUCCESS_COUNT++))
|
||
else
|
||
echo " ❌ Failed"
|
||
((FAILURE_COUNT++))
|
||
|
||
# Extract failing test names and errors
|
||
echo "$TEST_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$FAILURE_LOG"
|
||
|
||
# Extract specific test failures with errors
|
||
echo "$TEST_OUTPUT" | grep -A 5 "FAIL.*\.go" | head -6 >> "$FAILURE_LOG"
|
||
echo "---" >> "$FAILURE_LOG"
|
||
fi
|
||
done
|
||
|
||
echo
|
||
END_TIME=$(date +%s)
|
||
DURATION=$((END_TIME - START_TIME))
|
||
|
||
echo "Validation Complete"
|
||
echo "=================="
|
||
echo "Total Runs: $RUN_COUNT"
|
||
echo "Success: ${GREEN}$SUCCESS_COUNT${NC}"
|
||
echo "Failures: ${RED}$FAILURE_COUNT${NC}"
|
||
echo "Duration: $DURATION seconds"
|
||
echo
|
||
|
||
# Check if there were any failures
|
||
if [ $FAILURE_COUNT -eq 0 ]; then
|
||
echo "${GREEN}✅ All tests passed successfully!${NC}"
|
||
echo "Test suite is stable and ready for production"
|
||
exit 0
|
||
else
|
||
echo "${RED}❌ Some tests failed during validation${NC}"
|
||
echo
|
||
|
||
# Process failure log to get unique failures with counts
|
||
if [ -s "$FAILURE_LOG" ]; then
|
||
echo "Failure Analysis"
|
||
echo "================"
|
||
|
||
# Count occurrences of each failing test
|
||
echo "Failing Test Summary:"
|
||
grep "FAIL" "$FAILURE_LOG" | sort | uniq -c | sort -rn | while read count test; do
|
||
test_name=$(echo "$test" | sed 's/FAIL[[:space:]]*//')
|
||
echo " $count × $test_name"
|
||
done
|
||
|
||
echo
|
||
echo "Unique Failure Patterns:"
|
||
|
||
# Extract unique failure patterns
|
||
grep -E "^(FAIL|---)" "$FAILURE_LOG" | sort | uniq | while read line; do
|
||
if [[ "$line" == FAIL* ]]; then
|
||
echo " • $line"
|
||
fi
|
||
done
|
||
|
||
echo
|
||
echo "Detailed Failure Log:"
|
||
echo "======================"
|
||
cat "$FAILURE_LOG"
|
||
|
||
echo
|
||
echo "Recommendations:"
|
||
echo " 1. Mark flaky tests with @flaky tag"
|
||
echo " 2. Investigate and fix failing tests"
|
||
echo " 3. Run with FIXED_TEST_PORT=true to debug port issues"
|
||
echo " 4. Check for race conditions in failing tests"
|
||
fi
|
||
|
||
exit 1
|
||
fi |