🧪 test: enhance validation script with separate unit/BDD tests and better failure parsing

2026-04-10 15:00:33 +02:00
parent 230ee699e4
commit 65b842b070
1 changed files with 99 additions and 55 deletions
--- a/scripts/validate-test-suite.sh
+++ b/scripts/validate-test-suite.sh
@@ -1,16 +1,14 @@
 #!/bin/bash

 # Test Suite Validation Script
-# Runs tests N times and collects failure metrics
-# Usage: ./scripts/validate-test-suite.sh [N] [test_path]
+# Runs tests N times with separate unit and BDD test phases
+# Usage: ./scripts/validate-test-suite.sh [N]
 #   N          - Number of times to run tests (default: 20)
-#   test_path  - Test path (default: ./...)

 set -e

 # Default values
 RUN_COUNT=${1:-20}
-TEST_PATH=${2:-./...}
 SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")

 # Colors for output
@@ -21,26 +19,29 @@ BLUE='\033[0;34m'
 NC='\033[0m' # No Color

 # Temporary files
-FAILURE_LOG=$(mktemp)
-UNIQUE_FAILURES=$(mktemp)
+UNIT_FAILURE_LOG=$(mktemp)
+BDD_FAILURE_LOG=$(mktemp)
 SUMMARY_REPORT=$(mktemp)

 # Cleanup temporary files on exit
 cleanup() {
-    rm -f "$FAILURE_LOG" "$UNIQUE_FAILURES" "$SUMMARY_REPORT"
+    rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT"
 }
 trap cleanup EXIT

 echo "🧪 Test Suite Validation Script"
 echo "=============================="
 echo "Runs: $RUN_COUNT"
-echo "Tests: $TEST_PATH"
+echo "Unit Tests: ./cmd/... ./pkg/..."
+echo "BDD Tests: ./features/..."
 echo "Date: $(date)"
 echo

 # Initialize counters
-SUCCESS_COUNT=0
-FAILURE_COUNT=0
+UNIT_SUCCESS=0
+UNIT_FAILURE=0
+BDD_SUCCESS=0
+BDD_FAILURE=0
 START_TIME=$(date +%s)

 echo "Starting validation runs..."
@@ -50,28 +51,48 @@ echo
 for (( run=1; run<=$RUN_COUNT; run++ )); do
    echo "Run $run/$RUN_COUNT..."
    
-    # Clean test cache for each run
+    # ===== UNIT TESTS =====
+    echo "  🧪 Unit tests..."
    go clean -testcache > /dev/null 2>&1
    
-    # Run tests and capture output
    set +e  # Temporarily disable exit on error
-    TEST_OUTPUT=$(go test $TEST_PATH -v 2>&1)
-    TEST_EXIT_CODE=$?
+    UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1)
+    UNIT_EXIT_CODE=$?
    set -e  # Re-enable exit on error
    
-    if [ $TEST_EXIT_CODE -eq 0 ]; then
-        echo "  ✅ Passed"
-        ((SUCCESS_COUNT++))
+    if [ $UNIT_EXIT_CODE -eq 0 ]; then
+        echo "    ✅ Passed"
+        ((UNIT_SUCCESS++))
    else
-        echo "  ❌ Failed"
-        ((FAILURE_COUNT++))
+        echo "    ❌ Failed"
+        ((UNIT_FAILURE++))
        
-        # Extract failing test names and errors
-        echo "$TEST_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$FAILURE_LOG"
+        # Extract detailed unit test failures
+        echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG"
+        echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG"
+        echo "---" >> "$UNIT_FAILURE_LOG"
+    fi
+    
+    # ===== BDD TESTS =====
+    echo "  🧪 BDD tests..."
+    go clean -testcache > /dev/null 2>&1
+    
+    set +e  # Temporarily disable exit on error
+    BDD_OUTPUT=$(go test ./features/... -v 2>&1)
+    BDD_EXIT_CODE=$?
+    set -e  # Re-enable exit on error
+    
+    if [ $BDD_EXIT_CODE -eq 0 ]; then
+        echo "    ✅ Passed"
+        ((BDD_SUCCESS++))
+    else
+        echo "    ❌ Failed"
+        ((BDD_FAILURE++))
        
-        # Extract specific test failures with errors
-        echo "$TEST_OUTPUT" | grep -A 5 "FAIL.*\.go" | head -6 >> "$FAILURE_LOG"
-        echo "---" >> "$FAILURE_LOG"
+        # Extract detailed BDD test failures with actual test names
+        echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG"
+        echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG"
+        echo "---" >> "$BDD_FAILURE_LOG"
    fi
 done

@@ -82,13 +103,19 @@ DURATION=$((END_TIME - START_TIME))
 echo "Validation Complete"
 echo "=================="
 echo "Total Runs: $RUN_COUNT"
-echo "Success: ${GREEN}$SUCCESS_COUNT${NC}"
-echo "Failures: ${RED}$FAILURE_COUNT${NC}"
+echo "Unit Tests:"
+echo "  Success: ${GREEN}$UNIT_SUCCESS${NC}"
+echo "  Failures: ${RED}$UNIT_FAILURE${NC}"
+echo "BDD Tests:"
+echo "  Success: ${GREEN}$BDD_SUCCESS${NC}"
+echo "  Failures: ${RED}$BDD_FAILURE${NC}"
 echo "Duration: $DURATION seconds"
 echo

-# Check if there were any failures
-if [ $FAILURE_COUNT -eq 0 ]; then
+# Check overall success
+TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE))
+
+if [ $TOTAL_FAILURES -eq 0 ]; then
    echo "${GREEN}✅ All tests passed successfully!${NC}"
    echo "Test suite is stable and ready for production"
    exit 0
@@ -96,40 +123,57 @@ else
    echo "${RED}❌ Some tests failed during validation${NC}"
    echo
    
-    # Process failure log to get unique failures with counts
-    if [ -s "$FAILURE_LOG" ]; then
-        echo "Failure Analysis"
+    # Process unit test failures
+    if [ -s "$UNIT_FAILURE_LOG" ]; then
+        echo "Unit Test Failures:"
+        echo "=================="
+        
+        # Count unit test failures
+        UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn)
+        if [ -n "$UNIT_FAILURES" ]; then
+            echo "$UNIT_FAILURES"
+        else
+            echo "  None (check log for details)"
+        fi
+        
+        echo
+    fi
+    
+    # Process BDD test failures
+    if [ -s "$BDD_FAILURE_LOG" ]; then
+        echo "BDD Test Failures:"
        echo "================"
        
-        # Count occurrences of each failing test
-        echo "Failing Test Summary:"
-        grep "FAIL" "$FAILURE_LOG" | sort | uniq -c | sort -rn | while read count test; do
-            test_name=$(echo "$test" | sed 's/FAIL[[:space:]]*//')
-            echo "  $count × $test_name"
-        done
+        # Count BDD test failures with granularity
+        BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | sort | uniq -c | sort -rn)
+        if [ -n "$BDD_FAILURES" ]; then
+            echo "Summary:"
+            while IFS= read -r line; do
+                count=$(echo "$line" | awk '{print $1}')
+                test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//')
+                echo "  $count × $test"
+            done <<< "$BDD_FAILURES"
+        else
+            echo "  None (check log for details)"
+        fi
        
        echo
-        echo "Unique Failure Patterns:"
-        
-        # Extract unique failure patterns
-        grep -E "^(FAIL|---)" "$FAILURE_LOG" | sort | uniq | while read line; do
-            if [[ "$line" == FAIL* ]]; then
-                echo "  • $line"
-            fi
-        done
+        echo "Detailed BDD Failure Log (first 20 lines):"
+        echo "=========================================="
+        # Show only the relevant failure lines with actual test names
+        grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | head -20
        
        echo
-        echo "Detailed Failure Log:"
-        echo "======================"
-        cat "$FAILURE_LOG"
-        
-        echo
-        echo "Recommendations:"
-        echo "  1. Mark flaky tests with @flaky tag"
-        echo "  2. Investigate and fix failing tests"
-        echo "  3. Run with FIXED_TEST_PORT=true to debug port issues"
-        echo "  4. Check for race conditions in failing tests"
+        echo "Full failure log saved in: $BDD_FAILURE_LOG"
    fi
    
+    echo
+    echo "Recommendations:"
+    echo "  1. Mark flaky BDD tests with @flaky tag"
+    echo "  2. Investigate unit test failures first (faster to fix)"
+    echo "  3. Check for race conditions in failing tests"
+    echo "  4. Run with FIXED_TEST_PORT=true for debugging"
+    echo "  5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags"
+    
    exit 1
 fi