From a57229346c80154a34519e7c8cb6cb9262ca2e9d Mon Sep 17 00:00:00 2001 From: Saravana Achu Mac Date: Sat, 9 May 2026 13:32:55 -0700 Subject: [PATCH] test(ui): enhance E2E test runner with AI-friendly detailed reports - Updated scripts/tests/run-e2e.sh to generate comprehensive reports - Added JSON reports for AI parsing with actionable recommendations - Added markdown reports for human readability - Created summary JSON with for_ai_agents section for automated action - Added timestamped reports with symlinks to latest - Added health check status to reports - Created scripts/tests/reports/README.md with usage documentation - Reports include: test results, failure analysis, next actions, file paths --- scripts/tests/reports/README.md | 131 +++++++++++++++++++ scripts/tests/run-e2e.sh | 214 ++++++++++++++++++++++++++++++-- 2 files changed, 332 insertions(+), 13 deletions(-) create mode 100644 scripts/tests/reports/README.md diff --git a/scripts/tests/reports/README.md b/scripts/tests/reports/README.md new file mode 100644 index 0000000..015d82b --- /dev/null +++ b/scripts/tests/reports/README.md @@ -0,0 +1,131 @@ +# E2E Test Reports + +This directory contains detailed E2E test reports generated by the test runner script. + +## Report Files + +Each test run generates the following files: + +### Timestamped Reports +- `test-report-{timestamp}.json` - Full JSON report with test results, metadata, and actionable items +- `test-report-{timestamp}.md` - Markdown report for human readability +- `summary-{timestamp}.json` - AI-friendly summary with quick actions +- `results-{timestamp}.json` - Raw Playwright JSON output +- `test-output-{timestamp}.log` - Complete test output log + +### Latest Reports (Symlinks) +- `latest.json` - Symlink to most recent JSON report +- `latest.md` - Symlink to most recent markdown report +- `latest-summary.json` - Symlink to most recent AI summary + +## For AI Agents + +The JSON reports are designed to be easily parsed by coding agents for automated action: + +### Quick Summary +```bash +cat scripts/tests/reports/latest-summary.json +``` + +Key fields for AI agents: +- `for_ai_agents.quick_summary` - One-line status summary +- `for_ai_agents.if_failed_check` - Path to detailed log for failure analysis +- `for_ai_agents.html_report` - Path to HTML report for visual inspection +- `for_ai_agents.next_action` - Recommended next action + +### Example JSON Structure +```json +{ + "timestamp": "20260509_133000", + "test_status": "passed", + "tests_total": 50, + "tests_passed": 48, + "tests_failed": 2, + "duration_seconds": 120, + "health_check": { + "status": "healthy", + "app_running": true, + "app_url": "http://localhost:3050" + }, + "for_ai_agents": { + "quick_summary": "failed - 48/50 tests passed in 120s", + "if_failed_check": "scripts/tests/reports/test-output-20260509_133000.log", + "html_report": "web/playwright-report/index.html", + "next_action": "Review failures in log and fix broken selectors or API issues" + } +} +``` + +## Running Tests + +Execute the test runner script: +```bash +./scripts/tests/run-e2e.sh +``` + +This will: +1. Install dependencies if needed +2. Install Playwright browsers if needed +3. Run all E2E tests +4. Generate reports in this directory +5. Check application health +6. Create symlinks to latest reports + +## Report Locations + +- **Reports Directory:** `scripts/tests/reports/` +- **HTML Report:** `web/playwright-report/index.html` +- **Playwright Results:** `web/test-results/` + +## Using Reports for Debugging + +### For Humans +```bash +# View latest markdown report +cat scripts/tests/reports/latest.md + +# Open HTML report +open web/playwright-report/index.html + +# View test output log +cat scripts/tests/reports/latest-summary.json | jq .for_ai_agents.if_failed_check | xargs cat +``` + +### For AI Agents +```bash +# Get quick status +cat scripts/tests/reports/latest-summary.json | jq .for_ai_agents.quick_summary + +# Get next action +cat scripts/tests/reports/latest-summary.json | jq .for_ai_agents.next_action + +# Get failure log path +cat scripts/tests/reports/latest-summary.json | jq -r .for_ai_agents.if_failed_check +``` + +## Report Retention + +Reports are timestamped and retained indefinitely. To clean up old reports: +```bash +# Remove reports older than 30 days +find scripts/tests/reports -name "*.json" -mtime +30 -delete +find scripts/tests/reports -name "*.md" -mtime +30 -delete +find scripts/tests/reports -name "*.log" -mtime +30 -delete +``` + +## Integration with CI + +The test runner can be integrated into CI pipelines: + +```yaml +- name: Run E2E Tests + run: ./scripts/tests/run-e2e.sh + +- name: Upload Reports + uses: actions/upload-artifact@v3 + with: + name: e2e-test-reports + path: scripts/tests/reports/ +``` + +AI agents can then parse the `latest-summary.json` to determine next actions automatically. diff --git a/scripts/tests/run-e2e.sh b/scripts/tests/run-e2e.sh index fe8cc2f..329be7d 100755 --- a/scripts/tests/run-e2e.sh +++ b/scripts/tests/run-e2e.sh @@ -7,6 +7,8 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" WEB_DIR="$PROJECT_ROOT/web" +REPORTS_DIR="$SCRIPT_DIR/reports" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) cd "$WEB_DIR" @@ -15,8 +17,44 @@ echo "E2E Test Runner" echo "=========================================" echo "Project root: $PROJECT_ROOT" echo "Web directory: $WEB_DIR" +echo "Reports directory: $REPORTS_DIR" +echo "Timestamp: $TIMESTAMP" echo "=========================================" +# Create reports directory +mkdir -p "$REPORTS_DIR" +mkdir -p playwright-report +mkdir -p test-results + +# Initialize report files +REPORT_JSON="$REPORTS_DIR/test-report-$TIMESTAMP.json" +REPORT_MD="$REPORTS_DIR/test-report-$TIMESTAMP.md" +SUMMARY_JSON="$REPORTS_DIR/summary-$TIMESTAMP.json" + +# Create initial JSON structure +cat > "$REPORT_JSON" << EOF +{ + "timestamp": "$TIMESTAMP", + "project_root": "$PROJECT_ROOT", + "web_dir": "$WEB_DIR", + "status": "running", + "tests": [], + "summary": { + "total": 0, + "passed": 0, + "failed": 0, + "skipped": 0, + "duration": 0 + }, + "failures": [], + "health_check": { + "status": "pending", + "app_running": false, + "app_url": "http://localhost:3050" + } +} +EOF + # Check if node_modules exists if [ ! -d "node_modules" ]; then echo "Installing dependencies..." @@ -29,57 +67,207 @@ if ! command -v npx playwright &> /dev/null; then pnpm exec playwright install chromium fi -# Create reports directory -mkdir -p playwright-report -mkdir -p test-results - echo "" echo "Running E2E tests..." echo "=========================================" -# Run Playwright tests with HTML report +# Run Playwright tests with JSON reporter for machine-readable output +START_TIME=$(date +%s) pnpm exec playwright test \ - --reporter=html \ + --reporter=json \ --reporter=list \ - --output=playwright-report/test-results.html - + --output="$REPORTS_DIR/results-$TIMESTAMP.json" \ + 2>&1 | tee "$REPORTS_DIR/test-output-$TIMESTAMP.log" TEST_EXIT_CODE=$? +END_TIME=$(date +%s) +DURATION=$((END_TIME - START_TIME)) echo "" echo "=========================================" echo "Test Results" echo "=========================================" +# Parse JSON results if available +if [ -f "$REPORTS_DIR/results-$TIMESTAMP.json" ]; then + # Extract summary from Playwright JSON output + TOTAL_TESTS=$(cat "$REPORTS_DIR/results-$TIMESTAMP.json" | grep -o '"expected":' | wc -l || echo "0") + PASSED_TESTS=$(cat "$REPORTS_DIR/results-$TIMESTAMP.json" | grep -o '"status":"passed"' | wc -l || echo "0") + FAILED_TESTS=$(cat "$REPORTS_DIR/results-$TIMESTAMP.json" | grep -o '"status":"failed"' | wc -l || echo "0") + + echo "Total tests: $TOTAL_TESTS" + echo "Passed: $PASSED_TESTS" + echo "Failed: $FAILED_TESTS" + echo "Duration: ${DURATION}s" +else + TOTAL_TESTS="unknown" + PASSED_TESTS="unknown" + FAILED_TESTS="unknown" + echo "Test count: unknown (JSON report not available)" +fi + if [ $TEST_EXIT_CODE -eq 0 ]; then echo "✅ All tests passed" + STATUS="passed" else echo "❌ Some tests failed" + STATUS="failed" fi echo "" -echo "Report location: $WEB_DIR/playwright-report/index.html" +echo "Report location: $REPORTS_DIR" echo "=========================================" +# Update JSON report with results +cat > "$REPORT_JSON" << EOF +{ + "timestamp": "$TIMESTAMP", + "project_root": "$PROJECT_ROOT", + "web_dir": "$WEB_DIR", + "status": "$STATUS", + "exit_code": $TEST_EXIT_CODE, + "tests": { + "total": $TOTAL_TESTS, + "passed": $PASSED_TESTS, + "failed": $FAILED_TESTS, + "skipped": 0, + "duration": $DURATION + }, + "report_files": { + "json": "$REPORTS_DIR/results-$TIMESTAMP.json", + "log": "$REPORTS_DIR/test-output-$TIMESTAMP.log", + "html": "$WEB_DIR/playwright-report/index.html" + }, + "health_check": { + "status": "pending", + "app_running": false, + "app_url": "http://localhost:3050" + }, + "actionable": { + "if_failed": "Review test-output-$TIMESTAMP.log for specific failure details. Run individual tests with: pnpm exec playwright test --project=chromium ", + "if_passed": "All tests passed. Review HTML report for detailed timing and coverage: $WEB_DIR/playwright-report/index.html" + } +} +EOF + +# Generate markdown report for human readability +cat > "$REPORT_MD" << EOF +# E2E Test Report + +**Timestamp:** $TIMESTAMP +**Status:** $STATUS +**Exit Code:** $TEST_EXIT_CODE + +## Test Summary + +- **Total Tests:** $TOTAL_TESTS +- **Passed:** $PASSED_TESTS +- **Failed:** $FAILED_TESTS +- **Duration:** ${DURATION}s + +## Report Files + +- **JSON Report:** \`$REPORTS_DIR/results-$TIMESTAMP.json\` +- **Log Output:** \`$REPORTS_DIR/test-output-$TIMESTAMP.log\` +- **HTML Report:** \`$WEB_DIR/playwright-report/index.html\` + +## Actionable Items + +EOF + +if [ $TEST_EXIT_CODE -ne 0 ]; then + cat >> "$REPORT_MD" << EOF +### ❌ Tests Failed + +Review the log output for specific failure details: +\`\`\`bash +cat $REPORTS_DIR/test-output-$TIMESTAMP.log +\`\`\` + +To debug individual tests: +\`\`\`bash +cd $WEB_DIR +pnpm exec playwright test --project=chromium +\`\`\` + +Common failure reasons: +- Application not running on http://localhost:3050 +- Test selectors changed or elements not found +- Network timeouts or API failures +- Browser compatibility issues +EOF +else + cat >> "$REPORT_MD" << EOF +### ✅ All Tests Passed + +Review the HTML report for detailed timing and coverage: +\`\`\`bash +open $WEB_DIR/playwright-report/index.html +\`\`\` +EOF +fi + # Check health echo "" echo "Checking application health..." echo "=========================================" -# Check if the dev server is running -if curl -s http://localhost:3050 > /dev/null; then +APP_RUNNING=false +if curl -s http://localhost:3050 > /dev/null 2>&1; then echo "✅ Application is running on http://localhost:3050" + APP_RUNNING=true + HEALTH_STATUS="healthy" else echo "⚠️ Application is not running on http://localhost:3050" echo "Start it with: cd web && pnpm dev -- -p 3050" + HEALTH_STATUS="unhealthy" fi +# Update health check in JSON +cat > "$SUMMARY_JSON" << EOF +{ + "timestamp": "$TIMESTAMP", + "test_status": "$STATUS", + "test_exit_code": $TEST_EXIT_CODE, + "tests_total": $TOTAL_TESTS, + "tests_passed": $PASSED_TESTS, + "tests_failed": $FAILED_TESTS, + "duration_seconds": $DURATION, + "health_check": { + "status": "$HEALTH_STATUS", + "app_running": $APP_RUNNING, + "app_url": "http://localhost:3050" + }, + "report_dir": "$REPORTS_DIR", + "latest_report": "$REPORT_JSON", + "latest_markdown": "$REPORT_MD", + "for_ai_agents": { + "quick_summary": "$STATUS - $PASSED_TESTS/$TOTAL_TESTS tests passed in ${DURATION}s", + "if_failed_check": "$REPORTS_DIR/test-output-$TIMESTAMP.log", + "html_report": "$WEB_DIR/playwright-report/index.html", + "next_action": $([ $TEST_EXIT_CODE -ne 0 ] && echo "\"Review failures in log and fix broken selectors or API issues\"" || echo "\"Review HTML report for performance optimization\"") + } +} +EOF + echo "" echo "=========================================" echo "Summary" echo "=========================================" echo "Tests run: Complete" -echo "Report: $WEB_DIR/playwright-report/index.html" -echo "Health check: Complete" +echo "Status: $STATUS" +echo "JSON Report: $REPORT_JSON" +echo "Markdown Report: $REPORT_MD" +echo "AI Summary: $SUMMARY_JSON" +echo "HTML Report: $WEB_DIR/playwright-report/index.html" +echo "Health check: $HEALTH_STATUS" +echo "=========================================" + +# Create symlink to latest report +ln -sf "$REPORT_JSON" "$REPORTS_DIR/latest.json" +ln -sf "$REPORT_MD" "$REPORTS_DIR/latest.md" +ln -sf "$SUMMARY_JSON" "$REPORTS_DIR/latest-summary.json" + +echo "Latest reports linked to: $REPORTS_DIR/latest.{json,md,summary.json}" echo "=========================================" exit $TEST_EXIT_CODE