Terrace/test/test-runner.js

#!/usr/bin/env node

import { execSync } from 'node:child_process'
import fs from 'node:fs/promises'
import path from 'node:path'
import { fileURLToPath } from 'node:url'

const __dirname = path.dirname(fileURLToPath(import.meta.url))

// Language implementations and their test commands
const LANGUAGES = {
  js: {
    command: 'node test/index.js',
    cwd: path.join(__dirname, '..', 'packages', 'js'),
  },
  c: {
    command: './test/test-runner',
    cwd: path.join(__dirname, '..', 'packages', 'c'),
    buildCommand: 'make',
  },
  python: {
    command: 'python3 test/index.py',
    cwd: path.join(__dirname, '..', 'packages', 'python'),
  },
  go: {
    command: 'go run test/test-runner.go',
    cwd: path.join(__dirname, '..', 'packages', 'go'),
  },
  rust: {
    command: 'cargo run --bin test-runner --',
    cwd: path.join(__dirname, '..', 'packages', 'rust'),
  }
}

// Test result tracking
let totalTests = 0
let passedTests = 0
let failedTests = 0
const failures = []

async function runTest(language, testName, input) {
  const langConfig = LANGUAGES[language]
  if (!langConfig) {
    throw new Error(`Unknown language: ${language}`)
  }

  // Build if necessary
  if (langConfig.buildCommand) {
    try {
      execSync(langConfig.buildCommand, {
        cwd: langConfig.cwd,
        stdio: 'pipe'
      })
    } catch (error) {
      return {
        error: `Build failed: ${error.message}`,
        success: false
      }
    }
  }

  try {
    let command
    // All languages now take input from stdin and produce comparable output
    command = `${langConfig.command} ${testName}`
    const result = execSync(command, {
      cwd: langConfig.cwd,
      input: input || '',
      encoding: 'utf8',
      timeout: 10000 // 10 second timeout
    })
    return { output: result.trim(), success: true }
  } catch (error) {
    return {
      error: error.message,
      stderr: error.stderr?.toString() || '',
      success: false
    }
  }
}

async function findTestFiles() {
  const allTceFiles = []

  async function scanDirectory(dir) {
    try {
      const entries = await fs.readdir(dir, { withFileTypes: true })

      for (const entry of entries) {
        const fullPath = path.join(dir, entry.name)

        if (entry.isDirectory() && !entry.name.startsWith('.') && entry.name !== 'node_modules') {
          await scanDirectory(fullPath)
        } else if (entry.isFile() && entry.name.endsWith('.tce')) {
          allTceFiles.push(fullPath)
        }
      }
    } catch (error) {
      // Skip directories we can't read
    }
  }

  await scanDirectory(path.join(__dirname, '..'))

  // Filter to only test files (those containing #schema test)
  const testFiles = []
  for (const file of allTceFiles) {
    try {
      const content = await fs.readFile(file, 'utf8')
      if (content.includes('#schema test')) {
        testFiles.push(file)
      }
    } catch (error) {
      // Skip files we can't read
    }
  }

  return testFiles
}

async function loadTestMap(filePath) {
  // Import the helpers from the test directory
  const { loadTestMap: loadMap } = await import('./helpers.js')
  return await loadMap(filePath)
}

async function runUnifiedTests(languageFilter = null) {
  console.log('🔍 Discovering test files...')
  const testFiles = await findTestFiles()
  console.log(`📁 Found ${testFiles.length} test file(s)`)

  if (languageFilter) {
    console.log(`🎯 Filtering to languages: ${languageFilter.join(', ')}`)
  }

  for (const testFile of testFiles) {
    console.log(`\n📄 Running tests from: ${path.relative(path.join(__dirname, '..'), testFile)}`)

    try {
      const testMap = await loadTestMap(testFile)

      for (const [suiteName, tests] of Object.entries(testMap)) {
        console.log(`\n🧪 Test Suite: ${suiteName}`)

        for (const test of tests) {
          console.log(`\n  📝 ${test.it}`)
          totalTests++

          const expectedOutput = test.output
          const input = test.input
          let packages = test.packages.length > 0 ? test.packages : Object.keys(LANGUAGES)

          // Apply language filter if specified
          if (languageFilter) {
            packages = packages.filter(pkg => languageFilter.includes(pkg))
          }

          // Skip test if no packages match the filter
          if (packages.length === 0) {
            console.log(`    ⏭️  Skipped (no matching languages)`)
            continue
          }

          let testPassed = true
          const results = {}

          for (const pkg of packages) {
            if (!LANGUAGES[pkg]) {
              console.log(`    ⚠️  Unknown package: ${pkg}`)
              continue
            }

            try {
              const result = await runTest(pkg, test.key, input)

              if (result.skipped) {
                results[pkg] = { status: 'skipped' }
                continue
              }

              if (result.success) {
                const actualOutput = result.output
                // Compare outputs for all languages
                if (actualOutput === expectedOutput) {
                  results[pkg] = { status: 'passed', output: actualOutput }
                } else {
                  results[pkg] = {
                    status: 'failed',
                    output: actualOutput,
                    expected: expectedOutput
                  }
                  testPassed = false
                }
              } else {
                results[pkg] = {
                  status: 'error',
                  error: result.error,
                  stderr: result.stderr
                }
                testPassed = false
              }
            } catch (error) {
              results[pkg] = { status: 'error', error: error.message }
              testPassed = false
            }
          }

          // Display results
          for (const [pkg, result] of Object.entries(results)) {
            if (result.status === 'passed') {
              console.log(`    ✅ ${pkg}: PASSED`)
            } else if (result.status === 'skipped') {
              console.log(`    ⏭️  ${pkg}: SKIPPED`)
            } else if (result.status === 'failed') {
              console.log(`    ❌ ${pkg}: FAILED`)
              console.log(`       Expected: ${result.expected}`)
              console.log(`       Got:      ${result.output}`)
            } else if (result.status === 'error') {
              console.log(`    💥 ${pkg}: ERROR`)
              if (result.error) console.log(`       Error: ${result.error}`)
              if (result.stderr) console.log(`       Stderr: ${result.stderr}`)
            }
          }

          if (testPassed) {
            passedTests++
          } else {
            failedTests++
            failures.push({
              suite: suiteName,
              test: test.it,
              file: testFile,
              results
            })
          }
        }
      }
    } catch (error) {
      console.error(`❌ Error loading test file ${testFile}: ${error.message}`)
      failedTests++
      failures.push({
        file: testFile,
        error: error.message
      })
    }
  }

  // Summary
  console.log('\n' + '='.repeat(50))
  console.log('🏁 TEST SUMMARY')
  console.log('='.repeat(50))
  console.log(`Total Tests: ${totalTests}`)
  console.log(`Passed: ${passedTests}`)
  console.log(`Failed: ${failedTests}`)
  console.log(`Success Rate: ${totalTests > 0 ? ((passedTests / totalTests) * 100).toFixed(1) : 0}%`)

  if (failures.length > 0) {
    console.log('\n❌ FAILURES:')
    for (const failure of failures) {
      console.log(`  - ${failure.suite || 'File'}: ${failure.test || failure.file}`)
    }
    process.exit(1)
  } else {
    console.log('\n✅ All tests passed!')
  }
}

async function runSpecificTest(testName, packages = null) {
  const targetPackages = packages ? packages.split(',') : Object.keys(LANGUAGES)

  console.log(`🎯 Running specific test: ${testName}`)
  console.log(`📦 Target packages: ${targetPackages.join(', ')}`)

  let testPassed = true
  const results = {}

  for (const pkg of targetPackages) {
    if (!LANGUAGES[pkg]) {
      console.log(`⚠️  Unknown package: ${pkg}`)
      continue
    }

    try {
      const result = await runTest(pkg, testName)

      if (result.skipped) {
        results[pkg] = { status: 'skipped' }
      } else if (result.success) {
        results[pkg] = { status: 'passed', output: result.output }
      } else {
        results[pkg] = { status: 'error', error: result.error, stderr: result.stderr }
        testPassed = false
      }
    } catch (error) {
      results[pkg] = { status: 'error', error: error.message }
      testPassed = false
    }
  }

  // Display results
  for (const [pkg, result] of Object.entries(results)) {
    if (result.status === 'passed') {
      console.log(`✅ ${pkg}: PASSED`)
      if (result.output) console.log(`   Output: ${result.output}`)
    } else if (result.status === 'skipped') {
      console.log(`⏭️  ${pkg}: SKIPPED`)
    } else if (result.status === 'error') {
      console.log(`❌ ${pkg}: ERROR`)
      if (result.error) console.log(`   Error: ${result.error}`)
      if (result.stderr) console.log(`   Stderr: ${result.stderr}`)
    }
  }

  if (!testPassed) {
    process.exit(1)
  }
}

// Main execution
const args = process.argv.slice(2)

if (args.length === 0) {
  // Run all tests
  await runUnifiedTests()
} else if (args.length === 1) {
  // Run specific test across all packages
  await runSpecificTest(args[0])
} else if (args.length === 2 && args[0] === '--lang') {
  // Run all tests for specific language
  await runUnifiedTests([args[1]])
} else if (args.length === 2 && args[0] === '--langs') {
  // Run all tests for specific languages
  await runUnifiedTests(args[1].split(','))
} else if (args.length === 3 && args[0] === '--lang') {
  // Run specific test for specific language
  await runSpecificTest(args[2], args[1])
} else if (args.length === 3 && args[0] === '--langs') {
  // Run specific test for specific languages
  await runSpecificTest(args[2], args[1])
} else {
  console.log('Usage:')
  console.log('  node test-runner.js                          # Run all tests')
  console.log('  node test-runner.js <test-name>              # Run specific test on all packages')
  console.log('  node test-runner.js --lang <lang>            # Run all tests for specific language')
  console.log('  node test-runner.js --langs <lang1,lang2>    # Run all tests for specific languages')
  console.log('  node test-runner.js --lang <lang> <test-name> # Run specific test for specific language')
  console.log('  node test-runner.js --langs <lang1,lang2> <test-name> # Run specific test for specific languages')
  process.exit(1)
}