MangaReader/backend/test-concurrent-downloads.js

/**
 * Integration Test: Concurrent Downloads
 * Tests downloading multiple chapters in parallel to verify:
 * - No race conditions
 * - No file corruption
 * - Proper concurrent access to storage
 * - Correct storage statistics
 * - Independent chapter management
 */

import assert from 'assert';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import storage from './storage.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));

// Test configuration
const TEST_CONFIG = {
  mangaSlug: 'one-piece_1695365223767',
  chapters: [787, 788, 789, 790, 791], // Test with 5 chapters
  baseUrl: 'http://localhost:3001',
  timeout: 180000, // 3 minutes for concurrent downloads
  maxConcurrent: 3 // Limit concurrent downloads to avoid overwhelming the server
};

// Color codes for terminal output
const colors = {
  reset: '\x1b[0m',
  bright: '\x1b[1m',
  green: '\x1b[32m',
  red: '\x1b[31m',
  yellow: '\x1b[33m',
  blue: '\x1b[34m',
  cyan: '\x1b[36m',
  magenta: '\x1b[35m'
};

function log(message, color = 'reset') {
  console.log(`${colors[color]}${message}${colors.reset}`);
}

function logSection(title) {
  console.log('\n' + '='.repeat(70));
  log(title, 'bright');
  console.log('='.repeat(70));
}

function logTest(testName) {
  log(`\n▶ ${testName}`, 'cyan');
}

function logSuccess(message) {
  log(`✓ ${message}`, 'green');
}

function logError(message) {
  log(`✗ ${message}`, 'red');
}

function logInfo(message) {
  log(`  ℹ ${message}`, 'blue');
}

function logWarning(message) {
  log(`⚠ ${message}`, 'yellow');
}

// Progress tracking
class ProgressTracker {
  constructor() {
    this.operations = new Map();
  }

  start(id, name) {
    this.operations.set(id, {
      name,
      startTime: Date.now(),
      status: 'in_progress'
    });
    log(`  [${id}] STARTING: ${name}`, 'magenta');
  }

  complete(id, message) {
    const op = this.operations.get(id);
    if (op) {
      op.status = 'completed';
      op.endTime = Date.now();
      op.duration = op.endTime - op.startTime;
      log(`  [${id}] COMPLETED in ${op.duration}ms: ${message}`, 'green');
    }
  }

  fail(id, error) {
    const op = this.operations.get(id);
    if (op) {
      op.status = 'failed';
      op.endTime = Date.now();
      op.duration = op.endTime - op.startTime;
      op.error = error;
      log(`  [${id}] FAILED after ${op.duration}ms: ${error}`, 'red');
    }
  }

  getStats() {
    const ops = Array.from(this.operations.values());
    return {
      total: ops.length,
      completed: ops.filter(o => o.status === 'completed').length,
      failed: ops.filter(o => o.status === 'failed').length,
      inProgress: ops.filter(o => o.status === 'in_progress').length,
      avgDuration: ops
        .filter(o => o.duration)
        .reduce((sum, o) => sum + o.duration, 0) / (ops.filter(o => o.duration).length || 1)
    };
  }

  printSummary() {
    const stats = this.getStats();
    console.log('\n  Progress Summary:');
    console.log('  ' + '-'.repeat(60));
    console.log(`  Total operations: ${stats.total}`);
    log(`  Completed: ${stats.completed}`, 'green');
    log(`  Failed: ${stats.failed}`, 'red');
    log(`  In progress: ${stats.inProgress}`, 'yellow');
    console.log(`  Avg duration: ${Math.round(stats.avgDuration)}ms`);
  }
}

// HTTP helpers
async function fetchWithTimeout(url, options = {}, timeout = TEST_CONFIG.timeout) {
  const controller = new AbortController();
  const id = setTimeout(() => controller.abort(), timeout);

  try {
    const response = await fetch(url, {
      ...options,
      signal: controller.signal
    });
    clearTimeout(id);
    return response;
  } catch (error) {
    clearTimeout(id);
    throw error;
  }
}

// Storage helpers
function getChapterDir(mangaSlug, chapterNumber) {
  return path.join(__dirname, '../storage/manga', mangaSlug, `chapter_${chapterNumber}`);
}

function cleanupTestChapters() {
  TEST_CONFIG.chapters.forEach(chapterNumber => {
    const chapterDir = getChapterDir(TEST_CONFIG.mangaSlug, chapterNumber);
    if (fs.existsSync(chapterDir)) {
      fs.rmSync(chapterDir, { recursive: true, force: true });
    }
  });
  logInfo('Cleaned up all test chapter directories');
}

// Test assertions with detailed logging
function assertTruthy(value, message) {
  if (!value) {
    logError(`ASSERTION FAILED: ${message}`);
    throw new Error(message);
  }
  logSuccess(message);
}

function assertEqual(actual, expected, message) {
  if (actual !== expected) {
    logError(`ASSERTION FAILED: ${message}\n  Expected: ${expected}\n  Actual: ${actual}`);
    throw new Error(`${message}: expected ${expected}, got ${actual}`);
  }
  logSuccess(message);
}

// Fetch chapter images from API
async function getChapterImages(chapterNumber) {
  const chapterSlug = `${TEST_CONFIG.mangaSlug}-${chapterNumber}`;
  const response = await fetchWithTimeout(
    `${TEST_CONFIG.baseUrl}/api/chapter/${chapterSlug}/images?force=true`
  );

  if (response.status !== 200) {
    throw new Error(`Failed to fetch images for chapter ${chapterNumber}: HTTP ${response.status}`);
  }

  const data = await response.json();

  if (!Array.isArray(data.images) || data.images.length === 0) {
    throw new Error(`No images found for chapter ${chapterNumber}`);
  }

  // Return only first 5 images for faster testing
  return data.images.slice(0, 5);
}

// Download a single chapter
async function downloadChapter(chapterNumber, tracker) {
  const opId = `CH-${chapterNumber}`;
  tracker.start(opId, `Download chapter ${chapterNumber}`);

  try {
    // Fetch images
    const imageUrls = await getChapterImages(chapterNumber);

    // Download to storage
    const result = await storage.downloadChapter(
      TEST_CONFIG.mangaSlug,
      chapterNumber,
      imageUrls
    );

    if (!result.success) {
      throw new Error('Download failed');
    }

    tracker.complete(opId, `Downloaded ${result.downloaded} pages`);
    return { chapterNumber, success: true, result };
  } catch (error) {
    tracker.fail(opId, error.message);
    return { chapterNumber, success: false, error: error.message };
  }
}

// Concurrent download with limited parallelism
async function downloadChaptersConcurrently(chapters, maxConcurrent) {
  const tracker = new ProgressTracker();
  const results = [];

  // Process in batches
  for (let i = 0; i < chapters.length; i += maxConcurrent) {
    const batch = chapters.slice(i, i + maxConcurrent);
    logInfo(`Processing batch: chapters ${batch.join(', ')}`);

    const batchResults = await Promise.all(
      batch.map(chapter => downloadChapter(chapter, tracker))
    );

    results.push(...batchResults);
  }

  tracker.printSummary();
  return results;
}

// Verify no file corruption
function verifyChapterIntegrity(chapterNumber) {
  const chapterDir = getChapterDir(TEST_CONFIG.mangaSlug, chapterNumber);

  if (!fs.existsSync(chapterDir)) {
    throw new Error(`Chapter ${chapterNumber} directory not found`);
  }

  const manifestPath = path.join(chapterDir, 'manifest.json');
  if (!fs.existsSync(manifestPath)) {
    throw new Error(`Chapter ${chapterNumber} manifest not found`);
  }

  const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));

  // Verify all images in manifest exist
  const missingImages = [];
  const corruptedImages = [];

  manifest.images.forEach(img => {
    const imagePath = path.join(chapterDir, img.filename);

    if (!fs.existsSync(imagePath)) {
      missingImages.push(img.filename);
    } else {
      const stats = fs.statSync(imagePath);
      if (stats.size === 0) {
        corruptedImages.push(img.filename);
      }
    }
  });

  return {
    manifest,
    missingImages,
    corruptedImages,
    totalImages: manifest.images.length,
    validImages: manifest.images.length - missingImages.length - corruptedImages.length
  };
}

// Test suite
async function runTests() {
  const results = {
    passed: 0,
    failed: 0,
    tests: []
  };

  function recordTest(name, passed, error = null) {
    results.tests.push({ name, passed, error });
    if (passed) {
      results.passed++;
    } else {
      results.failed++;
    }
  }

  try {
    logSection('CONCURRENT DOWNLOADS INTEGRATION TEST');
    logInfo(`Manga: ${TEST_CONFIG.mangaSlug}`);
    logInfo(`Chapters to test: ${TEST_CONFIG.chapters.join(', ')}`);
    logInfo(`Max concurrent: ${TEST_CONFIG.maxConcurrent}`);

    // Clean up any previous test data
    logSection('SETUP');
    logTest('Cleaning up previous test data');
    cleanupTestChapters();
    recordTest('Cleanup', true);

    // Test 1: Pre-download check
    logSection('TEST 1: Pre-Download Verification');
    logTest('Verifying no test chapters exist before download');
    try {
      let allClean = true;
      TEST_CONFIG.chapters.forEach(chapterNumber => {
        const isDownloaded = storage.isChapterDownloaded(TEST_CONFIG.mangaSlug, chapterNumber);
        if (isDownloaded) {
          logWarning(`Chapter ${chapterNumber} already exists, cleaning up...`);
          storage.deleteChapter(TEST_CONFIG.mangaSlug, chapterNumber);
          allClean = false;
        }
      });

      assertTruthy(allClean, 'All test chapters were clean or cleaned up');
      recordTest('Pre-Download Clean', true);
    } catch (error) {
      logError(`Pre-download check failed: ${error.message}`);
      recordTest('Pre-Download Clean', false, error);
      throw error;
    }

    // Test 2: Concurrent downloads
    logSection('TEST 2: Concurrent Downloads');
    logTest(`Downloading ${TEST_CONFIG.chapters.length} chapters concurrently`);
    try {
      const downloadResults = await downloadChaptersConcurrently(
        TEST_CONFIG.chapters,
        TEST_CONFIG.maxConcurrent
      );

      const successful = downloadResults.filter(r => r.success);
      const failed = downloadResults.filter(r => !r.success);

      logInfo(`Successful downloads: ${successful.length}/${downloadResults.length}`);
      logInfo(`Failed downloads: ${failed.length}/${downloadResults.length}`);

      assertTruthy(
        successful.length === TEST_CONFIG.chapters.length,
        'All chapters downloaded successfully'
      );

      recordTest('Concurrent Downloads', true);
    } catch (error) {
      logError(`Concurrent download failed: ${error.message}`);
      recordTest('Concurrent Downloads', false, error);
      throw error;
    }

    // Test 3: Verify all chapters exist
    logSection('TEST 3: Post-Download Verification');
    logTest('Verifying all chapters exist in storage');
    try {
      let allExist = true;
      const chapterStatus = [];

      TEST_CONFIG.chapters.forEach(chapterNumber => {
        const exists = storage.isChapterDownloaded(TEST_CONFIG.mangaSlug, chapterNumber);
        chapterStatus.push({ chapter: chapterNumber, exists });
        if (!exists) {
          allExist = false;
        }
      });

      chapterStatus.forEach(status => {
        logInfo(`Chapter ${status.chapter}: ${status.exists ? '✓' : '✗'}`);
      });

      assertTruthy(allExist, 'All chapters exist in storage');
      recordTest('Chapters Exist', true);
    } catch (error) {
      logError(`Post-download verification failed: ${error.message}`);
      recordTest('Chapters Exist', false, error);
      throw error;
    }

    // Test 4: Verify no corruption
    logSection('TEST 4: Integrity Check');
    logTest('Verifying no file corruption across all chapters');
    try {
      let totalCorrupted = 0;
      let totalMissing = 0;
      let totalValid = 0;

      const integrityReports = [];

      TEST_CONFIG.chapters.forEach(chapterNumber => {
        const report = verifyChapterIntegrity(chapterNumber);
        integrityReports.push({ chapter: chapterNumber, ...report });

        totalCorrupted += report.corruptedImages.length;
        totalMissing += report.missingImages.length;
        totalValid += report.validImages;
      });

      // Print detailed report
      console.log('\n  Integrity Report:');
      console.log('  ' + '-'.repeat(60));
      integrityReports.forEach(report => {
        console.log(`\n  Chapter ${report.chapter}:`);
        console.log(`    Total images: ${report.totalImages}`);
        log(`    Valid: ${report.validValid}`, 'green');
        if (report.missingImages.length > 0) {
          log(`    Missing: ${report.missingImages.length}`, 'red');
        }
        if (report.corruptedImages.length > 0) {
          log(`    Corrupted: ${report.corruptedImages.length}`, 'red');
        }
      });

      console.log('\n  Summary:');
      console.log(`    Total valid images: ${totalValid}`);
      console.log(`    Total missing: ${totalMissing}`);
      console.log(`    Total corrupted: ${totalCorrupted}`);

      assertEqual(totalCorrupted, 0, 'No corrupted images');
      assertEqual(totalMissing, 0, 'No missing images');
      assertTruthy(totalValid > 0, 'Valid images exist');

      recordTest('Integrity Check', true);
    } catch (error) {
      logError(`Integrity check failed: ${error.message}`);
      recordTest('Integrity Check', false, error);
      throw error;
    }

    // Test 5: Verify manifests are independent
    logSection('TEST 5: Manifest Independence');
    logTest('Verifying each chapter has independent manifest');
    try {
      const manifests = [];

      TEST_CONFIG.chapters.forEach(chapterNumber => {
        const manifest = storage.getChapterManifest(TEST_CONFIG.mangaSlug, chapterNumber);
        assertTruthy(manifest !== null, `Manifest exists for chapter ${chapterNumber}`);
        manifests.push(manifest);
      });

      // Verify no manifest references another chapter
      let allIndependent = true;
      manifests.forEach((manifest, index) => {
        if (manifest.chapterNumber !== TEST_CONFIG.chapters[index]) {
          logError(`Manifest corruption: chapter ${manifest.chapterNumber} in wrong entry`);
          allIndependent = false;
        }
      });

      assertTruthy(allIndependent, 'All manifests are independent');
      recordTest('Manifest Independence', true);
    } catch (error) {
      logError(`Manifest independence check failed: ${error.message}`);
      recordTest('Manifest Independence', false, error);
      throw error;
    }

    // Test 6: Verify storage stats
    logSection('TEST 6: Storage Statistics');
    logTest('Verifying storage statistics are accurate');
    try {
      const stats = storage.getStorageStats();
      const mangaStats = stats.mangaDetails.find(
        m => m.mangaSlug === TEST_CONFIG.mangaSlug
      );

      assertTruthy(mangaStats !== undefined, 'Manga exists in stats');
      assertEqual(mangaStats.chapters, TEST_CONFIG.chapters.length, 'Chapter count matches');

      logInfo(`Storage stats show ${mangaStats.chapters} chapters`);
      logInfo(`Total size: ${mangaStats.totalSizeMB} MB`);

      recordTest('Storage Stats', true);
    } catch (error) {
      logError(`Storage stats verification failed: ${error.message}`);
      recordTest('Storage Stats', false, error);
      throw error;
    }

    // Test 7: List downloaded chapters
    logSection('TEST 7: List Downloaded Chapters');
    logTest('Verifying list function returns all test chapters');
    try {
      const chapters = storage.listDownloadedChapters(TEST_CONFIG.mangaSlug);

      logInfo(`Found ${chapters.length} chapters in storage`);

      const foundChapters = chapters.map(ch => ch.chapterNumber);
      const missingChapters = TEST_CONFIG.chapters.filter(
        ch => !foundChapters.includes(ch)
      );

      assertEqual(missingChapters.length, 0, 'All test chapters are in list');

      // Verify each chapter has valid metadata
      chapters.forEach(ch => {
        assertTruthy(ch.downloadDate, 'Chapter has download date');
        assertTruthy(ch.totalPages > 0, 'Chapter has pages');
        assertTruthy(ch.totalSize > 0, 'Chapter has size');
      });

      recordTest('List Chapters', true);
    } catch (error) {
      logError(`List chapters verification failed: ${error.message}`);
      recordTest('List Chapters', false, error);
      throw error;
    }

    // Test 8: Concurrent delete
    logSection('TEST 8: Concurrent Deletion');
    logTest('Deleting all test chapters concurrently');
    try {
      const deleteTracker = new ProgressTracker();

      const deletePromises = TEST_CONFIG.chapters.map(async chapterNumber => {
        const opId = `DEL-${chapterNumber}`;
        deleteTracker.start(opId, `Delete chapter ${chapterNumber}`);

        try {
          const result = storage.deleteChapter(TEST_CONFIG.mangaSlug, chapterNumber);
          if (result.success) {
            deleteTracker.complete(opId, 'Deleted successfully');
            return { chapter: chapterNumber, success: true };
          } else {
            throw new Error(result.error);
          }
        } catch (error) {
          deleteTracker.fail(opId, error.message);
          return { chapter: chapterNumber, success: false, error: error.message };
        }
      });

      const deleteResults = await Promise.all(deletePromises);

      deleteTracker.printSummary();

      const successfulDeletes = deleteResults.filter(r => r.success).length;
      assertEqual(successfulDeletes, TEST_CONFIG.chapters.length, 'All chapters deleted');

      recordTest('Concurrent Delete', true);
    } catch (error) {
      logError(`Concurrent delete failed: ${error.message}`);
      recordTest('Concurrent Delete', false, error);
      throw error;
    }

    // Test 9: Verify complete cleanup
    logSection('TEST 9: Verify Complete Cleanup');
    logTest('Verifying all chapters and files are removed');
    try {
      let allClean = true;
      const remainingChapters = [];

      TEST_CONFIG.chapters.forEach(chapterNumber => {
        const exists = storage.isChapterDownloaded(TEST_CONFIG.mangaSlug, chapterNumber);
        if (exists) {
          remainingChapters.push(chapterNumber);
          allClean = false;
        }

        // Also check directory doesn't exist
        const chapterDir = getChapterDir(TEST_CONFIG.mangaSlug, chapterNumber);
        if (fs.existsSync(chapterDir)) {
          logWarning(`Directory still exists for chapter ${chapterNumber}`);
          allClean = false;
        }
      });

      if (remainingChapters.length > 0) {
        logError(`Remaining chapters: ${remainingChapters.join(', ')}`);
      }

      assertTruthy(allClean, 'All chapters completely removed');

      // Final stats check
      const finalStats = storage.getStorageStats();
      const mangaStats = finalStats.mangaDetails.find(
        m => m.mangaSlug === TEST_CONFIG.mangaSlug
      );

      if (mangaStats) {
        assertEqual(mangaStats.chapters, 0, 'Manga has 0 chapters in stats');
      }

      recordTest('Complete Cleanup', true);
    } catch (error) {
      logError(`Cleanup verification failed: ${error.message}`);
      recordTest('Complete Cleanup', false, error);
      throw error;
    }

    // Test 10: No race conditions detected
    logSection('TEST 10: Race Condition Check');
    logTest('Analyzing operations for race conditions');
    try {
      // If we got here without errors, no obvious race conditions occurred
      // All operations completed successfully with independent data
      logInfo('No race conditions detected in concurrent operations');
      logInfo('All manifests were independent');
      logInfo('All files were properly created and managed');
      logInfo('No corrupted or missing data detected');

      recordTest('Race Condition Check', true);
    } catch (error) {
      logError(`Race condition check failed: ${error.message}`);
      recordTest('Race Condition Check', false, error);
      throw error;
    }

  } catch (error) {
    logError(`\n❌ Test suite failed: ${error.message}`);
    console.error(error.stack);
  }

  // Print results summary
  logSection('TEST RESULTS SUMMARY');
  console.log(`\nTotal Tests: ${results.tests.length}`);
  log(`Passed: ${results.passed}`, 'green');
  log(`Failed: ${results.failed}`, 'red');

  if (results.failed > 0) {
    console.log('\nFailed Tests:');
    results.tests
      .filter(t => !t.passed)
      .forEach(t => {
        logError(`  - ${t.name}`);
        if (t.error) {
          console.log(`    Error: ${t.error.message}`);
        }
      });
  }

  console.log('\n' + '='.repeat(70));

  if (results.failed === 0) {
    log('🎉 ALL TESTS PASSED!', 'green');
    log('\n✓ Concurrent downloads work correctly', 'green');
    log('✓ No race conditions detected', 'green');
    log('✓ No file corruption found', 'green');
    log('✓ Storage handles concurrent access properly', 'green');
    process.exit(0);
  } else {
    log('❌ SOME TESTS FAILED', 'red');
    process.exit(1);
  }
}

// Run tests
console.log('\n🚀 Starting Concurrent Downloads Integration Tests...\n');
runTests().catch(error => {
  console.error('\n❌ Fatal error:', error);
  process.exit(1);
});