/** * Integration Test: Concurrent Downloads * Tests downloading multiple chapters in parallel to verify: * - No race conditions * - No file corruption * - Proper concurrent access to storage * - Correct storage statistics * - Independent chapter management */ import assert from 'assert'; import fs from 'fs'; import path from 'path'; import { fileURLToPath } from 'url'; import storage from './storage.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); // Test configuration const TEST_CONFIG = { mangaSlug: 'one-piece_1695365223767', chapters: [787, 788, 789, 790, 791], // Test with 5 chapters baseUrl: 'http://localhost:3001', timeout: 180000, // 3 minutes for concurrent downloads maxConcurrent: 3 // Limit concurrent downloads to avoid overwhelming the server }; // Color codes for terminal output const colors = { reset: '\x1b[0m', bright: '\x1b[1m', green: '\x1b[32m', red: '\x1b[31m', yellow: '\x1b[33m', blue: '\x1b[34m', cyan: '\x1b[36m', magenta: '\x1b[35m' }; function log(message, color = 'reset') { console.log(`${colors[color]}${message}${colors.reset}`); } function logSection(title) { console.log('\n' + '='.repeat(70)); log(title, 'bright'); console.log('='.repeat(70)); } function logTest(testName) { log(`\n▶ ${testName}`, 'cyan'); } function logSuccess(message) { log(`✓ ${message}`, 'green'); } function logError(message) { log(`✗ ${message}`, 'red'); } function logInfo(message) { log(` ℹ ${message}`, 'blue'); } function logWarning(message) { log(`⚠ ${message}`, 'yellow'); } // Progress tracking class ProgressTracker { constructor() { this.operations = new Map(); } start(id, name) { this.operations.set(id, { name, startTime: Date.now(), status: 'in_progress' }); log(` [${id}] STARTING: ${name}`, 'magenta'); } complete(id, message) { const op = this.operations.get(id); if (op) { op.status = 'completed'; op.endTime = Date.now(); op.duration = op.endTime - op.startTime; log(` [${id}] COMPLETED in ${op.duration}ms: ${message}`, 'green'); } } fail(id, error) { const op = this.operations.get(id); if (op) { op.status = 'failed'; op.endTime = Date.now(); op.duration = op.endTime - op.startTime; op.error = error; log(` [${id}] FAILED after ${op.duration}ms: ${error}`, 'red'); } } getStats() { const ops = Array.from(this.operations.values()); return { total: ops.length, completed: ops.filter(o => o.status === 'completed').length, failed: ops.filter(o => o.status === 'failed').length, inProgress: ops.filter(o => o.status === 'in_progress').length, avgDuration: ops .filter(o => o.duration) .reduce((sum, o) => sum + o.duration, 0) / (ops.filter(o => o.duration).length || 1) }; } printSummary() { const stats = this.getStats(); console.log('\n Progress Summary:'); console.log(' ' + '-'.repeat(60)); console.log(` Total operations: ${stats.total}`); log(` Completed: ${stats.completed}`, 'green'); log(` Failed: ${stats.failed}`, 'red'); log(` In progress: ${stats.inProgress}`, 'yellow'); console.log(` Avg duration: ${Math.round(stats.avgDuration)}ms`); } } // HTTP helpers async function fetchWithTimeout(url, options = {}, timeout = TEST_CONFIG.timeout) { const controller = new AbortController(); const id = setTimeout(() => controller.abort(), timeout); try { const response = await fetch(url, { ...options, signal: controller.signal }); clearTimeout(id); return response; } catch (error) { clearTimeout(id); throw error; } } // Storage helpers function getChapterDir(mangaSlug, chapterNumber) { return path.join(__dirname, '../storage/manga', mangaSlug, `chapter_${chapterNumber}`); } function cleanupTestChapters() { TEST_CONFIG.chapters.forEach(chapterNumber => { const chapterDir = getChapterDir(TEST_CONFIG.mangaSlug, chapterNumber); if (fs.existsSync(chapterDir)) { fs.rmSync(chapterDir, { recursive: true, force: true }); } }); logInfo('Cleaned up all test chapter directories'); } // Test assertions with detailed logging function assertTruthy(value, message) { if (!value) { logError(`ASSERTION FAILED: ${message}`); throw new Error(message); } logSuccess(message); } function assertEqual(actual, expected, message) { if (actual !== expected) { logError(`ASSERTION FAILED: ${message}\n Expected: ${expected}\n Actual: ${actual}`); throw new Error(`${message}: expected ${expected}, got ${actual}`); } logSuccess(message); } // Fetch chapter images from API async function getChapterImages(chapterNumber) { const chapterSlug = `${TEST_CONFIG.mangaSlug}-${chapterNumber}`; const response = await fetchWithTimeout( `${TEST_CONFIG.baseUrl}/api/chapter/${chapterSlug}/images?force=true` ); if (response.status !== 200) { throw new Error(`Failed to fetch images for chapter ${chapterNumber}: HTTP ${response.status}`); } const data = await response.json(); if (!Array.isArray(data.images) || data.images.length === 0) { throw new Error(`No images found for chapter ${chapterNumber}`); } // Return only first 5 images for faster testing return data.images.slice(0, 5); } // Download a single chapter async function downloadChapter(chapterNumber, tracker) { const opId = `CH-${chapterNumber}`; tracker.start(opId, `Download chapter ${chapterNumber}`); try { // Fetch images const imageUrls = await getChapterImages(chapterNumber); // Download to storage const result = await storage.downloadChapter( TEST_CONFIG.mangaSlug, chapterNumber, imageUrls ); if (!result.success) { throw new Error('Download failed'); } tracker.complete(opId, `Downloaded ${result.downloaded} pages`); return { chapterNumber, success: true, result }; } catch (error) { tracker.fail(opId, error.message); return { chapterNumber, success: false, error: error.message }; } } // Concurrent download with limited parallelism async function downloadChaptersConcurrently(chapters, maxConcurrent) { const tracker = new ProgressTracker(); const results = []; // Process in batches for (let i = 0; i < chapters.length; i += maxConcurrent) { const batch = chapters.slice(i, i + maxConcurrent); logInfo(`Processing batch: chapters ${batch.join(', ')}`); const batchResults = await Promise.all( batch.map(chapter => downloadChapter(chapter, tracker)) ); results.push(...batchResults); } tracker.printSummary(); return results; } // Verify no file corruption function verifyChapterIntegrity(chapterNumber) { const chapterDir = getChapterDir(TEST_CONFIG.mangaSlug, chapterNumber); if (!fs.existsSync(chapterDir)) { throw new Error(`Chapter ${chapterNumber} directory not found`); } const manifestPath = path.join(chapterDir, 'manifest.json'); if (!fs.existsSync(manifestPath)) { throw new Error(`Chapter ${chapterNumber} manifest not found`); } const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')); // Verify all images in manifest exist const missingImages = []; const corruptedImages = []; manifest.images.forEach(img => { const imagePath = path.join(chapterDir, img.filename); if (!fs.existsSync(imagePath)) { missingImages.push(img.filename); } else { const stats = fs.statSync(imagePath); if (stats.size === 0) { corruptedImages.push(img.filename); } } }); return { manifest, missingImages, corruptedImages, totalImages: manifest.images.length, validImages: manifest.images.length - missingImages.length - corruptedImages.length }; } // Test suite async function runTests() { const results = { passed: 0, failed: 0, tests: [] }; function recordTest(name, passed, error = null) { results.tests.push({ name, passed, error }); if (passed) { results.passed++; } else { results.failed++; } } try { logSection('CONCURRENT DOWNLOADS INTEGRATION TEST'); logInfo(`Manga: ${TEST_CONFIG.mangaSlug}`); logInfo(`Chapters to test: ${TEST_CONFIG.chapters.join(', ')}`); logInfo(`Max concurrent: ${TEST_CONFIG.maxConcurrent}`); // Clean up any previous test data logSection('SETUP'); logTest('Cleaning up previous test data'); cleanupTestChapters(); recordTest('Cleanup', true); // Test 1: Pre-download check logSection('TEST 1: Pre-Download Verification'); logTest('Verifying no test chapters exist before download'); try { let allClean = true; TEST_CONFIG.chapters.forEach(chapterNumber => { const isDownloaded = storage.isChapterDownloaded(TEST_CONFIG.mangaSlug, chapterNumber); if (isDownloaded) { logWarning(`Chapter ${chapterNumber} already exists, cleaning up...`); storage.deleteChapter(TEST_CONFIG.mangaSlug, chapterNumber); allClean = false; } }); assertTruthy(allClean, 'All test chapters were clean or cleaned up'); recordTest('Pre-Download Clean', true); } catch (error) { logError(`Pre-download check failed: ${error.message}`); recordTest('Pre-Download Clean', false, error); throw error; } // Test 2: Concurrent downloads logSection('TEST 2: Concurrent Downloads'); logTest(`Downloading ${TEST_CONFIG.chapters.length} chapters concurrently`); try { const downloadResults = await downloadChaptersConcurrently( TEST_CONFIG.chapters, TEST_CONFIG.maxConcurrent ); const successful = downloadResults.filter(r => r.success); const failed = downloadResults.filter(r => !r.success); logInfo(`Successful downloads: ${successful.length}/${downloadResults.length}`); logInfo(`Failed downloads: ${failed.length}/${downloadResults.length}`); assertTruthy( successful.length === TEST_CONFIG.chapters.length, 'All chapters downloaded successfully' ); recordTest('Concurrent Downloads', true); } catch (error) { logError(`Concurrent download failed: ${error.message}`); recordTest('Concurrent Downloads', false, error); throw error; } // Test 3: Verify all chapters exist logSection('TEST 3: Post-Download Verification'); logTest('Verifying all chapters exist in storage'); try { let allExist = true; const chapterStatus = []; TEST_CONFIG.chapters.forEach(chapterNumber => { const exists = storage.isChapterDownloaded(TEST_CONFIG.mangaSlug, chapterNumber); chapterStatus.push({ chapter: chapterNumber, exists }); if (!exists) { allExist = false; } }); chapterStatus.forEach(status => { logInfo(`Chapter ${status.chapter}: ${status.exists ? '✓' : '✗'}`); }); assertTruthy(allExist, 'All chapters exist in storage'); recordTest('Chapters Exist', true); } catch (error) { logError(`Post-download verification failed: ${error.message}`); recordTest('Chapters Exist', false, error); throw error; } // Test 4: Verify no corruption logSection('TEST 4: Integrity Check'); logTest('Verifying no file corruption across all chapters'); try { let totalCorrupted = 0; let totalMissing = 0; let totalValid = 0; const integrityReports = []; TEST_CONFIG.chapters.forEach(chapterNumber => { const report = verifyChapterIntegrity(chapterNumber); integrityReports.push({ chapter: chapterNumber, ...report }); totalCorrupted += report.corruptedImages.length; totalMissing += report.missingImages.length; totalValid += report.validImages; }); // Print detailed report console.log('\n Integrity Report:'); console.log(' ' + '-'.repeat(60)); integrityReports.forEach(report => { console.log(`\n Chapter ${report.chapter}:`); console.log(` Total images: ${report.totalImages}`); log(` Valid: ${report.validValid}`, 'green'); if (report.missingImages.length > 0) { log(` Missing: ${report.missingImages.length}`, 'red'); } if (report.corruptedImages.length > 0) { log(` Corrupted: ${report.corruptedImages.length}`, 'red'); } }); console.log('\n Summary:'); console.log(` Total valid images: ${totalValid}`); console.log(` Total missing: ${totalMissing}`); console.log(` Total corrupted: ${totalCorrupted}`); assertEqual(totalCorrupted, 0, 'No corrupted images'); assertEqual(totalMissing, 0, 'No missing images'); assertTruthy(totalValid > 0, 'Valid images exist'); recordTest('Integrity Check', true); } catch (error) { logError(`Integrity check failed: ${error.message}`); recordTest('Integrity Check', false, error); throw error; } // Test 5: Verify manifests are independent logSection('TEST 5: Manifest Independence'); logTest('Verifying each chapter has independent manifest'); try { const manifests = []; TEST_CONFIG.chapters.forEach(chapterNumber => { const manifest = storage.getChapterManifest(TEST_CONFIG.mangaSlug, chapterNumber); assertTruthy(manifest !== null, `Manifest exists for chapter ${chapterNumber}`); manifests.push(manifest); }); // Verify no manifest references another chapter let allIndependent = true; manifests.forEach((manifest, index) => { if (manifest.chapterNumber !== TEST_CONFIG.chapters[index]) { logError(`Manifest corruption: chapter ${manifest.chapterNumber} in wrong entry`); allIndependent = false; } }); assertTruthy(allIndependent, 'All manifests are independent'); recordTest('Manifest Independence', true); } catch (error) { logError(`Manifest independence check failed: ${error.message}`); recordTest('Manifest Independence', false, error); throw error; } // Test 6: Verify storage stats logSection('TEST 6: Storage Statistics'); logTest('Verifying storage statistics are accurate'); try { const stats = storage.getStorageStats(); const mangaStats = stats.mangaDetails.find( m => m.mangaSlug === TEST_CONFIG.mangaSlug ); assertTruthy(mangaStats !== undefined, 'Manga exists in stats'); assertEqual(mangaStats.chapters, TEST_CONFIG.chapters.length, 'Chapter count matches'); logInfo(`Storage stats show ${mangaStats.chapters} chapters`); logInfo(`Total size: ${mangaStats.totalSizeMB} MB`); recordTest('Storage Stats', true); } catch (error) { logError(`Storage stats verification failed: ${error.message}`); recordTest('Storage Stats', false, error); throw error; } // Test 7: List downloaded chapters logSection('TEST 7: List Downloaded Chapters'); logTest('Verifying list function returns all test chapters'); try { const chapters = storage.listDownloadedChapters(TEST_CONFIG.mangaSlug); logInfo(`Found ${chapters.length} chapters in storage`); const foundChapters = chapters.map(ch => ch.chapterNumber); const missingChapters = TEST_CONFIG.chapters.filter( ch => !foundChapters.includes(ch) ); assertEqual(missingChapters.length, 0, 'All test chapters are in list'); // Verify each chapter has valid metadata chapters.forEach(ch => { assertTruthy(ch.downloadDate, 'Chapter has download date'); assertTruthy(ch.totalPages > 0, 'Chapter has pages'); assertTruthy(ch.totalSize > 0, 'Chapter has size'); }); recordTest('List Chapters', true); } catch (error) { logError(`List chapters verification failed: ${error.message}`); recordTest('List Chapters', false, error); throw error; } // Test 8: Concurrent delete logSection('TEST 8: Concurrent Deletion'); logTest('Deleting all test chapters concurrently'); try { const deleteTracker = new ProgressTracker(); const deletePromises = TEST_CONFIG.chapters.map(async chapterNumber => { const opId = `DEL-${chapterNumber}`; deleteTracker.start(opId, `Delete chapter ${chapterNumber}`); try { const result = storage.deleteChapter(TEST_CONFIG.mangaSlug, chapterNumber); if (result.success) { deleteTracker.complete(opId, 'Deleted successfully'); return { chapter: chapterNumber, success: true }; } else { throw new Error(result.error); } } catch (error) { deleteTracker.fail(opId, error.message); return { chapter: chapterNumber, success: false, error: error.message }; } }); const deleteResults = await Promise.all(deletePromises); deleteTracker.printSummary(); const successfulDeletes = deleteResults.filter(r => r.success).length; assertEqual(successfulDeletes, TEST_CONFIG.chapters.length, 'All chapters deleted'); recordTest('Concurrent Delete', true); } catch (error) { logError(`Concurrent delete failed: ${error.message}`); recordTest('Concurrent Delete', false, error); throw error; } // Test 9: Verify complete cleanup logSection('TEST 9: Verify Complete Cleanup'); logTest('Verifying all chapters and files are removed'); try { let allClean = true; const remainingChapters = []; TEST_CONFIG.chapters.forEach(chapterNumber => { const exists = storage.isChapterDownloaded(TEST_CONFIG.mangaSlug, chapterNumber); if (exists) { remainingChapters.push(chapterNumber); allClean = false; } // Also check directory doesn't exist const chapterDir = getChapterDir(TEST_CONFIG.mangaSlug, chapterNumber); if (fs.existsSync(chapterDir)) { logWarning(`Directory still exists for chapter ${chapterNumber}`); allClean = false; } }); if (remainingChapters.length > 0) { logError(`Remaining chapters: ${remainingChapters.join(', ')}`); } assertTruthy(allClean, 'All chapters completely removed'); // Final stats check const finalStats = storage.getStorageStats(); const mangaStats = finalStats.mangaDetails.find( m => m.mangaSlug === TEST_CONFIG.mangaSlug ); if (mangaStats) { assertEqual(mangaStats.chapters, 0, 'Manga has 0 chapters in stats'); } recordTest('Complete Cleanup', true); } catch (error) { logError(`Cleanup verification failed: ${error.message}`); recordTest('Complete Cleanup', false, error); throw error; } // Test 10: No race conditions detected logSection('TEST 10: Race Condition Check'); logTest('Analyzing operations for race conditions'); try { // If we got here without errors, no obvious race conditions occurred // All operations completed successfully with independent data logInfo('No race conditions detected in concurrent operations'); logInfo('All manifests were independent'); logInfo('All files were properly created and managed'); logInfo('No corrupted or missing data detected'); recordTest('Race Condition Check', true); } catch (error) { logError(`Race condition check failed: ${error.message}`); recordTest('Race Condition Check', false, error); throw error; } } catch (error) { logError(`\n❌ Test suite failed: ${error.message}`); console.error(error.stack); } // Print results summary logSection('TEST RESULTS SUMMARY'); console.log(`\nTotal Tests: ${results.tests.length}`); log(`Passed: ${results.passed}`, 'green'); log(`Failed: ${results.failed}`, 'red'); if (results.failed > 0) { console.log('\nFailed Tests:'); results.tests .filter(t => !t.passed) .forEach(t => { logError(` - ${t.name}`); if (t.error) { console.log(` Error: ${t.error.message}`); } }); } console.log('\n' + '='.repeat(70)); if (results.failed === 0) { log('🎉 ALL TESTS PASSED!', 'green'); log('\n✓ Concurrent downloads work correctly', 'green'); log('✓ No race conditions detected', 'green'); log('✓ No file corruption found', 'green'); log('✓ Storage handles concurrent access properly', 'green'); process.exit(0); } else { log('❌ SOME TESTS FAILED', 'red'); process.exit(1); } } // Run tests console.log('\n🚀 Starting Concurrent Downloads Integration Tests...\n'); runTests().catch(error => { console.error('\n❌ Fatal error:', error); process.exit(1); });