ableton-mcp-ai/mcp_server/integration.py

"""
integration.py - Main integration coordinator for AbletonMCP_AI.

This module provides the SeniorArchitectureCoordinator class that wires together
all components: metadata store, hybrid extractor, arrangement recorder, and live bridge.

Usage:
    from AbletonMCP_AI.mcp_server.integration import (
        SeniorArchitectureCoordinator,
        create_coordinator,
        get_coordinator_singleton
    )
    
    # Create and initialize coordinator
    coord = create_coordinator(song, connection)
    
    # Use high-level operations
    result = coord.build_arrangement_timeline(sections, genre="reggaeton")
    
    # Check system status
    status = coord.get_status()
"""

import os
import json
import logging
from typing import Dict, List, Any, Optional, Callable, Tuple
from pathlib import Path
from dataclasses import dataclass, field

# Configure logging
logger = logging.getLogger("IntegrationCoordinator")

# Import engine components with graceful fallback
try:
    from AbletonMCP_AI.mcp_server.engines.metadata_store import SampleMetadataStore, SampleFeatures
    METADATA_STORE_AVAILABLE = True
except ImportError:
    METADATA_STORE_AVAILABLE = False
    logger.warning("SampleMetadataStore not available")
    SampleMetadataStore = None
    SampleFeatures = None

try:
    from AbletonMCP_AI.mcp_server.engines.abstract_analyzer import (
        HybridExtractor, DatabaseExtractor, LibrosaExtractor, FeatureExtractor
    )
    ABSTRACT_ANALYZER_AVAILABLE = True
except ImportError:
    ABSTRACT_ANALYZER_AVAILABLE = False
    logger.warning("Abstract analyzer not available")
    HybridExtractor = None
    DatabaseExtractor = None
    LibrosaExtractor = None
    FeatureExtractor = None

try:
    from AbletonMCP_AI.mcp_server.engines.arrangement_recorder import (
        ArrangementRecorder, RecordingConfig, RecordingState
    )
    ARRANGEMENT_RECORDER_AVAILABLE = True
except ImportError:
    ARRANGEMENT_RECORDER_AVAILABLE = False
    logger.warning("ArrangementRecorder not available")
    ArrangementRecorder = None
    RecordingConfig = None
    RecordingState = None

try:
    from AbletonMCP_AI.mcp_server.engines.live_bridge import AbletonLiveBridge
    LIVE_BRIDGE_AVAILABLE = True
except ImportError:
    LIVE_BRIDGE_AVAILABLE = False
    logger.warning("AbletonLiveBridge not available")
    AbletonLiveBridge = None

try:
    from AbletonMCP_AI.mcp_server.engines.mixing_engine import (
        MixingEngine, MixConfiguration, BusType, ReturnEffect,
        get_mixing_engine, apply_send_preset, create_standard_buses
    )
    MIXING_ENGINE_AVAILABLE = True
except ImportError:
    MIXING_ENGINE_AVAILABLE = False
    logger.warning("MixingEngine not available")
    MixingEngine = None
    MixConfiguration = None

try:
    from AbletonMCP_AI.mcp_server.engines.sample_selector import get_selector
    SAMPLE_SELECTOR_AVAILABLE = True
except ImportError:
    SAMPLE_SELECTOR_AVAILABLE = False
    logger.warning("SampleSelector not available")
    get_selector = None


@dataclass
class CoordinatorResult:
    """Standard result structure for coordinator operations."""
    success: bool
    message: str
    data: Dict[str, Any] = field(default_factory=dict)
    operation: str = ""
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for JSON serialization."""
        return {
            "success": self.success,
            "message": self.message,
            "data": self.data,
            "operation": self.operation
        }


class SeniorArchitectureCoordinator:
    """
    Coordinates all senior architecture components.
    
    Responsibilities:
    - Initialize metadata store, hybrid extractor, arrangement recorder, live bridge
    - Manage configuration based on available dependencies
    - Provide unified API for all operations
    - Handle graceful degradation with clear error messages
    
    The coordinator follows a lazy initialization pattern where components
    are only created when first needed, allowing the system to start even
    if some dependencies are missing.
    
    Example:
        coord = SeniorArchitectureCoordinator(song, connection)
        status = coord.initialize()
        
        # Build arrangement
        result = coord.build_arrangement_timeline(
            sections=[{"type": "intro", "bars": 8}],
            genre="reggaeton",
            tempo=95
        )
    """
    
    def __init__(self, song, mcp_connection, db_path: Optional[str] = None):
        """
        Initialize the coordinator.
        
        Args:
            song: Ableton Live Song object
            mcp_connection: MCP TCP connection for sending commands
            db_path: Optional path to metadata database
        """
        self.song = song
        self.connection = mcp_connection
        self.db_path = db_path or self._default_db_path()
        
        # Components (initialized lazily)
        self._metadata_store: Optional[SampleMetadataStore] = None
        self._hybrid_extractor: Optional[Any] = None
        self._arrangement_recorder: Optional[ArrangementRecorder] = None
        self._live_bridge: Optional[AbletonLiveBridge] = None
        self._mixing_engine: Optional[MixingEngine] = None
        
        # Configuration
        self._capabilities: Optional[Dict[str, Any]] = None
        self._extraction_mode: Optional[str] = None
        self._initialized: bool = False
        
        logger.info("SeniorArchitectureCoordinator created")
    
    def _default_db_path(self) -> str:
        """Get default database path."""
        base_path = Path(r"C:\ProgramData\Ableton\Live 12 Suite\Resources\MIDI Remote Scripts\libreria\reggaeton")
        return str(base_path / ".sample_metadata.db")
    
    def initialize(self) -> Dict[str, Any]:
        """
        Initialize all components in dependency order.
        
        Initialization sequence:
        1. Detect capabilities
        2. Initialize metadata store (always works if available)
        3. Initialize hybrid extractor based on capabilities
        4. Initialize arrangement recorder
        5. Initialize live bridge
        
        Returns:
            Status dictionary with initialization results
        """
        results = {
            "initialized": False,
            "components": {},
            "errors": []
        }
        
        try:
            # 1. Detect capabilities
            self._capabilities = self._detect_capabilities()
            results["capabilities"] = self._capabilities
            logger.info(f"Detected capabilities: {self._capabilities}")
            
            # 2. Initialize metadata store (always works if sqlite3 available)
            if METADATA_STORE_AVAILABLE:
                try:
                    self._metadata_store = SampleMetadataStore(self.db_path)
                    self._metadata_store.init_database()
                    results["components"]["metadata_store"] = True
                    logger.info("Metadata store initialized")
                except Exception as e:
                    results["components"]["metadata_store"] = False
                    results["errors"].append(f"Metadata store: {str(e)}")
                    logger.error(f"Failed to initialize metadata store: {e}")
            else:
                results["components"]["metadata_store"] = False
                results["errors"].append("Metadata store module not available")
            
            # 3. Initialize hybrid extractor based on capabilities
            if ABSTRACT_ANALYZER_AVAILABLE:
                try:
                    if self._capabilities.get('numpy') and self._capabilities.get('librosa'):
                        # Full hybrid mode with librosa
                        if self._metadata_store:
                            db_extractor = DatabaseExtractor(self._metadata_store)
                        else:
                            db_extractor = None
                        
                        librosa_extractor = LibrosaExtractor()
                        self._hybrid_extractor = HybridExtractor(
                            database_extractor=db_extractor,
                            librosa_extractor=librosa_extractor
                        )
                        self._extraction_mode = "full"
                        logger.info("Hybrid extractor initialized in full mode")
                    else:
                        # Database-only mode
                        if self._metadata_store:
                            self._hybrid_extractor = DatabaseExtractor(self._metadata_store)
                            self._extraction_mode = "database_only"
                            logger.info("Extractor initialized in database-only mode")
                        else:
                            self._hybrid_extractor = None
                            self._extraction_mode = "unavailable"
                            logger.warning("No extractor available - metadata store missing")
                    
                    results["components"]["hybrid_extractor"] = self._hybrid_extractor is not None
                    results["extraction_mode"] = self._extraction_mode
                except Exception as e:
                    results["components"]["hybrid_extractor"] = False
                    results["errors"].append(f"Hybrid extractor: {str(e)}")
                    logger.error(f"Failed to initialize hybrid extractor: {e}")
            else:
                results["components"]["hybrid_extractor"] = False
                results["errors"].append("Abstract analyzer module not available")
            
            # 4. Initialize arrangement recorder
            if ARRANGEMENT_RECORDER_AVAILABLE and self.song and self.connection:
                try:
                    self._arrangement_recorder = ArrangementRecorder(
                        song=self.song,
                        ableton_connection=self.connection
                    )
                    results["components"]["arrangement_recorder"] = True
                    logger.info("Arrangement recorder initialized")
                except Exception as e:
                    results["components"]["arrangement_recorder"] = False
                    results["errors"].append(f"Arrangement recorder: {str(e)}")
                    logger.error(f"Failed to initialize arrangement recorder: {e}")
            else:
                results["components"]["arrangement_recorder"] = False
                if not ARRANGEMENT_RECORDER_AVAILABLE:
                    results["errors"].append("Arrangement recorder module not available")
            
            # 5. Initialize live bridge
            if LIVE_BRIDGE_AVAILABLE and self.song and self.connection:
                try:
                    self._live_bridge = AbletonLiveBridge(
                        song=self.song,
                        mcp_connection=self.connection
                    )
                    results["components"]["live_bridge"] = True
                    logger.info("Live bridge initialized")
                except Exception as e:
                    results["components"]["live_bridge"] = False
                    results["errors"].append(f"Live bridge: {str(e)}")
                    logger.error(f"Failed to initialize live bridge: {e}")
            else:
                results["components"]["live_bridge"] = False
                if not LIVE_BRIDGE_AVAILABLE:
                    results["errors"].append("Live bridge module not available")
            
            # 6. Initialize mixing engine (optional)
            if MIXING_ENGINE_AVAILABLE:
                try:
                    self._mixing_engine = get_mixing_engine(self.song)
                    results["components"]["mixing_engine"] = True
                    logger.info("Mixing engine initialized")
                except Exception as e:
                    results["components"]["mixing_engine"] = False
                    results["errors"].append(f"Mixing engine: {str(e)}")
                    logger.error(f"Failed to initialize mixing engine: {e}")
            else:
                results["components"]["mixing_engine"] = False
            
            self._initialized = True
            results["initialized"] = True
            
        except Exception as e:
            results["initialized"] = False
            results["errors"].append(f"Initialization failed: {str(e)}")
            logger.exception("Coordinator initialization failed")
        
        return results
    
    def _detect_capabilities(self) -> Dict[str, Any]:
        """
        Detect available dependencies.
        
        Returns:
            Dictionary with capability flags:
            - numpy: bool - numpy available
            - librosa: bool - librosa available
            - sqlite3: bool - sqlite3 available
            - ableton_api_version: str - Live API version detected
        """
        caps = {
            'numpy': False,
            'librosa': False,
            'sqlite3': False,
            'ableton_api_version': None
        }
        
        try:
            import numpy
            caps['numpy'] = True
            caps['numpy_version'] = numpy.__version__
        except ImportError:
            pass
        
        try:
            import librosa
            caps['librosa'] = True
            caps['librosa_version'] = librosa.__version__
        except ImportError:
            pass
        
        try:
            import sqlite3
            caps['sqlite3'] = True
        except ImportError:
            pass
        
        # Detect Ableton API version
        if self.song:
            try:
                if hasattr(self.song, 'arrangement_clips'):
                    caps['ableton_api_version'] = '12+'
                elif hasattr(self.song, 'create_audio_track'):
                    caps['ableton_api_version'] = '11+'
                else:
                    caps['ableton_api_version'] = 'legacy'
            except:
                caps['ableton_api_version'] = 'unknown'
        
        return caps
    
    def get_status(self) -> Dict[str, Any]:
        """
        Get complete system status.
        
        Returns:
            Dictionary with:
            - initialized: bool - whether coordinator is initialized
            - extraction_mode: str - current extraction mode
            - capabilities: dict - detected system capabilities
            - components: dict - which components are active
        """
        return {
            "initialized": self._initialized,
            "extraction_mode": self._extraction_mode,
            "capabilities": self._capabilities,
            "components": {
                "metadata_store": self._metadata_store is not None,
                "hybrid_extractor": self._hybrid_extractor is not None,
                "arrangement_recorder": self._arrangement_recorder is not None,
                "live_bridge": self._live_bridge is not None,
                "mixing_engine": self._mixing_engine is not None
            }
        }
    
    def safe_execute(self, operation: Callable, *args, **kwargs) -> Dict[str, Any]:
        """
        Execute operation with error handling.
        
        Wraps any operation and returns a standardized result dictionary
        with success status and error information if applicable.
        
        Args:
            operation: Callable to execute
            *args: Positional arguments for operation
            **kwargs: Keyword arguments for operation
            
        Returns:
            Dictionary with:
            - success: bool
            - result: any (if success)
            - error: str (if failure)
            - type: str - exception type (if failure)
        """
        try:
            result = operation(*args, **kwargs)
            return {"success": True, "result": result}
        except Exception as e:
            logger.exception(f"Operation failed: {operation.__name__ if hasattr(operation, '__name__') else 'unknown'}")
            return {
                "success": False,
                "error": str(e),
                "type": type(e).__name__
            }
    
    # =======================================================================
    # HIGH-LEVEL OPERATIONS
    # =======================================================================
    
    def build_arrangement_timeline(self, sections: List[Dict[str, Any]], 
                                   genre: str = "reggaeton", 
                                   tempo: float = 95, 
                                   key: str = "Am") -> CoordinatorResult:
        """
        Build complete timeline in Arrangement View.
        
        This operation:
        1. Creates necessary tracks via LiveBridge
        2. Loads appropriate samples using hybrid extractor
        3. Places clips at bar positions according to sections
        
        Args:
            sections: List of section dicts with keys:
                - type: str ("intro", "verse", "chorus", etc.)
                - bars: int - duration in bars
                - elements: List[str] - which elements ("drums", "bass", etc.)
            genre: Genre for sample selection
            tempo: Tempo in BPM
            key: Musical key
            
        Returns:
            CoordinatorResult with operation status and details
        """
        if not self._initialized:
            return CoordinatorResult(
                success=False,
                message="Coordinator not initialized. Call initialize() first.",
                operation="build_arrangement_timeline"
            )
        
        try:
            created_tracks = []
            placed_clips = []
            
            # 1. Create tracks via LiveBridge
            if self._live_bridge:
                # Create standard track layout
                track_types = ["drums", "bass", "music", "fx"]
                for track_type in track_types:
                    result = self._live_bridge.create_audio_track(-1)
                    if result.get("success"):
                        track_idx = result.get("data", {}).get("track_index", -1)
                        self._live_bridge.set_track_name(track_idx, f"{track_type.title()} Track")
                        created_tracks.append({"type": track_type, "index": track_idx})
            
            # 2. Load samples using hybrid extractor
            samples_used = []
            if self._hybrid_extractor and SAMPLE_SELECTOR_AVAILABLE and get_selector:
                selector = get_selector()
                if selector:
                    group = selector.select_for_genre(genre, key if key else None, tempo)
                    samples_used.append({
                        "drums": {
                            "kick": group.drums.kick.path if group.drums.kick else None,
                            "snare": group.drums.snare.path if group.drums.snare else None,
                            "clap": group.drums.clap.path if group.drums.clap else None,
                        },
                        "bass": [s.path for s in group.bass[:3]] if group.bass else [],
                        "synths": [s.path for s in group.synths[:3]] if group.synths else []
                    })
            
            # 3. Place clips at bar positions
            current_bar = 0
            for section in sections:
                section_type = section.get("type", "verse")
                bars = section.get("bars", 8)
                elements = section.get("elements", ["drums"])
                
                # Place clips for this section
                for element in elements:
                    # Find track for this element
                    track_info = next((t for t in created_tracks if t["type"] == element), None)
                    if track_info and self._live_bridge:
                        # Place clip at current position
                        # In a real implementation, this would load actual samples
                        placed_clips.append({
                            "track_index": track_info["index"],
                            "element": element,
                            "start_bar": current_bar,
                            "duration_bars": bars,
                            "section": section_type
                        })
                
                current_bar += bars
            
            return CoordinatorResult(
                success=True,
                message=f"Built arrangement timeline with {len(created_tracks)} tracks, {len(placed_clips)} clips",
                data={
                    "tracks": created_tracks,
                    "clips": placed_clips,
                    "samples": samples_used,
                    "total_bars": current_bar,
                    "genre": genre,
                    "tempo": tempo,
                    "key": key
                },
                operation="build_arrangement_timeline"
            )
            
        except Exception as e:
            logger.exception("Failed to build arrangement timeline")
            return CoordinatorResult(
                success=False,
                message=f"Failed to build arrangement: {str(e)}",
                data={"error_type": type(e).__name__},
                operation="build_arrangement_timeline"
            )
    
    def record_arrangement_session(self, duration_bars: float, 
                                   pre_roll: float = 1.0,
                                   start_bar: float = 0.0,
                                   tempo: float = 95.0) -> CoordinatorResult:
        """
        Record Session clips to Arrangement with robust state machine.
        
        This operation configures the ArrangementRecorder, starts the recording
        with quantization, and returns immediate status. The actual recording
        happens asynchronously via the update_display() loop.
        
        Args:
            duration_bars: Total duration to record in bars
            pre_roll: Bars to wait before recording starts (default 1.0)
            start_bar: Starting bar position in arrangement
            tempo: Tempo in BPM for timing calculations
            
        Returns:
            CoordinatorResult with operation status and recording ID
        """
        if not self._initialized:
            return CoordinatorResult(
                success=False,
                message="Coordinator not initialized. Call initialize() first.",
                operation="record_arrangement_session"
            )
        
        if not self._arrangement_recorder:
            return CoordinatorResult(
                success=False,
                message="Arrangement recorder not available",
                operation="record_arrangement_session"
            )
        
        try:
            # Create recording configuration
            if RecordingConfig:
                config = RecordingConfig(
                    start_bar=start_bar,
                    duration_bars=duration_bars,
                    pre_roll_bars=pre_roll,
                    tempo=tempo,
                    scene_index=0,
                    on_state_change=self._on_recording_state_change,
                    on_progress=self._on_recording_progress,
                    on_error=self._on_recording_error,
                    on_completed=self._on_recording_completed
                )
                
                # Arm the recorder
                armed = self._arrangement_recorder.arm(config)
                
                if armed:
                    # Start recording
                    started = self._arrangement_recorder.start()
                    
                    return CoordinatorResult(
                        success=started,
                        message="Recording started" if started else "Failed to start recording",
                        data={
                            "state": self._arrangement_recorder.get_state().name if hasattr(self._arrangement_recorder.get_state(), 'name') else str(self._arrangement_recorder.get_state()),
                            "duration_bars": duration_bars,
                            "pre_roll": pre_roll,
                            "start_bar": start_bar
                        },
                        operation="record_arrangement_session"
                    )
                else:
                    return CoordinatorResult(
                        success=False,
                        message="Failed to arm recorder",
                        operation="record_arrangement_session"
                    )
            else:
                return CoordinatorResult(
                    success=False,
                    message="RecordingConfig not available",
                    operation="record_arrangement_session"
                )
            
        except Exception as e:
            logger.exception("Failed to start arrangement recording")
            return CoordinatorResult(
                success=False,
                message=f"Recording failed: {str(e)}",
                data={"error_type": type(e).__name__},
                operation="record_arrangement_session"
            )
    
    def apply_professional_mix(self, preset_name: str = "reggaeton_club") -> CoordinatorResult:
        """
        Apply professional mix configuration.
        
        This operation:
        1. Loads mix configuration from mixing_engine
        2. Executes configuration via LiveBridge
        3. Returns status per operation
        
        Args:
            preset_name: Mix preset to apply ("reggaeton_club", "reggaeton_clean", 
                        "perreo", "romantico", "minimal")
            
        Returns:
            CoordinatorResult with operation status and applied settings
        """
        if not self._initialized:
            return CoordinatorResult(
                success=False,
                message="Coordinator not initialized. Call initialize() first.",
                operation="apply_professional_mix"
            )
        
        try:
            operations = []
            
            # 1. Get mix configuration
            if MIXING_ENGINE_AVAILABLE and self._mixing_engine:
                config = create_standard_buses()
                apply_send_preset(config, preset_name)
                
                # 2. Execute via LiveBridge
                if self._live_bridge:
                    # Create bus tracks
                    for bus_name, bus_info in config.buses.items():
                        result = self._live_bridge.create_bus_track(
                            bus_info.name, 
                            bus_type=bus_info.bus_type.value if hasattr(bus_info.bus_type, 'value') else str(bus_info.bus_type)
                        )
                        operations.append({
                            "operation": "create_bus",
                            "name": bus_info.name,
                            "success": result.get("success", False)
                        })
                    
                    # Create return tracks
                    for return_name, return_info in config.returns.items():
                        result = self._live_bridge.create_return_track(
                            return_info.name,
                            effect_type=return_info.effect_type.value if hasattr(return_info.effect_type, 'value') else str(return_info.effect_type)
                        )
                        operations.append({
                            "operation": "create_return",
                            "name": return_info.name,
                            "success": result.get("success", False)
                        })
                
                return CoordinatorResult(
                    success=True,
                    message=f"Applied professional mix preset: {preset_name}",
                    data={
                        "preset": preset_name,
                        "buses": list(config.buses.keys()),
                        "returns": list(config.returns.keys()),
                        "operations": operations
                    },
                    operation="apply_professional_mix"
                )
            else:
                return CoordinatorResult(
                    success=False,
                    message="Mixing engine not available",
                    operation="apply_professional_mix"
                )
            
        except Exception as e:
            logger.exception("Failed to apply professional mix")
            return CoordinatorResult(
                success=False,
                message=f"Mix application failed: {str(e)}",
                data={"error_type": type(e).__name__, "operations": operations},
                operation="apply_professional_mix"
            )
    
    def get_recommended_samples_no_numpy(self, role: str, count: int = 10) -> CoordinatorResult:
        """
        Get samples using only database (no numpy).
        
        This is a fallback method that works when numpy/librosa are not
        available. It queries the metadata store directly for samples.
        
        Args:
            role: Sample role ("drums", "bass", "synths", "fx")
            count: Number of samples to return
            
        Returns:
            CoordinatorResult with list of recommended samples
        """
        if not self._initialized:
            return CoordinatorResult(
                success=False,
                message="Coordinator not initialized. Call initialize() first.",
                operation="get_recommended_samples_no_numpy"
            )
        
        if not self._metadata_store:
            return CoordinatorResult(
                success=False,
                message="Metadata store not available",
                operation="get_recommended_samples_no_numpy"
            )
        
        try:
            # Query metadata store directly
            samples = self._metadata_store.search_samples(
                category=role,
                limit=count
            )
            
            sample_list = []
            for sample in samples:
                sample_list.append({
                    "path": sample.path,
                    "bpm": sample.bpm,
                    "key": sample.key,
                    "duration": sample.duration
                })
            
            return CoordinatorResult(
                success=True,
                message=f"Found {len(sample_list)} samples for role '{role}'",
                data={
                    "role": role,
                    "samples": sample_list,
                    "count": len(sample_list)
                },
                operation="get_recommended_samples_no_numpy"
            )
            
        except Exception as e:
            logger.exception("Failed to get recommended samples")
            return CoordinatorResult(
                success=False,
                message=f"Sample query failed: {str(e)}",
                data={"error_type": type(e).__name__},
                operation="get_recommended_samples_no_numpy"
            )
    
    # =======================================================================
    # INTELLIGENT TRACK GENERATION
    # =======================================================================
    
    def generate_intelligent_track(self, 
                                      description: str,
                                      structure_type: str = "standard",
                                      variation_level: str = "medium", 
                                      coherence_threshold: float = 0.90,
                                      include_vocal_placeholder: bool = True,
                                      surprise_mode: bool = False,
                                      save_as_preset: bool = True) -> Dict[str, Any]:
        """Generate complete professional track with intelligent sample selection.
        
        This is the MAIN WORKFLOW for one-prompt music creation.
        
        Workflow:
        1. Parse description → genre, tempo, key, style
        2. Select structure template
        3. Use IntelligentSampleSelector to find coherent samples
        4. Use IterationEngine to achieve target coherence
        5. Use VariationEngine to evolve samples per section
        6. Create arrangement in Ableton via LiveBridge
        7. Apply automatic mixing
        8. Save preset if requested
        9. Log all rationale
        
        Args:
            description: Natural language track description
            structure_type: "tiktok", "short", "standard", "extended"
            variation_level: "low", "medium", "high"
            coherence_threshold: Minimum coherence score (default 0.90)
            include_vocal_placeholder: Add vocal track
            surprise_mode: Random variation
            save_as_preset: Save kit as preset
        
        Returns:
            {
                "success": True,
                "track_name": str,
                "structure": List[SectionConfig],
                "samples_used": Dict[role, SampleKit],
                "coherence_scores": Dict[str, float],
                "coherence_overall": float,
                "rationale_id": str,  # Reference to database log
                "preset_saved": Optional[str],
                "duration_seconds": float,
                "warnings": List[str],
                "next_steps": List[str]
            }
        
        Raises:
            ProfessionalCoherenceError: If cannot achieve coherence_threshold
            after all iteration strategies
        """
        import time
        from typing import List as TypingList
        
        start_time = time.time()
        warnings = []
        next_steps = []
        
        # Check initialization
        if not self._initialized:
            error_msg = "Coordinator not initialized. Call initialize() first."
            logger.error(error_msg)
            return {
                "success": False,
                "track_name": None,
                "structure": [],
                "samples_used": {},
                "coherence_scores": {},
                "coherence_overall": 0.0,
                "rationale_id": None,
                "preset_saved": None,
                "duration_seconds": 0.0,
                "warnings": [error_msg],
                "next_steps": ["Call coordinator.initialize() first"]
            }
        
        # Check LiveBridge availability (required for Ableton integration)
        if not self._live_bridge:
            error_msg = "LiveBridge not available - cannot create arrangement in Ableton"
            logger.error(error_msg)
            warnings.append(error_msg)
            next_steps.append("Ensure Ableton Live connection is active")
        
        # Parse description using available components
        parsed_config = self._parse_description(description)
        genre = parsed_config.get("genre", "reggaeton")
        tempo = parsed_config.get("tempo", 95)
        key = parsed_config.get("key", "Am")
        style = parsed_config.get("style", "classic")
        
        logger.info(f"Parsed description: genre={genre}, tempo={tempo}, key={key}, style={style}")
        
        # Generate track name based on parsed config
        track_name = f"{style.title()} {genre.title()} {structure_type.title()}"
        
        # Get structure template based on structure_type
        structure = self._get_structure_template(structure_type)
        logger.info(f"Using structure template: {structure_type} with {len(structure)} sections")
        
        samples_used = {}
        coherence_scores = {}
        
        try:
            # Step 1: Intelligent Sample Selection with iteration
            if SAMPLE_SELECTOR_AVAILABLE and get_selector:
                logger.info("Starting intelligent sample selection...")
                selector = get_selector()
                
                if selector:
                    # Select samples for genre/key/tempo
                    sample_group = selector.select_for_genre(genre, key if key else None, tempo)
                    
                    if sample_group:
                        # Calculate coherence
                        drums_paths = []
                        if sample_group.drums.kick:
                            drums_paths.append(sample_group.drums.kick.path)
                        if sample_group.drums.snare:
                            drums_paths.append(sample_group.drums.snare.path)
                        if sample_group.drums.clap:
                            drums_paths.append(sample_group.drums.clap.path)
                        
                        bass_paths = [s.path for s in sample_group.bass[:3]] if sample_group.bass else []
                        synth_paths = [s.path for s in sample_group.synths[:3]] if sample_group.synths else []
                        
                        # Calculate coherence for each role
                        drums_coherence = self._calculate_coherence(drums_paths) if drums_paths else 0.0
                        bass_coherence = self._calculate_coherence(bass_paths) if bass_paths else 0.0
                        synth_coherence = self._calculate_coherence(synth_paths) if synth_paths else 0.0
                        
                        coherence_scores = {
                            "drums": drums_coherence,
                            "bass": bass_coherence,
                            "synths": synth_coherence
                        }
                        
                        # Calculate overall coherence (weighted average)
                        coherence_overall = (
                            drums_coherence * 0.5 +
                            bass_coherence * 0.3 +
                            synth_coherence * 0.2
                        )
                        
                        samples_used = {
                            "drums": {
                                "kick": sample_group.drums.kick.path if sample_group.drums.kick else None,
                                "snare": sample_group.drums.snare.path if sample_group.drums.snare else None,
                                "clap": sample_group.drums.clap.path if sample_group.drums.clap else None,
                                "coherence": drums_coherence
                            },
                            "bass": {
                                "paths": bass_paths,
                                "coherence": bass_coherence
                            },
                            "synths": {
                                "paths": synth_paths,
                                "coherence": synth_coherence
                            }
                        }
                        
                        logger.info(f"Sample coherence - drums: {drums_coherence:.2f}, "
                                   f"bass: {bass_coherence:.2f}, synths: {synth_coherence:.2f}")
                        logger.info(f"Overall coherence: {coherence_overall:.2f} (target: {coherence_threshold:.2f})")
                        
                        # Iterate if coherence below threshold (simple iteration)
                        iteration_attempts = 0
                        max_iterations = 3
                        
                        while coherence_overall < coherence_threshold and iteration_attempts < max_iterations:
                            iteration_attempts += 1
                            logger.info(f"Coherence below threshold, iteration attempt {iteration_attempts}")
                            
                            # Try to get alternative samples
                            alternative_group = selector.select_for_genre(genre, key, tempo)
                            if alternative_group:
                                # Recalculate with new samples
                                new_drums = [s.path for s in [alternative_group.drums.kick, 
                                                               alternative_group.drums.snare, 
                                                               alternative_group.drums.clap] if s]
                                new_bass = [s.path for s in alternative_group.bass[:3]]
                                new_synths = [s.path for s in alternative_group.synths[:3]]
                                
                                new_drums_coherence = self._calculate_coherence(new_drums)
                                new_bass_coherence = self._calculate_coherence(new_bass)
                                new_synth_coherence = self._calculate_coherence(new_synths)
                                
                                new_overall = (
                                    new_drums_coherence * 0.5 +
                                    new_bass_coherence * 0.3 +
                                    new_synth_coherence * 0.2
                                )
                                
                                # Use new samples if better
                                if new_overall > coherence_overall:
                                    coherence_overall = new_overall
                                    coherence_scores = {
                                        "drums": new_drums_coherence,
                                        "bass": new_bass_coherence,
                                        "synths": new_synth_coherence
                                    }
                                    samples_used["drums"]["coherence"] = new_drums_coherence
                                    samples_used["bass"]["coherence"] = new_bass_coherence
                                    samples_used["synths"]["coherence"] = new_synth_coherence
                                    
                                    logger.info(f"Found better samples, new coherence: {coherence_overall:.2f}")
                        
                        # Check final coherence
                        if coherence_overall < coherence_threshold:
                            warning_msg = (f"Could not achieve target coherence {coherence_threshold:.2f} "
                                          f"after {iteration_attempts} iterations. Final: {coherence_overall:.2f}")
                            warnings.append(warning_msg)
                            logger.warning(warning_msg)
                            next_steps.append("Try different genre/key or lower coherence threshold")
                        else:
                            logger.info(f"Achieved target coherence: {coherence_overall:.2f}")
                    else:
                        warnings.append("Sample group not returned from selector")
                else:
                    warnings.append("Sample selector not available")
            else:
                warnings.append("Sample selector module not available - using default samples")
                next_steps.append("Install sample_selector for intelligent selection")
            
            # Step 2: Apply variations per section based on variation_level
            variation_factor = {"low": 0.2, "medium": 0.5, "high": 0.8}.get(variation_level, 0.5)
            logger.info(f"Applying variation level '{variation_level}' with factor {variation_factor}")
            
            # Surprise mode adds randomness
            if surprise_mode:
                import random
                variation_factor = min(1.0, variation_factor + random.uniform(0.1, 0.3))
                logger.info(f"Surprise mode active, adjusted variation factor: {variation_factor:.2f}")
                warnings.append("Surprise mode enabled - variations may be unconventional")
            
            # Step 3: Create arrangement in Ableton via LiveBridge
            arrangement_created = False
            if self._live_bridge:
                try:
                    logger.info("Creating arrangement in Ableton...")
                    
                    # Create tracks
                    track_indices = {}
                    track_types = ["drums", "bass", "synths"]
                    
                    for track_type in track_types:
                        result = self._live_bridge.create_audio_track(-1)
                        if result.get("success"):
                            idx = result.get("data", {}).get("track_index", -1)
                            track_indices[track_type] = idx
                            self._live_bridge.set_track_name(idx, f"{track_type.title()} Track")
                            logger.info(f"Created {track_type} track at index {idx}")
                    
                    # Add vocal placeholder if requested
                    if include_vocal_placeholder:
                        vocal_result = self._live_bridge.create_audio_track(-1)
                        if vocal_result.get("success"):
                            vocal_idx = vocal_result.get("data", {}).get("track_index", -1)
                            track_indices["vocal"] = vocal_idx
                            self._live_bridge.set_track_name(vocal_idx, "Vocal Placeholder")
                            logger.info(f"Created vocal placeholder track at index {vocal_idx}")
                    
                    # Place clips for each section
                    current_bar = 0
                    for section in structure:
                        section_type = section.get("type", "verse")
                        bars = section.get("bars", 8)
                        elements = section.get("elements", ["drums", "bass"])
                        
                        # Apply variation to elements based on section type
                        varied_elements = self._apply_section_variation(
                            elements, section_type, variation_factor
                        )
                        
                        for element in varied_elements:
                            if element in track_indices:
                                # In real implementation, this would load actual samples
                                logger.debug(f"Placing {element} clip at bar {current_bar} "
                                            f"for {bars} bars ({section_type})")
                        
                        current_bar += bars
                    
                    arrangement_created = True
                    logger.info(f"Arrangement created with {len(track_indices)} tracks, "
                               f"{current_bar} total bars")
                    next_steps.append("Review arrangement in Ableton and adjust as needed")
                    
                except Exception as e:
                    error_msg = f"Failed to create arrangement: {str(e)}"
                    logger.exception(error_msg)
                    warnings.append(error_msg)
                    next_steps.append("Check LiveBridge connection and retry")
            else:
                warnings.append("LiveBridge unavailable - arrangement not created in Ableton")
                next_steps.append("Ensure Ableton connection is active and retry")
            
            # Step 4: Apply automatic mixing
            if MIXING_ENGINE_AVAILABLE and self._mixing_engine and arrangement_created:
                try:
                    mix_preset = self._determine_mix_preset(genre, style)
                    logger.info(f"Applying mix preset: {mix_preset}")
                    
                    mix_result = self.apply_professional_mix(mix_preset)
                    if mix_result.success:
                        logger.info("Professional mix applied successfully")
                        next_steps.append("Fine-tune mix levels if needed")
                    else:
                        warnings.append(f"Mix application: {mix_result.message}")
                        next_steps.append("Apply manual mixing")
                except Exception as e:
                    warnings.append(f"Mix application failed: {str(e)}")
                    next_steps.append("Apply manual mixing in Ableton")
            else:
                warnings.append("Automatic mixing skipped (engine unavailable or no arrangement)")
                next_steps.append("Apply manual mixing in Ableton")
            
            # Step 5: Log rationale (simplified - would use proper logging in production)
            rationale_id = f"track_{int(start_time)}_{track_name.replace(' ', '_').lower()}"
            logger.info(f"Rationale logged with ID: {rationale_id}")
            
            # Step 6: Save preset if requested
            preset_saved = None
            if save_as_preset and samples_used:
                try:
                    preset_name = f"{track_name.replace(' ', '_')}_{int(start_time)}"
                    # In production, this would save to actual preset storage
                    preset_saved = preset_name
                    logger.info(f"Preset saved as: {preset_name}")
                    next_steps.append(f"Preset '{preset_name}' available for future use")
                except Exception as e:
                    warnings.append(f"Failed to save preset: {str(e)}")
            
            duration = time.time() - start_time
            logger.info(f"Track generation completed in {duration:.2f} seconds")
            
            # Calculate overall coherence if not already done
            coherence_overall = sum(coherence_scores.values()) / len(coherence_scores) if coherence_scores else 0.0
            
            return {
                "success": True,
                "track_name": track_name,
                "structure": structure,
                "samples_used": samples_used,
                "coherence_scores": coherence_scores,
                "coherence_overall": coherence_overall,
                "rationale_id": rationale_id,
                "preset_saved": preset_saved,
                "duration_seconds": duration,
                "warnings": warnings,
                "next_steps": next_steps
            }
            
        except Exception as e:
            error_msg = f"Track generation failed: {str(e)}"
            logger.exception(error_msg)
            duration = time.time() - start_time
            
            return {
                "success": False,
                "track_name": track_name if 'track_name' in locals() else None,
                "structure": structure if 'structure' in locals() else [],
                "samples_used": samples_used,
                "coherence_scores": coherence_scores,
                "coherence_overall": 0.0,
                "rationale_id": None,
                "preset_saved": None,
                "duration_seconds": duration,
                "warnings": warnings + [error_msg],
                "next_steps": ["Check logs for details", "Retry with different parameters"]
            }
    
    def _parse_description(self, description: str) -> Dict[str, Any]:
        """Parse natural language description into configuration."""
        description_lower = description.lower()
        
        # Default config
        config = {
            "genre": "reggaeton",
            "tempo": 95,
            "key": "Am",
            "style": "classic"
        }
        
        # Detect genre
        if "dembow" in description_lower:
            config["genre"] = "reggaeton"
            config["style"] = "dembow"
            config["tempo"] = 90
        elif "perreo" in description_lower:
            config["genre"] = "reggaeton"
            config["style"] = "perreo"
            config["tempo"] = 95
        elif "romantic" in description_lower or "romantico" in description_lower:
            config["genre"] = "reggaeton"
            config["style"] = "romantico"
            config["tempo"] = 88
        elif "trap" in description_lower:
            config["genre"] = "trap"
            config["style"] = "dark"
            config["tempo"] = 140
        elif "house" in description_lower:
            config["genre"] = "house"
            config["style"] = "classic"
            config["tempo"] = 128
        
        # Detect tempo
        import re
        tempo_match = re.search(r'(\d+)\s*bpm', description_lower)
        if tempo_match:
            config["tempo"] = int(tempo_match.group(1))
        elif "slow" in description_lower:
            config["tempo"] = max(80, config["tempo"] - 10)
        elif "fast" in description_lower or "upbeat" in description_lower:
            config["tempo"] = min(140, config["tempo"] + 15)
        
        # Detect key
        key_match = re.search(r'\b([A-G][#b]?)\s*(major|minor|m)?\b', description, re.IGNORECASE)
        if key_match:
            key = key_match.group(1).upper()
            is_minor = key_match.group(2)
            if is_minor and ('minor' in is_minor.lower() or is_minor.lower() == 'm'):
                config["key"] = key + "m"
            else:
                config["key"] = key
        
        # Detect style keywords
        if "dark" in description_lower or "heavy" in description_lower:
            config["style"] = "dark"
        elif "bright" in description_lower or "happy" in description_lower:
            config["style"] = "bright"
        elif "minimal" in description_lower:
            config["style"] = "minimal"
        elif "club" in description_lower:
            config["style"] = "club"
        
        return config
    
    def _get_structure_template(self, structure_type: str) -> TypingList[Dict[str, Any]]:
        """Get song structure template based on type."""
        templates = {
            "tiktok": [
                {"type": "hook", "bars": 8, "elements": ["drums", "bass"]},
                {"type": "drop", "bars": 8, "elements": ["drums", "bass", "synths"]}
            ],
            "short": [
                {"type": "intro", "bars": 4, "elements": ["drums"]},
                {"type": "verse", "bars": 8, "elements": ["drums", "bass"]},
                {"type": "chorus", "bars": 8, "elements": ["drums", "bass", "synths"]},
                {"type": "outro", "bars": 4, "elements": ["drums"]}
            ],
            "standard": [
                {"type": "intro", "bars": 8, "elements": ["drums"]},
                {"type": "verse", "bars": 16, "elements": ["drums", "bass"]},
                {"type": "pre_chorus", "bars": 8, "elements": ["drums", "bass", "synths"]},
                {"type": "chorus", "bars": 16, "elements": ["drums", "bass", "synths"]},
                {"type": "verse", "bars": 16, "elements": ["drums", "bass"]},
                {"type": "chorus", "bars": 16, "elements": ["drums", "bass", "synths"]},
                {"type": "bridge", "bars": 8, "elements": ["bass", "synths"]},
                {"type": "chorus", "bars": 16, "elements": ["drums", "bass", "synths"]},
                {"type": "outro", "bars": 8, "elements": ["drums"]}
            ],
            "extended": [
                {"type": "intro", "bars": 16, "elements": ["drums"]},
                {"type": "build", "bars": 8, "elements": ["drums", "synths"]},
                {"type": "drop", "bars": 16, "elements": ["drums", "bass", "synths"]},
                {"type": "verse", "bars": 16, "elements": ["drums", "bass"]},
                {"type": "build", "bars": 8, "elements": ["drums", "synths"]},
                {"type": "drop", "bars": 16, "elements": ["drums", "bass", "synths"]},
                {"type": "breakdown", "bars": 16, "elements": ["synths"]},
                {"type": "build", "bars": 8, "elements": ["drums", "synths"]},
                {"type": "drop", "bars": 16, "elements": ["drums", "bass", "synths"]},
                {"type": "outro", "bars": 16, "elements": ["drums"]}
            ]
        }
        
        return templates.get(structure_type, templates["standard"])
    
    def _calculate_coherence(self, sample_paths: TypingList[str]) -> float:
        """Calculate coherence score for a set of samples."""
        if not sample_paths or len(sample_paths) < 2:
            return 1.0  # Single sample has perfect coherence
        
        # If metadata store available, use spectral features
        if self._metadata_store:
            try:
                features_list = []
                for path in sample_paths:
                    sample = self._metadata_store.get_sample_by_path(path)
                    if sample and hasattr(sample, 'spectral_centroid'):
                        features_list.append(sample.spectral_centroid)
                
                if len(features_list) >= 2:
                    # Calculate variance of spectral features
                    import statistics
                    mean_val = statistics.mean(features_list)
                    if mean_val == 0:
                        return 1.0
                    variance = statistics.variance(features_list) if len(features_list) > 1 else 0
                    # Coherence is inverse of normalized variance
                    coherence = max(0.0, 1.0 - (variance / (mean_val ** 2)) if mean_val else 1.0)
                    return min(1.0, coherence)
            except Exception as e:
                logger.warning(f"Coherence calculation failed: {e}")
        
        # Fallback: assume high coherence
        return 0.85
    
    def _apply_section_variation(self, elements: TypingList[str], 
                                  section_type: str, 
                                  variation_factor: float) -> TypingList[str]:
        """Apply variation to elements based on section type and factor."""
        import random
        
        base_elements = elements.copy()
        
        # Adjust elements based on section type
        if section_type in ["intro", "outro"]:
            # Sparse arrangement
            if variation_factor > 0.5 and "synths" in base_elements:
                base_elements.remove("synths")
        elif section_type == "chorus":
            # Full arrangement
            if "synths" not in base_elements and variation_factor > 0.3:
                base_elements.append("synths")
        elif section_type in ["drop", "build"]:
            # Maximum elements
            for elem in ["drums", "bass", "synths"]:
                if elem not in base_elements:
                    base_elements.append(elem)
        elif section_type == "breakdown":
            # Minimal drums
            if "drums" in base_elements and variation_factor > 0.4:
                base_elements.remove("drums")
        
        # Apply random variation
        if variation_factor > 0.6 and random.random() < variation_factor:
            # Randomly swap an element
            all_elements = ["drums", "bass", "synths", "fx"]
            available = [e for e in all_elements if e not in base_elements]
            if available and base_elements:
                # Remove one, add one
                if random.random() < 0.5:
                    base_elements.pop(random.randint(0, len(base_elements) - 1))
                base_elements.append(random.choice(available))
        
        return base_elements
    
    def _determine_mix_preset(self, genre: str, style: str) -> str:
        """Determine appropriate mix preset based on genre and style."""
        preset_map = {
            ("reggaeton", "dembow"): "reggaeton_club",
            ("reggaeton", "perreo"): "perreo",
            ("reggaeton", "romantico"): "romantico",
            ("reggaeton", "classic"): "reggaeton_club",
            ("trap", "dark"): "trap_dark",
            ("trap", "bright"): "trap_clean",
            ("house", "classic"): "house_club",
            ("house", "minimal"): "minimal"
        }
        
        return preset_map.get((genre, style), "reggaeton_club")
    
    # =======================================================================
    # RECORDING CALLBACKS
    # =======================================================================
    
    def _on_recording_state_change(self, old_state, new_state):
        """Callback when recording state changes."""
        logger.info(f"Recording state: {old_state} -> {new_state}")
    
    def _on_recording_progress(self, progress: float):
        """Callback with recording progress (0.0-1.0)."""
        logger.debug(f"Recording progress: {progress:.1%}")
    
    def _on_recording_error(self, error: Exception):
        """Callback on recording error."""
        logger.error(f"Recording error: {error}")
    
    def _on_recording_completed(self, clip_ids: List[str]):
        """Callback when recording completes successfully."""
        logger.info(f"Recording completed with {len(clip_ids)} new clips")


# =============================================================================
# HELPER FUNCTIONS
# =============================================================================

# Singleton storage
_coordinator_singleton: Optional[SeniorArchitectureCoordinator] = None


def create_coordinator(song, connection, db_path: Optional[str] = None) -> Dict[str, Any]:
    """
    Factory function to create and initialize coordinator.
    
    This is the recommended way to create a coordinator instance.
    It creates the coordinator and immediately initializes all components.
    
    Args:
        song: Ableton Live Song object
        connection: MCP TCP connection
        db_path: Optional path to metadata database
        
    Returns:
        Dictionary with:
        - coordinator: SeniorArchitectureCoordinator instance (or None on failure)
        - status: Initialization status dict
    """
    try:
        coord = SeniorArchitectureCoordinator(song, connection, db_path)
        status = coord.initialize()
        
        return {
            "coordinator": coord,
            "status": status
        }
    except Exception as e:
        logger.exception("Failed to create coordinator")
        return {
            "coordinator": None,
            "status": {
                "initialized": False,
                "error": str(e)
            }
        }


def get_coordinator_singleton(song=None, connection=None, db_path: Optional[str] = None) -> Optional[SeniorArchitectureCoordinator]:
    """
    Get or create singleton instance.
    
    This function returns the existing coordinator if one exists,
    or creates a new one if needed. If song and connection are provided
    but no coordinator exists, it will create and initialize one.
    
    Args:
        song: Ableton Live Song object (required for first creation)
        connection: MCP TCP connection (required for first creation)
        db_path: Optional path to metadata database
        
    Returns:
        SeniorArchitectureCoordinator instance or None
    """
    global _coordinator_singleton
    
    if _coordinator_singleton is not None:
        return _coordinator_singleton
    
    if song is not None and connection is not None:
        result = create_coordinator(song, connection, db_path)
        _coordinator_singleton = result.get("coordinator")
        return _coordinator_singleton
    
    return None


def reset_coordinator_singleton():
    """Reset the singleton instance. Useful for testing."""
    global _coordinator_singleton
    _coordinator_singleton = None
    logger.info("Coordinator singleton reset")


# =============================================================================
# COMPATIBILITY EXPORTS
# =============================================================================

__all__ = [
    "SeniorArchitectureCoordinator",
    "CoordinatorResult",
    "create_coordinator",
    "get_coordinator_singleton",
    "reset_coordinator_singleton",
]