feat: implement vertical crop, local LLM deepseek-v4-flash, multi-threading, word-level pink box highlight subtitles, and Nextcloud scan sync
This commit is contained in:
Renato
@@ -0,0 +1,6 @@
|
||||
"""Local-mode backends — no MuAPI calls, runs on your machine.
|
||||
|
||||
Used when the pipeline is invoked with mode="local". Requires the optional
|
||||
deps in requirements-local.txt (yt-dlp, faster-whisper, openai, google-genai,
|
||||
opencv, moviepy) plus an LLM API key for highlight ranking.
|
||||
"""
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Local clipping: ffmpeg subclip + OpenCV face-aware vertical crop.
|
||||
|
||||
Two stages per highlight:
|
||||
1. Cut the source video to [start, end] with ffmpeg (re-encoded, audio kept).
|
||||
2. Reframe the cut to the target aspect ratio. For 9:16 we slide a vertical
|
||||
window horizontally across the frame to keep faces centred (Haar
|
||||
cascade — same approach as the original repo, no external models).
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from ..config import LOCAL_OUTPUT_DIR
|
||||
|
||||
|
||||
def _ratio(aspect_ratio: str) -> float:
|
||||
"""Parse '9:16' → 9/16, '1:1' → 1.0."""
|
||||
try:
|
||||
w, h = aspect_ratio.split(":")
|
||||
return float(w) / float(h)
|
||||
except (ValueError, ZeroDivisionError):
|
||||
return 9.0 / 16.0
|
||||
|
||||
|
||||
def _cut_subclip(source_path: str, start: float, end: float, out_path: str) -> str:
|
||||
"""ffmpeg -ss start -to end → re-encoded mp4 with audio."""
|
||||
cmd = [
|
||||
"ffmpeg", "-y", "-loglevel", "error",
|
||||
"-i", source_path,
|
||||
"-ss", f"{start:.3f}",
|
||||
"-to", f"{end:.3f}",
|
||||
"-c:v", "libx264", "-preset", "fast", "-crf", "20",
|
||||
"-c:a", "aac", "-b:a", "128k",
|
||||
out_path,
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
return out_path
|
||||
|
||||
|
||||
def _reframe_vertical(in_path: str, out_path: str, aspect_ratio: str) -> str:
|
||||
"""Crop the cut clip to the target aspect ratio, tracking faces if possible."""
|
||||
try:
|
||||
import cv2 # type: ignore
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"opencv-python is required for --mode local. Install it with:\n"
|
||||
" pip install -r requirements-local.txt"
|
||||
) from e
|
||||
|
||||
target_ratio = _ratio(aspect_ratio)
|
||||
cap = cv2.VideoCapture(in_path)
|
||||
if not cap.isOpened():
|
||||
raise RuntimeError(f"could not open {in_path}")
|
||||
|
||||
src_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
||||
src_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
|
||||
# Compute the largest crop that fits inside the frame at the target ratio.
|
||||
if target_ratio < src_w / src_h:
|
||||
crop_h = src_h
|
||||
crop_w = int(crop_h * target_ratio)
|
||||
else:
|
||||
crop_w = src_w
|
||||
crop_h = int(crop_w / target_ratio)
|
||||
crop_w = max(2, crop_w - (crop_w % 2))
|
||||
crop_h = max(2, crop_h - (crop_h % 2))
|
||||
|
||||
face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
|
||||
|
||||
silent_path = out_path + ".silent.mp4"
|
||||
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
||||
writer = cv2.VideoWriter(silent_path, fourcc, fps, (crop_w, crop_h))
|
||||
|
||||
last_center: Optional[Tuple[int, int]] = None
|
||||
smoothing = 0.15 # how aggressively to chase a new face position
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
break
|
||||
|
||||
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||
faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=5, minSize=(40, 40))
|
||||
if len(faces) > 0:
|
||||
# Pick the largest face — usually the speaker.
|
||||
x, y, w, h = max(faces, key=lambda f: f[2] * f[3])
|
||||
cx = x + w // 2
|
||||
cy = y + h // 2
|
||||
if last_center is None:
|
||||
last_center = (cx, cy)
|
||||
else:
|
||||
lx, ly = last_center
|
||||
last_center = (
|
||||
int(lx + (cx - lx) * smoothing),
|
||||
int(ly + (cy - ly) * smoothing),
|
||||
)
|
||||
if last_center is None:
|
||||
last_center = (src_w // 2, src_h // 2)
|
||||
|
||||
cx, cy = last_center
|
||||
x0 = max(0, min(src_w - crop_w, cx - crop_w // 2))
|
||||
y0 = max(0, min(src_h - crop_h, cy - crop_h // 2))
|
||||
cropped = frame[y0:y0 + crop_h, x0:x0 + crop_w]
|
||||
writer.write(cropped)
|
||||
|
||||
cap.release()
|
||||
writer.release()
|
||||
|
||||
# Mux audio from the cut clip back onto the silent reframed video.
|
||||
cmd = [
|
||||
"ffmpeg", "-y", "-loglevel", "error",
|
||||
"-i", silent_path,
|
||||
"-i", in_path,
|
||||
"-c:v", "copy",
|
||||
"-c:a", "aac", "-b:a", "128k",
|
||||
"-map", "0:v:0", "-map", "1:a:0?",
|
||||
"-shortest",
|
||||
out_path,
|
||||
]
|
||||
subprocess.run(cmd, check=True)
|
||||
os.remove(silent_path)
|
||||
return out_path
|
||||
|
||||
|
||||
def crop_clip_local(
|
||||
source_path: str,
|
||||
start_time: float,
|
||||
end_time: float,
|
||||
aspect_ratio: str,
|
||||
out_path: str,
|
||||
) -> str:
|
||||
"""Cut + reframe one highlight, returning the local mp4 path."""
|
||||
cut_path = out_path + ".cut.mp4"
|
||||
try:
|
||||
_cut_subclip(source_path, start_time, end_time, cut_path)
|
||||
_reframe_vertical(cut_path, out_path, aspect_ratio)
|
||||
finally:
|
||||
if os.path.exists(cut_path):
|
||||
os.remove(cut_path)
|
||||
return out_path
|
||||
|
||||
|
||||
def crop_highlights_local(
|
||||
source_path: str,
|
||||
highlights: List[Dict],
|
||||
aspect_ratio: str = "9:16",
|
||||
out_dir: Optional[str] = None,
|
||||
) -> List[Dict]:
|
||||
out_dir = out_dir or LOCAL_OUTPUT_DIR
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
results: List[Dict] = []
|
||||
for i, h in enumerate(highlights, 1):
|
||||
out_path = os.path.join(out_dir, f"short_{i:02d}.mp4")
|
||||
print(f"[clip/local] {i}/{len(highlights)}: {h.get('title', '(untitled)')}", flush=True)
|
||||
try:
|
||||
crop_clip_local(
|
||||
source_path,
|
||||
float(h["start_time"]),
|
||||
float(h["end_time"]),
|
||||
aspect_ratio,
|
||||
out_path,
|
||||
)
|
||||
results.append({**h, "clip_url": out_path})
|
||||
except Exception as e:
|
||||
print(f"[clip/local] {i} failed: {e}", flush=True)
|
||||
results.append({**h, "clip_url": None, "error": str(e)})
|
||||
return results
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Local YouTube download via yt-dlp.
|
||||
|
||||
Returns a local mp4 path so the rest of the local pipeline can read it
|
||||
directly off disk.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
from typing import Optional
|
||||
|
||||
from ..config import LOCAL_OUTPUT_DIR
|
||||
|
||||
|
||||
def _import_ytdlp():
|
||||
try:
|
||||
import yt_dlp # type: ignore
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"yt-dlp is required for --mode local. Install it with:\n"
|
||||
" pip install -r requirements-local.txt"
|
||||
) from e
|
||||
return yt_dlp
|
||||
|
||||
|
||||
def _format_for(fmt: str) -> str:
|
||||
"""Map our '720' / '1080' shorthand to a yt-dlp format selector."""
|
||||
try:
|
||||
height = int(fmt)
|
||||
except ValueError:
|
||||
height = 720
|
||||
return (
|
||||
f"bestvideo[height<={height}][ext=mp4]+bestaudio[ext=m4a]/"
|
||||
f"best[height<={height}][ext=mp4]/best"
|
||||
)
|
||||
|
||||
|
||||
def _extract_youtube_video_id(source: str) -> Optional[str]:
|
||||
"""Best-effort extraction of a YouTube video id from a URL."""
|
||||
parsed = urlparse(source)
|
||||
host = (parsed.netloc or "").lower()
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
|
||||
if host in ("youtu.be", "www.youtu.be"):
|
||||
video_id = parsed.path.lstrip("/").split("/", 1)[0]
|
||||
return video_id or None
|
||||
|
||||
if "youtube.com" in host:
|
||||
if parsed.path.startswith("/watch"):
|
||||
qs = parse_qs(parsed.query)
|
||||
video_id = qs.get("v", [""])[0]
|
||||
return video_id or None
|
||||
match = re.search(r"/(?:shorts|embed|live)/([^/?#&]+)", parsed.path)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_local_path(source: str) -> Optional[str]:
|
||||
"""Return a local filesystem path if the input already points at one."""
|
||||
parsed = urlparse(source)
|
||||
if parsed.scheme == "file":
|
||||
raw_path = unquote(parsed.path)
|
||||
if parsed.netloc and parsed.netloc not in ("", "localhost"):
|
||||
raw_path = f"//{parsed.netloc}{raw_path}"
|
||||
candidate = Path(raw_path).expanduser()
|
||||
if candidate.exists() and candidate.is_file():
|
||||
return str(candidate.resolve())
|
||||
raise RuntimeError(f"Local file URL does not exist: {source}")
|
||||
|
||||
if parsed.scheme in ("http", "https"):
|
||||
return None
|
||||
|
||||
candidate = Path(source).expanduser()
|
||||
if candidate.exists() and candidate.is_file():
|
||||
return str(candidate.resolve())
|
||||
|
||||
if any(sep in source for sep in (os.sep, "/")) or source.startswith("~") or source.startswith("."):
|
||||
raise RuntimeError(f"Local file path does not exist: {source}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _existing_download(out_dir: str, video_id: str) -> Optional[str]:
|
||||
"""Return a cached download path if we already have this YouTube id."""
|
||||
for ext in (".mp4", ".mkv", ".webm"):
|
||||
candidate = os.path.join(out_dir, f"source_{video_id}{ext}")
|
||||
if os.path.exists(candidate):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def download_youtube_local(video_url: str, fmt: str = "720", out_dir: Optional[str] = None) -> str:
|
||||
"""Download a remote URL or return a local file path unchanged."""
|
||||
local_path = _resolve_local_path(video_url)
|
||||
if local_path:
|
||||
print(f"[download/local] using local file: {local_path}", flush=True)
|
||||
return local_path
|
||||
|
||||
yt_dlp = _import_ytdlp()
|
||||
out_dir = out_dir or LOCAL_OUTPUT_DIR
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
video_id = _extract_youtube_video_id(video_url)
|
||||
if video_id:
|
||||
cached = _existing_download(out_dir, video_id)
|
||||
if cached:
|
||||
print(f"[download/local] reusing cached download: {cached}", flush=True)
|
||||
return cached
|
||||
|
||||
print(f"[download/local] {video_url} @ {fmt}p → {out_dir}/", flush=True)
|
||||
ydl_opts = {
|
||||
"format": _format_for(fmt),
|
||||
"outtmpl": os.path.join(out_dir, "source_%(id)s.%(ext)s"),
|
||||
"merge_output_format": "mp4",
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
"noprogress": True,
|
||||
}
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
info = ydl.extract_info(video_url, download=True)
|
||||
path = ydl.prepare_filename(info)
|
||||
# merge_output_format may rename the extension after merge
|
||||
if not os.path.exists(path):
|
||||
stem, _ = os.path.splitext(path)
|
||||
for ext in (".mp4", ".mkv", ".webm"):
|
||||
if os.path.exists(stem + ext):
|
||||
path = stem + ext
|
||||
break
|
||||
|
||||
print(f"[download/local] ready: {path}", flush=True)
|
||||
return path
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Local LLM backend — OpenAI or Gemini, selected by LLM_PROVIDER."""
|
||||
from ..config import (
|
||||
GEMINI_MODEL,
|
||||
LLM_PROVIDER,
|
||||
OPENAI_MODEL,
|
||||
require_gemini_key,
|
||||
require_openai_key,
|
||||
)
|
||||
|
||||
|
||||
def call_openai_llm(prompt: str) -> str:
|
||||
"""OpenAI Chat Completions backend used by --mode local."""
|
||||
try:
|
||||
from openai import OpenAI # type: ignore
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"openai is required for --mode local. Install it with:\n"
|
||||
" pip install -r requirements-local.txt"
|
||||
) from e
|
||||
|
||||
client = OpenAI(api_key=require_openai_key())
|
||||
response = client.chat.completions.create(
|
||||
model=OPENAI_MODEL,
|
||||
temperature=0.7,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
|
||||
|
||||
def call_gemini_llm(prompt: str) -> str:
|
||||
"""Gemini backend used by --mode local when LLM_PROVIDER=gemini."""
|
||||
try:
|
||||
from google import genai # type: ignore
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"google-genai is required for LLM_PROVIDER=gemini. Install it with:\n"
|
||||
" pip install -r requirements-local.txt"
|
||||
) from e
|
||||
|
||||
client = genai.Client(api_key=require_gemini_key())
|
||||
response = client.models.generate_content(
|
||||
model=GEMINI_MODEL,
|
||||
contents=prompt,
|
||||
config={
|
||||
"temperature": 0.2,
|
||||
"response_mime_type": "application/json",
|
||||
"max_output_tokens": 8192,
|
||||
},
|
||||
)
|
||||
return response.text or ""
|
||||
|
||||
|
||||
def call_local_llm(prompt: str) -> str:
|
||||
"""Dispatch to the configured local LLM provider."""
|
||||
provider = (LLM_PROVIDER or "openai").strip().lower()
|
||||
if provider == "openai":
|
||||
return call_openai_llm(prompt)
|
||||
if provider == "gemini":
|
||||
return call_gemini_llm(prompt)
|
||||
raise RuntimeError(
|
||||
f"Unknown LLM_PROVIDER={provider!r}. Use 'openai' or 'gemini'."
|
||||
)
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Local transcription via faster-whisper.
|
||||
|
||||
Reads a local media file and returns the same shape the highlight generator
|
||||
expects: {duration, segments[start, end, text]}.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
from ..config import LOCAL_OUTPUT_DIR, LOCAL_WHISPER_DEVICE, LOCAL_WHISPER_MODEL
|
||||
|
||||
|
||||
def _transcript_cache_path(media_path: str) -> Path:
|
||||
"""Return the .srt cache path for a media file."""
|
||||
cache_dir = Path(LOCAL_OUTPUT_DIR)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
return cache_dir / (Path(media_path).stem + ".srt")
|
||||
|
||||
|
||||
def _format_srt_timestamp(seconds: float) -> str:
|
||||
total_ms = max(0, int(round(seconds * 1000)))
|
||||
ms = total_ms % 1000
|
||||
total_s = total_ms // 1000
|
||||
s = total_s % 60
|
||||
total_m = total_s // 60
|
||||
m = total_m % 60
|
||||
h = total_m // 60
|
||||
return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
|
||||
|
||||
|
||||
def _parse_srt_timestamp(value: str) -> float:
|
||||
match = re.fullmatch(r"(\d{2}):(\d{2}):(\d{2}),(\d{3})", value.strip())
|
||||
if not match:
|
||||
raise ValueError(f"Invalid SRT timestamp: {value!r}")
|
||||
hours, minutes, seconds, millis = map(int, match.groups())
|
||||
return hours * 3600 + minutes * 60 + seconds + (millis / 1000.0)
|
||||
|
||||
|
||||
def _write_srt_cache(media_path: str, transcript: Dict) -> Path:
|
||||
cache_path = _transcript_cache_path(media_path)
|
||||
lines = []
|
||||
for idx, segment in enumerate(transcript.get("segments", []), start=1):
|
||||
start = _format_srt_timestamp(float(segment["start"]))
|
||||
end = _format_srt_timestamp(float(segment["end"]))
|
||||
text = str(segment.get("text", "")).strip().replace("\r", "").replace("\n", " ")
|
||||
lines.append(str(idx))
|
||||
lines.append(f"{start} --> {end}")
|
||||
lines.append(text)
|
||||
lines.append("")
|
||||
|
||||
cache_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return cache_path
|
||||
|
||||
|
||||
def _load_srt_cache(cache_path: Path) -> Dict:
|
||||
content = cache_path.read_text(encoding="utf-8-sig").strip()
|
||||
if not content:
|
||||
return {"duration": 0.0, "segments": []}
|
||||
|
||||
segments = []
|
||||
for block in re.split(r"\n\s*\n", content):
|
||||
lines = [line.strip("\ufeff") for line in block.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
continue
|
||||
if "-->" not in lines[0] and len(lines) > 1 and "-->" in lines[1]:
|
||||
lines = lines[1:]
|
||||
if not lines or "-->" not in lines[0]:
|
||||
continue
|
||||
start_raw, end_raw = [part.strip() for part in lines[0].split("-->", 1)]
|
||||
text = "\n".join(lines[1:]).strip()
|
||||
segments.append(
|
||||
{
|
||||
"start": _parse_srt_timestamp(start_raw),
|
||||
"end": _parse_srt_timestamp(end_raw),
|
||||
"text": text,
|
||||
}
|
||||
)
|
||||
|
||||
duration = segments[-1]["end"] if segments else 0.0
|
||||
return {"duration": duration, "segments": segments}
|
||||
|
||||
|
||||
def _resolve_device() -> str:
|
||||
if LOCAL_WHISPER_DEVICE != "auto":
|
||||
return LOCAL_WHISPER_DEVICE
|
||||
try:
|
||||
import torch # type: ignore
|
||||
if torch.cuda.is_available():
|
||||
# Test that CUDA actually works (catches missing cuBLAS/cuDNN libs)
|
||||
torch.zeros(1, device="cuda")
|
||||
return "cuda"
|
||||
except (ImportError, OSError, RuntimeError):
|
||||
pass
|
||||
return "cpu"
|
||||
|
||||
|
||||
def transcribe_local(media_path: str, language: Optional[str] = None) -> Dict:
|
||||
"""Run faster-whisper on a local file path, caching the result as .srt."""
|
||||
cache_path = _transcript_cache_path(media_path)
|
||||
if cache_path.exists():
|
||||
source_mtime = os.path.getmtime(media_path)
|
||||
cache_mtime = cache_path.stat().st_mtime
|
||||
if cache_mtime >= source_mtime:
|
||||
print(f"[transcribe/local] reusing cached transcript: {cache_path}", flush=True)
|
||||
cached = _load_srt_cache(cache_path)
|
||||
# Treat empty cache as invalid (likely from a failed/partial run) — delete and re-transcribe
|
||||
if not cached["segments"] or cached["duration"] <= 0.0:
|
||||
print(f"[transcribe/local] cache is empty/invalid, deleting: {cache_path}", flush=True)
|
||||
cache_path.unlink(missing_ok=True)
|
||||
else:
|
||||
print(
|
||||
f"[transcribe/local] {len(cached['segments'])} cached segments, "
|
||||
f"{cached['duration']:.0f}s of audio",
|
||||
flush=True,
|
||||
)
|
||||
return cached
|
||||
|
||||
try:
|
||||
from faster_whisper import WhisperModel # type: ignore
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"faster-whisper is required for --mode local. Install it with:\n"
|
||||
" pip install -r requirements-local.txt"
|
||||
) from e
|
||||
|
||||
device = _resolve_device()
|
||||
compute_type = "float16" if device == "cuda" else "int8"
|
||||
print(f"[transcribe/local] faster-whisper model={LOCAL_WHISPER_MODEL} device={device}", flush=True)
|
||||
|
||||
from ..config import LOCAL_WHISPER_VAD_FILTER, LOCAL_WHISPER_VAD_PARAMETERS
|
||||
|
||||
model = WhisperModel(LOCAL_WHISPER_MODEL, device=device, compute_type=compute_type)
|
||||
|
||||
transcribe_kwargs = {
|
||||
"audio": media_path,
|
||||
"language": language,
|
||||
"beam_size": 5,
|
||||
"condition_on_previous_text": False,
|
||||
}
|
||||
if LOCAL_WHISPER_VAD_FILTER:
|
||||
transcribe_kwargs["vad_filter"] = True
|
||||
transcribe_kwargs["vad_parameters"] = LOCAL_WHISPER_VAD_PARAMETERS
|
||||
else:
|
||||
transcribe_kwargs["vad_filter"] = False
|
||||
|
||||
segments_iter, info = model.transcribe(**transcribe_kwargs)
|
||||
|
||||
segments = []
|
||||
for s in segments_iter:
|
||||
segments.append({
|
||||
"start": float(s.start),
|
||||
"end": float(s.end),
|
||||
"text": (s.text or "").strip(),
|
||||
})
|
||||
|
||||
duration = float(getattr(info, "duration", 0.0)) or (segments[-1]["end"] if segments else 0.0)
|
||||
print(f"[transcribe/local] {len(segments)} segments, {duration:.0f}s of audio", flush=True)
|
||||
transcript = {"duration": duration, "segments": segments}
|
||||
cache_path = _write_srt_cache(media_path, transcript)
|
||||
print(f"[transcribe/local] wrote cache: {cache_path}", flush=True)
|
||||
return transcript
|
||||
Reference in New Issue
Block a user