feat: implement vertical crop, local LLM deepseek-v4-flash, multi-threading, word-level pink box highlight subtitles, and Nextcloud scan sync
This commit is contained in:
Renato
@@ -0,0 +1,446 @@
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
import logging
|
||||
|
||||
# Configure silent logging for boto3 and botocore
|
||||
logging.getLogger('boto3').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('botocore').setLevel(logging.CRITICAL)
|
||||
logging.getLogger('s3transfer').setLevel(logging.CRITICAL)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
def upload_file_to_s3(file_path, bucket_name, s3_key):
|
||||
"""
|
||||
Upload a file to an S3 bucket silently.
|
||||
"""
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
||||
|
||||
if not access_key or not secret_key:
|
||||
return False
|
||||
|
||||
s3_client = boto3.client(
|
||||
's3',
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region
|
||||
)
|
||||
try:
|
||||
# Extra arguments for public read if needed, but the user didn't specify.
|
||||
# Given the bucket name, it might be for a web app.
|
||||
s3_client.upload_file(file_path, bucket_name, s3_key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
from botocore.config import Config
|
||||
import json
|
||||
import time as time_module
|
||||
|
||||
# Simple in-memory cache for gallery clips
|
||||
_clips_cache = {
|
||||
"data": None,
|
||||
"timestamp": 0
|
||||
}
|
||||
CACHE_TTL_SECONDS = 300 # 5 minutes
|
||||
|
||||
def get_s3_client():
|
||||
"""Returns an authenticated S3 client."""
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
||||
|
||||
if not access_key or not secret_key:
|
||||
return None
|
||||
|
||||
return boto3.client(
|
||||
's3',
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region,
|
||||
config=Config(signature_version='s3v4')
|
||||
)
|
||||
|
||||
def generate_presigned_url(bucket_name, object_key, expiration=3600):
|
||||
"""Generate a presigned URL to share an S3 object."""
|
||||
s3_client = get_s3_client()
|
||||
if not s3_client:
|
||||
return None
|
||||
try:
|
||||
response = s3_client.generate_presigned_url('get_object',
|
||||
Params={'Bucket': bucket_name,
|
||||
'Key': object_key},
|
||||
ExpiresIn=expiration)
|
||||
return response
|
||||
except ClientError as e:
|
||||
logger.error(e)
|
||||
return None
|
||||
|
||||
def list_all_clips(bucket_name=None, limit=50, force_refresh=False):
|
||||
"""
|
||||
List recent clips from the S3 bucket by finding metadata files.
|
||||
Returns a list of dicts containing clip info and signed URLs.
|
||||
|
||||
Args:
|
||||
bucket_name: S3 bucket name (defaults to AWS_S3_BUCKET env var)
|
||||
limit: Maximum number of clips to return (default 50 for speed)
|
||||
force_refresh: If True, bypass cache
|
||||
"""
|
||||
global _clips_cache
|
||||
|
||||
# Check cache first
|
||||
now = time_module.time()
|
||||
if not force_refresh and _clips_cache["data"] is not None:
|
||||
if now - _clips_cache["timestamp"] < CACHE_TTL_SECONDS:
|
||||
cached = _clips_cache["data"]
|
||||
return cached[:limit] if limit else cached
|
||||
|
||||
if not bucket_name:
|
||||
bucket_name = os.environ.get('AWS_S3_BUCKET', 'my-clips-bucket')
|
||||
|
||||
s3_client = get_s3_client()
|
||||
if not s3_client:
|
||||
return []
|
||||
|
||||
all_clips = []
|
||||
|
||||
try:
|
||||
# List all objects in bucket
|
||||
# Note: For very large buckets, pagination is needed.
|
||||
# Assuming reasonable size for now, but adding continuation token support is best practice.
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
pages = paginator.paginate(Bucket=bucket_name)
|
||||
|
||||
metadata_files = []
|
||||
for page in pages:
|
||||
if 'Contents' in page:
|
||||
for obj in page['Contents']:
|
||||
if obj['Key'].endswith('_metadata.json'):
|
||||
metadata_files.append(obj)
|
||||
|
||||
# Sort metadata by LastModified (newest first)
|
||||
metadata_files.sort(key=lambda x: x['LastModified'], reverse=True)
|
||||
|
||||
for meta_obj in metadata_files:
|
||||
key = meta_obj['Key']
|
||||
# key format: {job_id}/..._metadata.json
|
||||
|
||||
# Read metadata content
|
||||
try:
|
||||
obj_resp = s3_client.get_object(Bucket=bucket_name, Key=key)
|
||||
content = obj_resp['Body'].read().decode('utf-8')
|
||||
data = json.loads(content)
|
||||
|
||||
parts = key.split('/')
|
||||
job_id = parts[0] if len(parts) > 1 else "unknown"
|
||||
# Filename base for clips in same folder
|
||||
# Meta key: "job_id/filename_metadata.json"
|
||||
# Base name in metadata usually matches filename without ext
|
||||
meta_filename = os.path.basename(key)
|
||||
base_name = meta_filename.replace('_metadata.json', '')
|
||||
|
||||
clips_data = data.get('shorts', [])
|
||||
|
||||
for i, clip in enumerate(clips_data):
|
||||
clip_filename = f"{base_name}_clip_{i+1}.mp4"
|
||||
clip_key = f"{job_id}/{clip_filename}"
|
||||
|
||||
# Generate signed URL
|
||||
signed_url = generate_presigned_url(bucket_name, clip_key, expiration=7200) # 2 hours
|
||||
|
||||
if signed_url:
|
||||
all_clips.append({
|
||||
"job_id": job_id,
|
||||
"index": i,
|
||||
"url": signed_url,
|
||||
"title": clip.get('video_title_for_youtube_short', 'Untitled Clip'),
|
||||
"tiktok_desc": clip.get('video_description_for_tiktok', ''),
|
||||
"insta_desc": clip.get('video_description_for_instagram', ''),
|
||||
"created_at": meta_obj['LastModified'].isoformat(),
|
||||
"duration": clip.get('end', 0) - clip.get('start', 0)
|
||||
})
|
||||
|
||||
# Early exit if we have enough clips
|
||||
if limit and len(all_clips) >= limit:
|
||||
break
|
||||
|
||||
# Early exit if we have enough clips
|
||||
if limit and len(all_clips) >= limit:
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing metadata {key}: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing bucket: {e}")
|
||||
return []
|
||||
|
||||
# Update cache with full results (keep for pagination later)
|
||||
_clips_cache["data"] = all_clips
|
||||
_clips_cache["timestamp"] = now
|
||||
|
||||
return all_clips[:limit] if limit else all_clips
|
||||
|
||||
def upload_actor_to_s3(file_path, description=""):
|
||||
"""
|
||||
Upload an actor image to the public S3 bucket.
|
||||
Returns the public URL or None on failure.
|
||||
"""
|
||||
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
||||
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
||||
|
||||
s3_client = get_s3_client()
|
||||
if not s3_client:
|
||||
return None
|
||||
|
||||
import uuid
|
||||
unique_id = str(uuid.uuid4())[:8]
|
||||
filename = os.path.basename(file_path)
|
||||
name, ext = os.path.splitext(filename)
|
||||
s3_key = f"avatars/{name}_{unique_id}{ext}"
|
||||
|
||||
try:
|
||||
# Skip broken/tiny files
|
||||
if os.path.getsize(file_path) < 1000:
|
||||
logger.warning(f"Skipping tiny file ({os.path.getsize(file_path)} bytes): {file_path}")
|
||||
return None
|
||||
|
||||
s3_client.upload_file(
|
||||
file_path, bucket_name, s3_key,
|
||||
ExtraArgs={'ContentType': 'image/png'},
|
||||
)
|
||||
public_url = f"https://{bucket_name}.s3.{region}.amazonaws.com/{s3_key}"
|
||||
|
||||
# Save metadata JSON alongside the image
|
||||
if description:
|
||||
import datetime
|
||||
meta_key = s3_key.rsplit('.', 1)[0] + '.json'
|
||||
meta = json.dumps({
|
||||
"description": description,
|
||||
"url": public_url,
|
||||
"created_at": datetime.datetime.utcnow().isoformat() + "Z",
|
||||
}, ensure_ascii=False)
|
||||
s3_client.put_object(
|
||||
Bucket=bucket_name, Key=meta_key,
|
||||
Body=meta.encode('utf-8'),
|
||||
ContentType='application/json',
|
||||
)
|
||||
|
||||
logger.info(f"Uploaded actor to S3: {public_url}")
|
||||
return public_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload actor to S3: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def list_actor_gallery():
|
||||
"""
|
||||
List all actor images from the public S3 bucket.
|
||||
Returns list with URLs and descriptions, newest first.
|
||||
"""
|
||||
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
||||
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
||||
|
||||
s3_client = get_s3_client()
|
||||
if not s3_client:
|
||||
return []
|
||||
|
||||
try:
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
pages = paginator.paginate(Bucket=bucket_name, Prefix='avatars/')
|
||||
|
||||
all_objects = {}
|
||||
for page in pages:
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj['Key']
|
||||
base = key.rsplit('.', 1)[0]
|
||||
if base not in all_objects:
|
||||
all_objects[base] = {}
|
||||
if key.lower().endswith(('.png', '.jpg', '.jpeg', '.webp')):
|
||||
all_objects[base]['image'] = obj
|
||||
elif key.endswith('.json'):
|
||||
all_objects[base]['meta_key'] = key
|
||||
|
||||
images = []
|
||||
for base, data in all_objects.items():
|
||||
if 'image' not in data:
|
||||
continue
|
||||
obj = data['image']
|
||||
key = obj['Key']
|
||||
public_url = f"https://{bucket_name}.s3.{region}.amazonaws.com/{key}"
|
||||
entry = {
|
||||
"url": public_url,
|
||||
"key": key,
|
||||
"created_at": obj['LastModified'].isoformat(),
|
||||
"description": "",
|
||||
}
|
||||
# Try to read metadata JSON
|
||||
if 'meta_key' in data:
|
||||
try:
|
||||
meta_resp = s3_client.get_object(Bucket=bucket_name, Key=data['meta_key'])
|
||||
meta = json.loads(meta_resp['Body'].read().decode('utf-8'))
|
||||
entry['description'] = meta.get('description', '')
|
||||
except Exception:
|
||||
pass
|
||||
images.append(entry)
|
||||
|
||||
images.sort(key=lambda x: x['created_at'], reverse=True)
|
||||
return images
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list actor gallery: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# ── SaaS Video Gallery (public S3) ──────────────────────────────────
|
||||
|
||||
_video_gallery_cache = {
|
||||
"data": None,
|
||||
"timestamp": 0,
|
||||
}
|
||||
|
||||
def upload_video_to_gallery(video_path, actor_image_path, metadata, video_id=None):
|
||||
"""
|
||||
Upload a generated UGC video + actor + metadata to the public S3 bucket.
|
||||
Returns dict with public URLs or None on failure.
|
||||
"""
|
||||
import uuid
|
||||
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
||||
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
||||
|
||||
s3_client = get_s3_client()
|
||||
if not s3_client:
|
||||
return None
|
||||
|
||||
if not video_id:
|
||||
video_id = str(uuid.uuid4())[:8]
|
||||
|
||||
base_url = f"https://{bucket_name}.s3.{region}.amazonaws.com"
|
||||
results = {}
|
||||
|
||||
try:
|
||||
# Upload video
|
||||
if os.path.exists(video_path):
|
||||
s3_key = f"videos/{video_id}/video.mp4"
|
||||
s3_client.upload_file(video_path, bucket_name, s3_key,
|
||||
ExtraArgs={'ContentType': 'video/mp4'})
|
||||
results["video_url"] = f"{base_url}/{s3_key}"
|
||||
|
||||
# Upload actor image
|
||||
if actor_image_path and os.path.exists(actor_image_path):
|
||||
s3_key = f"videos/{video_id}/actor.png"
|
||||
s3_client.upload_file(actor_image_path, bucket_name, s3_key,
|
||||
ExtraArgs={'ContentType': 'image/png'})
|
||||
results["actor_url"] = f"{base_url}/{s3_key}"
|
||||
|
||||
# Build and upload metadata
|
||||
import datetime
|
||||
metadata["video_id"] = video_id
|
||||
metadata["video_url"] = results.get("video_url", "")
|
||||
metadata["actor_url"] = results.get("actor_url", "")
|
||||
metadata["created_at"] = datetime.datetime.utcnow().isoformat() + "Z"
|
||||
|
||||
meta_json = json.dumps(metadata, ensure_ascii=False, indent=2)
|
||||
s3_key = f"videos/{video_id}/metadata.json"
|
||||
s3_client.put_object(
|
||||
Bucket=bucket_name, Key=s3_key,
|
||||
Body=meta_json.encode('utf-8'),
|
||||
ContentType='application/json',
|
||||
)
|
||||
results["metadata_url"] = f"{base_url}/{s3_key}"
|
||||
results["video_id"] = video_id
|
||||
|
||||
logger.info(f"Uploaded video gallery: {video_id}")
|
||||
|
||||
# Invalidate cache
|
||||
_video_gallery_cache["data"] = None
|
||||
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload video to gallery: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def list_video_gallery(limit=50, force_refresh=False):
|
||||
"""
|
||||
List all UGC videos from the public S3 bucket.
|
||||
Returns list of metadata dicts, newest first.
|
||||
"""
|
||||
global _video_gallery_cache
|
||||
|
||||
now = time_module.time()
|
||||
if not force_refresh and _video_gallery_cache["data"] is not None:
|
||||
if now - _video_gallery_cache["timestamp"] < CACHE_TTL_SECONDS:
|
||||
cached = _video_gallery_cache["data"]
|
||||
return cached[:limit] if limit else cached
|
||||
|
||||
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
||||
|
||||
s3_client = get_s3_client()
|
||||
if not s3_client:
|
||||
return []
|
||||
|
||||
videos = []
|
||||
|
||||
try:
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
pages = paginator.paginate(Bucket=bucket_name, Prefix='videos/')
|
||||
|
||||
meta_files = []
|
||||
for page in pages:
|
||||
for obj in page.get('Contents', []):
|
||||
if obj['Key'].endswith('/metadata.json'):
|
||||
meta_files.append(obj)
|
||||
|
||||
# Newest first
|
||||
meta_files.sort(key=lambda x: x['LastModified'], reverse=True)
|
||||
|
||||
for meta_obj in meta_files:
|
||||
try:
|
||||
obj_resp = s3_client.get_object(Bucket=bucket_name, Key=meta_obj['Key'])
|
||||
content = obj_resp['Body'].read().decode('utf-8')
|
||||
data = json.loads(content)
|
||||
videos.append(data)
|
||||
if limit and len(videos) >= limit:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading metadata {meta_obj['Key']}: {e}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list video gallery: {e}")
|
||||
return []
|
||||
|
||||
_video_gallery_cache["data"] = videos
|
||||
_video_gallery_cache["timestamp"] = now
|
||||
|
||||
return videos[:limit] if limit else videos
|
||||
|
||||
|
||||
def upload_job_artifacts(directory, job_id):
|
||||
"""
|
||||
Upload all generated clips and metadata for a job to S3.
|
||||
"""
|
||||
bucket_name = os.environ.get('AWS_S3_BUCKET', 'my-clips-bucket')
|
||||
|
||||
if not os.path.exists(directory):
|
||||
return
|
||||
|
||||
for filename in os.listdir(directory):
|
||||
# Upload .mp4 clips and the metadata JSON
|
||||
if (filename.endswith(".mp4") or filename.endswith(".json")) and not filename.startswith("temp_"):
|
||||
file_path = os.path.join(directory, filename)
|
||||
s3_key = f"{job_id}/{filename}"
|
||||
upload_file_to_s3(file_path, bucket_name, s3_key)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user