447 lines
15 KiB
Python
447 lines
15 KiB
Python
import os
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
import boto3
|
|
from botocore.exceptions import ClientError
|
|
import logging
|
|
|
|
# Configure silent logging for boto3 and botocore
|
|
logging.getLogger('boto3').setLevel(logging.CRITICAL)
|
|
logging.getLogger('botocore').setLevel(logging.CRITICAL)
|
|
logging.getLogger('s3transfer').setLevel(logging.CRITICAL)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.INFO)
|
|
|
|
def upload_file_to_s3(file_path, bucket_name, s3_key):
|
|
"""
|
|
Upload a file to an S3 bucket silently.
|
|
"""
|
|
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
|
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
|
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
|
|
|
if not access_key or not secret_key:
|
|
return False
|
|
|
|
s3_client = boto3.client(
|
|
's3',
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
region_name=region
|
|
)
|
|
try:
|
|
# Extra arguments for public read if needed, but the user didn't specify.
|
|
# Given the bucket name, it might be for a web app.
|
|
s3_client.upload_file(file_path, bucket_name, s3_key)
|
|
return True
|
|
except ClientError:
|
|
return False
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
from botocore.config import Config
|
|
import json
|
|
import time as time_module
|
|
|
|
# Simple in-memory cache for gallery clips
|
|
_clips_cache = {
|
|
"data": None,
|
|
"timestamp": 0
|
|
}
|
|
CACHE_TTL_SECONDS = 300 # 5 minutes
|
|
|
|
def get_s3_client():
|
|
"""Returns an authenticated S3 client."""
|
|
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
|
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
|
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
|
|
|
if not access_key or not secret_key:
|
|
return None
|
|
|
|
return boto3.client(
|
|
's3',
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
region_name=region,
|
|
config=Config(signature_version='s3v4')
|
|
)
|
|
|
|
def generate_presigned_url(bucket_name, object_key, expiration=3600):
|
|
"""Generate a presigned URL to share an S3 object."""
|
|
s3_client = get_s3_client()
|
|
if not s3_client:
|
|
return None
|
|
try:
|
|
response = s3_client.generate_presigned_url('get_object',
|
|
Params={'Bucket': bucket_name,
|
|
'Key': object_key},
|
|
ExpiresIn=expiration)
|
|
return response
|
|
except ClientError as e:
|
|
logger.error(e)
|
|
return None
|
|
|
|
def list_all_clips(bucket_name=None, limit=50, force_refresh=False):
|
|
"""
|
|
List recent clips from the S3 bucket by finding metadata files.
|
|
Returns a list of dicts containing clip info and signed URLs.
|
|
|
|
Args:
|
|
bucket_name: S3 bucket name (defaults to AWS_S3_BUCKET env var)
|
|
limit: Maximum number of clips to return (default 50 for speed)
|
|
force_refresh: If True, bypass cache
|
|
"""
|
|
global _clips_cache
|
|
|
|
# Check cache first
|
|
now = time_module.time()
|
|
if not force_refresh and _clips_cache["data"] is not None:
|
|
if now - _clips_cache["timestamp"] < CACHE_TTL_SECONDS:
|
|
cached = _clips_cache["data"]
|
|
return cached[:limit] if limit else cached
|
|
|
|
if not bucket_name:
|
|
bucket_name = os.environ.get('AWS_S3_BUCKET', 'my-clips-bucket')
|
|
|
|
s3_client = get_s3_client()
|
|
if not s3_client:
|
|
return []
|
|
|
|
all_clips = []
|
|
|
|
try:
|
|
# List all objects in bucket
|
|
# Note: For very large buckets, pagination is needed.
|
|
# Assuming reasonable size for now, but adding continuation token support is best practice.
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
|
pages = paginator.paginate(Bucket=bucket_name)
|
|
|
|
metadata_files = []
|
|
for page in pages:
|
|
if 'Contents' in page:
|
|
for obj in page['Contents']:
|
|
if obj['Key'].endswith('_metadata.json'):
|
|
metadata_files.append(obj)
|
|
|
|
# Sort metadata by LastModified (newest first)
|
|
metadata_files.sort(key=lambda x: x['LastModified'], reverse=True)
|
|
|
|
for meta_obj in metadata_files:
|
|
key = meta_obj['Key']
|
|
# key format: {job_id}/..._metadata.json
|
|
|
|
# Read metadata content
|
|
try:
|
|
obj_resp = s3_client.get_object(Bucket=bucket_name, Key=key)
|
|
content = obj_resp['Body'].read().decode('utf-8')
|
|
data = json.loads(content)
|
|
|
|
parts = key.split('/')
|
|
job_id = parts[0] if len(parts) > 1 else "unknown"
|
|
# Filename base for clips in same folder
|
|
# Meta key: "job_id/filename_metadata.json"
|
|
# Base name in metadata usually matches filename without ext
|
|
meta_filename = os.path.basename(key)
|
|
base_name = meta_filename.replace('_metadata.json', '')
|
|
|
|
clips_data = data.get('shorts', [])
|
|
|
|
for i, clip in enumerate(clips_data):
|
|
clip_filename = f"{base_name}_clip_{i+1}.mp4"
|
|
clip_key = f"{job_id}/{clip_filename}"
|
|
|
|
# Generate signed URL
|
|
signed_url = generate_presigned_url(bucket_name, clip_key, expiration=7200) # 2 hours
|
|
|
|
if signed_url:
|
|
all_clips.append({
|
|
"job_id": job_id,
|
|
"index": i,
|
|
"url": signed_url,
|
|
"title": clip.get('video_title_for_youtube_short', 'Untitled Clip'),
|
|
"tiktok_desc": clip.get('video_description_for_tiktok', ''),
|
|
"insta_desc": clip.get('video_description_for_instagram', ''),
|
|
"created_at": meta_obj['LastModified'].isoformat(),
|
|
"duration": clip.get('end', 0) - clip.get('start', 0)
|
|
})
|
|
|
|
# Early exit if we have enough clips
|
|
if limit and len(all_clips) >= limit:
|
|
break
|
|
|
|
# Early exit if we have enough clips
|
|
if limit and len(all_clips) >= limit:
|
|
break
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing metadata {key}: {e}")
|
|
continue
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error listing bucket: {e}")
|
|
return []
|
|
|
|
# Update cache with full results (keep for pagination later)
|
|
_clips_cache["data"] = all_clips
|
|
_clips_cache["timestamp"] = now
|
|
|
|
return all_clips[:limit] if limit else all_clips
|
|
|
|
def upload_actor_to_s3(file_path, description=""):
|
|
"""
|
|
Upload an actor image to the public S3 bucket.
|
|
Returns the public URL or None on failure.
|
|
"""
|
|
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
|
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
|
|
|
s3_client = get_s3_client()
|
|
if not s3_client:
|
|
return None
|
|
|
|
import uuid
|
|
unique_id = str(uuid.uuid4())[:8]
|
|
filename = os.path.basename(file_path)
|
|
name, ext = os.path.splitext(filename)
|
|
s3_key = f"avatars/{name}_{unique_id}{ext}"
|
|
|
|
try:
|
|
# Skip broken/tiny files
|
|
if os.path.getsize(file_path) < 1000:
|
|
logger.warning(f"Skipping tiny file ({os.path.getsize(file_path)} bytes): {file_path}")
|
|
return None
|
|
|
|
s3_client.upload_file(
|
|
file_path, bucket_name, s3_key,
|
|
ExtraArgs={'ContentType': 'image/png'},
|
|
)
|
|
public_url = f"https://{bucket_name}.s3.{region}.amazonaws.com/{s3_key}"
|
|
|
|
# Save metadata JSON alongside the image
|
|
if description:
|
|
import datetime
|
|
meta_key = s3_key.rsplit('.', 1)[0] + '.json'
|
|
meta = json.dumps({
|
|
"description": description,
|
|
"url": public_url,
|
|
"created_at": datetime.datetime.utcnow().isoformat() + "Z",
|
|
}, ensure_ascii=False)
|
|
s3_client.put_object(
|
|
Bucket=bucket_name, Key=meta_key,
|
|
Body=meta.encode('utf-8'),
|
|
ContentType='application/json',
|
|
)
|
|
|
|
logger.info(f"Uploaded actor to S3: {public_url}")
|
|
return public_url
|
|
except Exception as e:
|
|
logger.error(f"Failed to upload actor to S3: {e}")
|
|
return None
|
|
|
|
|
|
def list_actor_gallery():
|
|
"""
|
|
List all actor images from the public S3 bucket.
|
|
Returns list with URLs and descriptions, newest first.
|
|
"""
|
|
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
|
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
|
|
|
s3_client = get_s3_client()
|
|
if not s3_client:
|
|
return []
|
|
|
|
try:
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
|
pages = paginator.paginate(Bucket=bucket_name, Prefix='avatars/')
|
|
|
|
all_objects = {}
|
|
for page in pages:
|
|
for obj in page.get('Contents', []):
|
|
key = obj['Key']
|
|
base = key.rsplit('.', 1)[0]
|
|
if base not in all_objects:
|
|
all_objects[base] = {}
|
|
if key.lower().endswith(('.png', '.jpg', '.jpeg', '.webp')):
|
|
all_objects[base]['image'] = obj
|
|
elif key.endswith('.json'):
|
|
all_objects[base]['meta_key'] = key
|
|
|
|
images = []
|
|
for base, data in all_objects.items():
|
|
if 'image' not in data:
|
|
continue
|
|
obj = data['image']
|
|
key = obj['Key']
|
|
public_url = f"https://{bucket_name}.s3.{region}.amazonaws.com/{key}"
|
|
entry = {
|
|
"url": public_url,
|
|
"key": key,
|
|
"created_at": obj['LastModified'].isoformat(),
|
|
"description": "",
|
|
}
|
|
# Try to read metadata JSON
|
|
if 'meta_key' in data:
|
|
try:
|
|
meta_resp = s3_client.get_object(Bucket=bucket_name, Key=data['meta_key'])
|
|
meta = json.loads(meta_resp['Body'].read().decode('utf-8'))
|
|
entry['description'] = meta.get('description', '')
|
|
except Exception:
|
|
pass
|
|
images.append(entry)
|
|
|
|
images.sort(key=lambda x: x['created_at'], reverse=True)
|
|
return images
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to list actor gallery: {e}")
|
|
return []
|
|
|
|
|
|
# ── SaaS Video Gallery (public S3) ──────────────────────────────────
|
|
|
|
_video_gallery_cache = {
|
|
"data": None,
|
|
"timestamp": 0,
|
|
}
|
|
|
|
def upload_video_to_gallery(video_path, actor_image_path, metadata, video_id=None):
|
|
"""
|
|
Upload a generated UGC video + actor + metadata to the public S3 bucket.
|
|
Returns dict with public URLs or None on failure.
|
|
"""
|
|
import uuid
|
|
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
|
region = os.environ.get('AWS_REGION', 'eu-west-3')
|
|
|
|
s3_client = get_s3_client()
|
|
if not s3_client:
|
|
return None
|
|
|
|
if not video_id:
|
|
video_id = str(uuid.uuid4())[:8]
|
|
|
|
base_url = f"https://{bucket_name}.s3.{region}.amazonaws.com"
|
|
results = {}
|
|
|
|
try:
|
|
# Upload video
|
|
if os.path.exists(video_path):
|
|
s3_key = f"videos/{video_id}/video.mp4"
|
|
s3_client.upload_file(video_path, bucket_name, s3_key,
|
|
ExtraArgs={'ContentType': 'video/mp4'})
|
|
results["video_url"] = f"{base_url}/{s3_key}"
|
|
|
|
# Upload actor image
|
|
if actor_image_path and os.path.exists(actor_image_path):
|
|
s3_key = f"videos/{video_id}/actor.png"
|
|
s3_client.upload_file(actor_image_path, bucket_name, s3_key,
|
|
ExtraArgs={'ContentType': 'image/png'})
|
|
results["actor_url"] = f"{base_url}/{s3_key}"
|
|
|
|
# Build and upload metadata
|
|
import datetime
|
|
metadata["video_id"] = video_id
|
|
metadata["video_url"] = results.get("video_url", "")
|
|
metadata["actor_url"] = results.get("actor_url", "")
|
|
metadata["created_at"] = datetime.datetime.utcnow().isoformat() + "Z"
|
|
|
|
meta_json = json.dumps(metadata, ensure_ascii=False, indent=2)
|
|
s3_key = f"videos/{video_id}/metadata.json"
|
|
s3_client.put_object(
|
|
Bucket=bucket_name, Key=s3_key,
|
|
Body=meta_json.encode('utf-8'),
|
|
ContentType='application/json',
|
|
)
|
|
results["metadata_url"] = f"{base_url}/{s3_key}"
|
|
results["video_id"] = video_id
|
|
|
|
logger.info(f"Uploaded video gallery: {video_id}")
|
|
|
|
# Invalidate cache
|
|
_video_gallery_cache["data"] = None
|
|
|
|
return results
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to upload video to gallery: {e}")
|
|
return None
|
|
|
|
|
|
def list_video_gallery(limit=50, force_refresh=False):
|
|
"""
|
|
List all UGC videos from the public S3 bucket.
|
|
Returns list of metadata dicts, newest first.
|
|
"""
|
|
global _video_gallery_cache
|
|
|
|
now = time_module.time()
|
|
if not force_refresh and _video_gallery_cache["data"] is not None:
|
|
if now - _video_gallery_cache["timestamp"] < CACHE_TTL_SECONDS:
|
|
cached = _video_gallery_cache["data"]
|
|
return cached[:limit] if limit else cached
|
|
|
|
bucket_name = os.environ.get('AWS_S3_PUBLIC_BUCKET', 'my-public-bucket')
|
|
|
|
s3_client = get_s3_client()
|
|
if not s3_client:
|
|
return []
|
|
|
|
videos = []
|
|
|
|
try:
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
|
pages = paginator.paginate(Bucket=bucket_name, Prefix='videos/')
|
|
|
|
meta_files = []
|
|
for page in pages:
|
|
for obj in page.get('Contents', []):
|
|
if obj['Key'].endswith('/metadata.json'):
|
|
meta_files.append(obj)
|
|
|
|
# Newest first
|
|
meta_files.sort(key=lambda x: x['LastModified'], reverse=True)
|
|
|
|
for meta_obj in meta_files:
|
|
try:
|
|
obj_resp = s3_client.get_object(Bucket=bucket_name, Key=meta_obj['Key'])
|
|
content = obj_resp['Body'].read().decode('utf-8')
|
|
data = json.loads(content)
|
|
videos.append(data)
|
|
if limit and len(videos) >= limit:
|
|
break
|
|
except Exception as e:
|
|
logger.error(f"Error reading metadata {meta_obj['Key']}: {e}")
|
|
continue
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to list video gallery: {e}")
|
|
return []
|
|
|
|
_video_gallery_cache["data"] = videos
|
|
_video_gallery_cache["timestamp"] = now
|
|
|
|
return videos[:limit] if limit else videos
|
|
|
|
|
|
def upload_job_artifacts(directory, job_id):
|
|
"""
|
|
Upload all generated clips and metadata for a job to S3.
|
|
"""
|
|
bucket_name = os.environ.get('AWS_S3_BUCKET', 'my-clips-bucket')
|
|
|
|
if not os.path.exists(directory):
|
|
return
|
|
|
|
for filename in os.listdir(directory):
|
|
# Upload .mp4 clips and the metadata JSON
|
|
if (filename.endswith(".mp4") or filename.endswith(".json")) and not filename.startswith("temp_"):
|
|
file_path = os.path.join(directory, filename)
|
|
s3_key = f"{job_id}/{filename}"
|
|
upload_file_to_s3(file_path, bucket_name, s3_key)
|
|
|
|
|