From 1db00c41821cd6708ab3719f14b7ad427cc51d26 Mon Sep 17 00:00:00 2001 From: MythEclipse Date: Tue, 22 Sep 2026 21:03:22 +0700 Subject: [PATCH] feat(shorts): cinematic v2 visual design + synced karaoke captions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix critical bug: scenes were stacked without — every segment rendered simultaneously, last scene covering the whole video (root cause of 'too simple/ugly' visual) - Scene variety: kinetic word-by-word title (hook), karaoke caption card, big-number stat scene (auto-detected: $12B, 65%, 3.2M), CTA center - Karaoke captions synced to TTS: edge-tts SentenceBoundary timing via new scripts/tts-words.py (mp3 + sentences.json per segment); active sentence bright, others dimmed - Rich background per scene: 4 color palettes (indigo/pink, cyan/teal, purple/pink, orange/red), drifting orbs, dot grid, ken-burns motion - Real fonts: @fontsource Inter 700 + Montserrat 800 (display/captions) - Glass card captions, progress bar per-scene, safe zone 900x1400 - Word-boundary truncation for stat labels (no mid-word cuts) - Perf kept: no blur/SVG-noise (software renderer); ~2 min per 32s short --- package-lock.json | 20 ++ package.json | 2 + scripts/render-worker.mjs | 48 ++++- scripts/tts-words.py | 33 ++++ src/remotion/ShortsVideo.tsx | 192 ++++++++++++++------ src/remotion/parts.tsx | 342 +++++++++++++++++++++++++---------- 6 files changed, 479 insertions(+), 158 deletions(-) create mode 100644 scripts/tts-words.py diff --git a/package-lock.json b/package-lock.json index e500722..8e23d22 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,8 @@ "hasInstallScript": true, "dependencies": { "@auth/prisma-adapter": "^1.6.0", + "@fontsource/inter": "^5.3.0", + "@fontsource/montserrat": "^5.3.0", "@prisma/client": "^6.19.2", "@remotion/cli": "^4.0.527", "@remotion/google-fonts": "^4.0.527", @@ -1047,6 +1049,24 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@fontsource/inter": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/inter/-/inter-5.3.0.tgz", + "integrity": "sha512-RofMylZmjlJEfELXeNHFWBRcSs75rGU/6bV2S2jfnvv/3rPXPGe0LgUJTklcHZ9lM4OZmAVFhcJPnACfb91A3g==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, + "node_modules/@fontsource/montserrat": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource/montserrat/-/montserrat-5.3.0.tgz", + "integrity": "sha512-iQPDtZOnmM5ih4JBGaMSXisRap0ixb9bAUw2hDtY7EjVfaqiiNlKtVuJl9XclxUwtwMMHNcWXfQR6zI9x05+Tg==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, "node_modules/@grammyjs/types": { "version": "3.28.0", "resolved": "https://registry.npmjs.org/@grammyjs/types/-/types-3.28.0.tgz", diff --git a/package.json b/package.json index 9fe9d0f..6c2ac68 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ }, "dependencies": { "@auth/prisma-adapter": "^1.6.0", + "@fontsource/inter": "^5.3.0", + "@fontsource/montserrat": "^5.3.0", "@prisma/client": "^6.19.2", "@remotion/cli": "^4.0.527", "@remotion/google-fonts": "^4.0.527", diff --git a/scripts/render-worker.mjs b/scripts/render-worker.mjs index 16e32be..f4a3f31 100644 --- a/scripts/render-worker.mjs +++ b/scripts/render-worker.mjs @@ -17,7 +17,7 @@ */ import { PrismaClient } from "@prisma/client"; import { spawnSync } from "child_process"; -import { existsSync, mkdirSync, writeFileSync } from "fs"; +import { existsSync, mkdirSync, writeFileSync, readFileSync } from "fs"; import path from "path"; const prisma = new PrismaClient(); @@ -39,6 +39,12 @@ mkdirSync(SCRATCH, { recursive: true }); const REMOTION_ROOT = path.resolve(process.env.REMOTION_ROOT || "./src/remotion"); const REMOTION_ENTRY = path.join(REMOTION_ROOT, "index.tsx"); +const TTS_PYTHON = + process.env.TTS_PYTHON || + (existsSync("/home/code/.hermes/hermes-agent/venv/bin/python3") + ? "/home/code/.hermes/hermes-agent/venv/bin/python3" + : "python3"); +const TTS_WORDS_PY = path.resolve(process.env.TTS_WORDS_PY || "./scripts/tts-words.py"); function sh(cmd, args, opts = {}) { const r = spawnSync(cmd, args, { @@ -54,11 +60,18 @@ function sh(cmd, args, opts = {}) { function tts(text, outBase) { const out = `${outBase}.mp3`; - sh(EDGE_TTS_BIN, ["--voice", VOICE, "--text", text, "--write-media", out], { - timeout: 60_000, - env: { ...process.env }, - }); - return { file: out, durationSec: ffprobeDuration(out) }; + const srt = `${outBase}.sentences.json`; + sh( + TTS_PYTHON, + [TTS_WORDS_PY, text, VOICE, out, srt], + { timeout: 60_000, env: { ...process.env } } + ); + let sentences = []; + try { + sentences = JSON.parse(readFileSync(srt, "utf-8")); + } catch { /* fallback: single sentence */ } + if (!sentences.length) sentences = [{ w: text, start: 0, end: 0 }]; + return { file: out, durationSec: ffprobeDuration(out), sentences }; } function ffprobeDuration(file) { @@ -81,6 +94,23 @@ function ffprobeSize(file) { function log(...a) { console.log(new Date().toISOString(), ...a); } +/** Truncate at a word boundary (never cut mid-word). */ +function clipWords(s, max) { + if (s.length <= max) return s; + const cut = s.slice(0, max); + const sp = cut.lastIndexOf(" "); + return (sp > max * 0.5 ? cut.slice(0, sp) : cut).trim() + "…"; +} + +/** Detect a stat worth rendering as a big-number scene: $12B, 65%, 3.2M. */ +function extractStatData(text) { + const m = text.match(/(\$?\d+(?:\.\d+)?\s?(?:[BMK]%|billion|million|%))/i); + if (!m) return null; + const rest = text.replace(m[0], "").replace(/^[^A-Za-z0-9]+/, "").trim(); + if (rest.length < 5) return null; + return { value: m[1].trim(), label: clipWords(rest, 70) }; +} + /** Build segment text list from a YoutubeScript's structured fields. */ function buildSegments(script) { const segs = []; @@ -133,12 +163,12 @@ async function renderScript(script) { } // 3. Remotion timeline (frames) - const timeline = voiced.map((s, i) => ({ + const timeline = voiced.map((s) => ({ text: s.text, duration: Math.max(30, Math.round(s.durationSec * FPS)), - size: baseSegs.length <= 4 ? 80 : 68, label: s.label, - highlight: s.highlight || undefined, + sentences: s.sentences || undefined, + stat: extractStatData(s.text), })); // 4. Render silent video via Remotion CLI (props = segments JSON) diff --git a/scripts/tts-words.py b/scripts/tts-words.py new file mode 100644 index 0000000..61fb94a --- /dev/null +++ b/scripts/tts-words.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""TTS with per-sentence timing (karaoke captions). + +Usage: tts-words.py +Writes mp3 + JSON [{w, start, end}] (seconds, relative to segment start). +Uses edge-tts SentenceBoundary stream events. +""" +import asyncio +import json +import sys + +import edge_tts + + +async def main(text: str, voice: str, mp3_path: str, sentences_path: str) -> None: + comm = edge_tts.Communicate(text, voice) + sentences = [] + with open(mp3_path, "wb") as f: + async for chunk in comm.stream(): + if chunk.get("type") == "audio": + f.write(chunk.get("data", b"")) + elif chunk.get("type") == "SentenceBoundary": + start = chunk.get("offset", 0) / 1e7 + dur = chunk.get("duration", 0) / 1e7 + sentences.append( + {"w": chunk.get("text", ""), "start": round(start, 3), "end": round(start + dur, 3)} + ) + with open(sentences_path, "w") as f: + json.dump(sentences, f) + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])) \ No newline at end of file diff --git a/src/remotion/ShortsVideo.tsx b/src/remotion/ShortsVideo.tsx index 3f5a7ce..94cff0e 100644 --- a/src/remotion/ShortsVideo.tsx +++ b/src/remotion/ShortsVideo.tsx @@ -1,51 +1,145 @@ import React from "react"; -import { AbsoluteFill } from "remotion"; -import { CaptionLine, SvgBackground, ProgressBar, BrandTag } from "./parts"; +import { AbsoluteFill, Sequence } from "remotion"; +import { + SvgBackground, + BrandTag, + ProgressBar, + GlassCard, + CaptionKaraoke, + KineticTitle, + StatCounter, + SlideMotion, +} from "./parts"; -/** A single timed segment: one text card rendered for `duration` frames. */ -const Segment: React.FC<{ +/** Sentence timing (karaoke), relative to segment start. */ +export type Sentence = { w: string; start: number; end: number }; + +export type Segment = { text: string; - highlight?: string; + duration: number; // frames label?: string; - size: number; - from: number; - duration: number; -}> = ({ text, highlight, label, size, from, duration }) => ( - - - {label ? : null} - - - {/* current time indicator */} -
- {(from / 30).toFixed(2)}s -
- -); + sentences?: Sentence[]; + /** optional explicit scene override */ + scene?: "title" | "stat" | "caption" | "cta"; + stat?: { value: string; label: string }; +}; -/** Composition props driven by the worker (JSON). */ -export const ShortsVideo: React.FC<{ - segments: { - text: string; - duration: number; // frames - highlight?: string; - label?: string; - size?: number; - }[]; - from?: number; - duration?: number; -}> = ({ segments }) => { - const baseSize = 68; +/** Truncate at a word boundary (never cut mid-word). */ +function clipWords(s: string, max: number): string { + if (s.length <= max) return s; + const cut = s.slice(0, max); + const sp = cut.lastIndexOf(" "); + return (sp > max * 0.5 ? cut.slice(0, sp) : cut).trim() + "…"; +} + +/** Detect a big number/stat worth showing as a stat scene. */ +function extractStat(text: string): { value: string; label: string } | null { + const m = text.match(/(\$?\d+(?:[.,]\d+)?\s?[BMTK%]?)\s+(?:million|billion|trillion|percent|people|users|x)?/i); + if (m && /\d/.test(m[1]) && /[$%]|\d\s?[BMTK]\b/i.test(m[1])) { + const rest = text.replace(m[0], "").trim(); + const label = clipWords(rest, 60); + if (label) return { value: m[1].trim(), label }; + } + if (/\$?\d+(\.\d+)?\s?(billion|B|%|million|M)\b/i.test(text)) { + const m2 = text.match(/(\$?\d+(?:\.\d+)?\s?(?:billion|B|%|million|M))/i); + if (m2) { + const rest = text.replace(m2[0], "").trim(); + return { value: m2[1], label: clipWords(rest, 60) || "…" }; + } + } + return null; +} + +/** Pick a scene for this segment (variety — pattern interrupt every cut). */ +function pickScene(seg: Segment, index: number): "title" | "stat" | "caption" | "cta" { + if (seg.scene) return seg.scene; + const label = (seg.label || "").toUpperCase(); + if (label === "HOOK" || label === "ON SCREEN") return "title"; + if (label === "CTA") return "cta"; + if (seg.stat) return "stat"; + const stat = extractStat(seg.text); + if (stat) return "stat"; + if (index % 3 === 2) return "title"; + return "caption"; +} + +/** One scene rendered for `duration` frames. */ +const Scene: React.FC<{ + seg: Segment; + index: number; + from: number; +}> = ({ seg, index, from }) => { + const { duration, label, text } = seg; + const scene = pickScene(seg, index); + const stat = seg.stat || (scene === "stat" ? extractStat(text) : null); + const sentences = + seg.sentences && seg.sentences.length + ? seg.sentences + : [{ w: text, start: 0, end: duration / 30 }]; + + return ( + + + + {label ? : null} + + {scene === "stat" && stat ? ( + + ) : scene === "title" || scene === "cta" ? ( + <> + + + + + + ) : ( + <> + + + + + + )} + + + {/* time code (subtle, right side) */} +
+ {(from / 30).toFixed(1)}s +
+ + + ); +}; + +/** Section headline above the caption card. */ +function titleFor(seg: Segment): string { + const label = (seg.label || "").toUpperCase(); + const map: Record = { + "THE STORY": "The story", + "THE CONFLICT": "The twist", + "THE INSIGHT": "The lesson", + }; + if (map[label]) return map[label]; + return label ? label.charAt(0) + label.slice(1).toLowerCase() : ""; +} + +/** Composition: each scene time-gated by Sequence (frame resets per scene). */ +export const ShortsVideo: React.FC<{ segments: Segment[] }> = ({ segments }) => { let acc = 0; return ( <> @@ -53,15 +147,9 @@ export const ShortsVideo: React.FC<{ const from = acc; acc += s.duration; return ( - + + + ); })} diff --git a/src/remotion/parts.tsx b/src/remotion/parts.tsx index 307f2f7..f1db90a 100644 --- a/src/remotion/parts.tsx +++ b/src/remotion/parts.tsx @@ -4,134 +4,290 @@ import { useCurrentFrame, useVideoConfig, interpolate, - Easing, spring, } from "remotion"; +import "@fontsource/inter/700.css"; +import "@fontsource/montserrat/800.css"; -/** Animated diagonal gradient background + soft floating blobs. */ -export const SvgBackground: React.FC = () => { +/* ───────────────────────────────────────────────────────────────────────── + Shorts design system. + Fonts: Inter 700 (captions/body), Montserrat 800 (display). + Safe zone: 900×1400 centered (universal TikTok/Reels/Shorts). + Perf rule: NO blur filters / SVG noise (software Chrome ~10x slower). + ───────────────────────────────────────────────────────────────────────── */ + +export const FONT_DISPLAY = "Montserrat, Inter, system-ui, sans-serif"; +export const FONT_BODY = "Inter, system-ui, sans-serif"; + +/** Per-segment background palette (variety between scenes). */ +const PALETTES = [ + { base: "linear-gradient(155deg,#0b1020 0%,#141a33 45%,#1b1030 100%)", a1: "99,102,241", a2: "236,72,153" }, + { base: "linear-gradient(155deg,#0a1628 0%,#0e2a3a 45%,#0a1f2e 100%)", a1: "56,189,248", a2: "16,185,129" }, + { base: "linear-gradient(155deg,#160b20 0%,#2a1040 45%,#1b0a2e 100%)", a1: "168,85,247", a2: "244,114,182" }, + { base: "linear-gradient(155deg,#1a0f0b 0%,#2e1a12 45%,#21100a 100%)", a1: "251,146,60", a2: "239,68,68" }, +]; + +/** Animated background: gradient + drifting pre-softened orbs + dot grid. */ +export const SvgBackground: React.FC<{ variant?: number }> = ({ variant = 0 }) => { const frame = useCurrentFrame(); const { width, height } = useVideoConfig(); - const drift = (speed: number, amp: number) => - Math.sin((frame / 60) * speed) * amp; + const t = frame / 60; + const p = PALETTES[variant % PALETTES.length]; return ( - - - {/* floating gradient blobs (radial gradients are pre-softened — no blur filter) */} +
+ {/* faint dot grid */} +
); }; -/** Big bold caption line with spring-in + fade + word-by-word pop. */ -export const CaptionLine: React.FC<{ +/** Glass card (translucent, rounded, border) for caption blocks. */ +export const GlassCard: React.FC<{ + children: React.ReactNode; + style?: React.CSSProperties; +}> = ({ children, style }) => { + const frame = useCurrentFrame(); + const { fps } = useVideoConfig(); + const pop = spring({ frame, fps, config: { damping: 15, stiffness: 110 } }); + return ( +
+ {children} +
+ ); +}; + +/** Karaoke caption: active sentence full-bright, others dimmed (synced to TTS). */ +export const CaptionKaraoke: React.FC<{ + sentences: { w: string; start: number; end: number }[]; + size?: number; +}> = ({ sentences, size = 44 }) => { + const frame = useCurrentFrame(); + const { fps } = useVideoConfig(); + const now = frame / fps; + + let activeIdx = 0; + for (let i = 0; i < sentences.length; i++) { + if (now >= sentences[i].start) { activeIdx = i; break; } + } + const active = sentences[activeIdx]; + const flash = active + ? interpolate(now, [active.start, active.start + 0.12, active.start + 0.35], [1, 1.06, 1], { + extrapolateLeft: "clamp", + extrapolateRight: "clamp", + }) + : 1; + + return ( +
+ {sentences.map((s, i) => ( + + {s.w} + {i < sentences.length - 1 ? " " : ""} + + ))} +
+ ); +}; + +/** Big display title with word-by-word kinetic entrance. */ +export const KineticTitle: React.FC<{ text: string; - highlight?: string; - size: number; -}> = ({ text, highlight, size }) => { + size?: number; + accentIndex?: number; + accentColor?: string; + position?: "top" | "center"; +}> = ({ text, size = 88, accentIndex = -1, accentColor = "#a5b4fc", position = "top" }) => { + const frame = useCurrentFrame(); + const { fps } = useVideoConfig(); + const pop = spring({ frame, fps, config: { damping: 13, stiffness: 100 } }); + const words = text.split(" "); + const dur = Math.max(1, Math.floor(fps * 0.15)); + + return ( +
+ {words.map((w, i) => ( + + {w} + + ))} +
+ ); +}; + +/** Huge stat/number card (pattern interrupt for facts). */ +export const StatCounter: React.FC<{ + value: string; + label: string; + color?: string; +}> = ({ value, label, color = "#a5b4fc" }) => { const frame = useCurrentFrame(); const { fps } = useVideoConfig(); const pop = spring({ frame, fps, config: { damping: 14, stiffness: 90 } }); - const enter = interpolate(frame, [0, 12], [0, 1], { - extrapolateRight: "clamp", - }); - - const words = text.split(" "); - const charDur = Math.max(2, Math.floor(fps * 0.13)); - return (
- {words.map((w, i) => ( - - {w} - - ))} + {value} +
+
+ {label}
); }; -/** Bottom progress bar that fills over the segment duration. */ -export const ProgressBar: React.FC<{ total: number }> = ({ total }) => { +/** Top-left scene label pill. */ +export const BrandTag: React.FC<{ label: string; variant?: number }> = ({ label, variant = 0 }) => { + const p = PALETTES[variant % PALETTES.length]; + return ( +
+ {label} +
+ ); +}; + +/** Bottom progress bar (per-segment fill). */ +export const ProgressBar: React.FC<{ total: number; variant?: number }> = ({ total, variant = 0 }) => { const frame = useCurrentFrame(); - const pct = interpolate(frame, [0, total], [0, 100], { - extrapolateRight: "clamp", - }); + const p = PALETTES[variant % PALETTES.length]; + const pct = interpolate(frame, [0, total], [0, 100], { extrapolateRight: "clamp" }); return (
@@ -140,31 +296,23 @@ export const ProgressBar: React.FC<{ total: number }> = ({ total }) => { width: `${pct}%`, height: "100%", borderRadius: 99, - background: "linear-gradient(90deg,#6366f1,#ec4899)", + background: `linear-gradient(90deg, rgb(${p.a1}), rgb(${p.a2}))`, }} />
); }; -/** Small brand tag. */ -export const BrandTag: React.FC<{ label: string }> = ({ label }) => ( -
- {label} -
-); \ No newline at end of file +/** Slow ken-burns style camera motion wrapper (transform only — cheap). */ +export const SlideMotion: React.FC<{ children: React.ReactNode }> = ({ children }) => { + const frame = useCurrentFrame(); + const { fps } = useVideoConfig(); + const t = frame / fps; + const y = Math.sin(t * 1.4) * 14; + const scale = 1 + Math.sin(t * 0.8) * 0.014; + return ( + + {children} + + ); +}; \ No newline at end of file