Initial commit: Nat20 Notes — TTRPG session transcription & summarization

backend/app/pipeline/audio.py

@@ -0,0 +1,35 @@
+import subprocess
+from pathlib import Path
+
+from ..errors import AudioExtractionError
+from ..logging_config import get_logger
+
+log = get_logger(__name__)
+
+
+def extract_audio(video_path: Path, audio_path: Path) -> Path:
+    """Extract 16kHz mono wav from any video/audio container."""
+    if not video_path.exists():
+        raise AudioExtractionError(f"Uploaded file not found on disk at {video_path}. It may not have finished uploading, or the upload volume isn't mounted correctly.")
+
+    if audio_path.exists():
+        log.info("Audio already extracted at %s, skipping", audio_path)
+        return audio_path
+
+    log.info("Extracting audio: %s -> %s", video_path, audio_path)
+    result = subprocess.run(
+        [
+            "ffmpeg", "-y", "-i", str(video_path),
+            "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1",
+            str(audio_path),
+        ],
+        capture_output=True, text=True,
+    )
+    if result.returncode != 0:
+        # surface ffmpeg's own error line rather than the whole verbose log
+        stderr_tail = "\n".join(result.stderr.strip().splitlines()[-5:])
+        raise AudioExtractionError(
+            f"ffmpeg failed to extract audio (exit code {result.returncode}). "
+            f"This usually means the file is corrupt or not a supported format. Last output:\n{stderr_tail}"
+        )
+    return audio_path

backend/app/pipeline/summarize.py

@@ -0,0 +1,152 @@
+"""
+Map-reduce summarization: chunk transcript -> per-chunk DM + player summaries
+-> combine into final notes. Backend-agnostic: works against a local Ollama
+instance or any OpenAI-compatible hosted API, selected via settings.
+"""
+from pathlib import Path
+
+import requests
+
+from .turns import chunk_turns, turns_to_text, fmt_time
+from ..errors import SummarizationError
+from ..logging_config import get_logger
+
+log = get_logger(__name__)
+
+DM_CHUNK_PROMPT = """You are summarizing a chunk of a tabletop RPG session transcript for the Game Master's private notes.
+{world_context_block}
+Extract concise bullet points:
+- Major plot events and decisions made
+- NPC interactions (names, what was said/promised/revealed)
+- Combat outcomes (who fought what, notable rolls, deaths/near-deaths)
+- Loot, rewards, or resources gained/lost
+- Any GM secrets, foreshadowing, or plot threads revealed at the table
+- Open questions or hooks left dangling
+
+Be concise and factual. Use the speaker names given. Do not invent anything not in the transcript.
+
+Transcript chunk (time range {time_range}):
+{chunk_text}
+"""
+
+PLAYER_CHUNK_PROMPT = """You are summarizing a chunk of a tabletop RPG session transcript for a player-facing recap.
+{world_context_block}
+Extract concise bullet points of what happened IN THE STORY from the players' in-character perspective:
+- Where the party went, who they met, what was said in-character
+- Combat encounters and outcomes
+- Loot/items the party found
+- Decisions the party made and their in-fiction consequences
+
+Exclude out-of-character rules discussion, GM asides, and anything that reads as a secret not yet revealed to characters.
+If unsure whether something is a spoiler, leave it out.
+
+Transcript chunk (time range {time_range}):
+{chunk_text}
+"""
+
+DM_FINAL_PROMPT = """Combine these chunk summaries from a single session into one cohesive GM session log.
+Organize into sections: Recap, NPC Interactions, Combat & Encounters, Loot & Rewards, Decisions & Consequences, Open Threads / Hooks for Next Session.
+Write in clear prose/bullets, no fluff, no repetition across chunks.
+
+Chunk summaries:
+{summaries}
+"""
+
+PLAYER_FINAL_PROMPT = """Combine these chunk summaries into a single "previously on" style recap for the players, in-character perspective only.
+Write it like a story recap they could reread before the next session. No GM secrets, no OOC content. Keep it engaging but concise.
+
+Chunk summaries:
+{summaries}
+"""
+
+
+def _call_ollama(host: str, model: str, prompt: str) -> str:
+    url = f"{host.rstrip('/')}/api/generate"
+    try:
+        resp = requests.post(
+            url,
+            json={"model": model, "prompt": prompt, "stream": False, "options": {"num_ctx": 8192}},
+            timeout=600,
+        )
+    except requests.exceptions.ConnectionError as e:
+        raise SummarizationError(
+            f"Could not connect to Ollama at {host}. Check the host/port in Settings and that Ollama is running.", cause=e
+        )
+    except requests.exceptions.Timeout as e:
+        raise SummarizationError(f"Ollama at {host} didn't respond within 10 minutes - it may be overloaded or stuck.", cause=e)
+
+    if resp.status_code == 404:
+        raise SummarizationError(
+            f"Ollama returned 404 for model '{model}'. This almost always means the model isn't pulled on that host. "
+            f"Run: ollama pull {model}", cause=None
+        )
+    try:
+        resp.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise SummarizationError(f"Ollama returned an error ({resp.status_code}): {resp.text[:300]}", cause=e)
+
+    return resp.json()["response"].strip()
+
+
+def _call_api(base_url: str, api_key: str, model: str, prompt: str) -> str:
+    url = f"{base_url.rstrip('/')}/chat/completions"
+    try:
+        resp = requests.post(
+            url,
+            headers={"Authorization": f"Bearer {api_key}"},
+            json={"model": model, "messages": [{"role": "user", "content": prompt}]},
+            timeout=600,
+        )
+    except requests.exceptions.ConnectionError as e:
+        raise SummarizationError(f"Could not connect to API at {base_url}. Check the base URL in Settings.", cause=e)
+    except requests.exceptions.Timeout as e:
+        raise SummarizationError(f"API at {base_url} didn't respond within 10 minutes.", cause=e)
+
+    if resp.status_code == 401:
+        raise SummarizationError("API rejected the request as unauthorized (401) - check your API key in Settings.", cause=None)
+    if resp.status_code == 404:
+        raise SummarizationError(f"API returned 404 - check the base URL and that model '{model}' exists for this provider.", cause=None)
+    try:
+        resp.raise_for_status()
+    except requests.exceptions.HTTPError as e:
+        raise SummarizationError(f"API returned an error ({resp.status_code}): {resp.text[:300]}", cause=e)
+
+    try:
+        return resp.json()["choices"][0]["message"]["content"].strip()
+    except (KeyError, IndexError) as e:
+        raise SummarizationError(f"API response didn't have the expected shape: {resp.text[:300]}", cause=e)
+
+
+def make_llm_caller(settings: dict):
+    if settings.get("llm_mode") == "api":
+        base_url, api_key, model = settings["api_base_url"], settings["api_key"], settings["api_model"]
+        return lambda prompt: _call_api(base_url, api_key, model, prompt)
+    host, model = settings["ollama_host"], settings["ollama_model"]
+    return lambda prompt: _call_ollama(host, model, prompt)
+
+
+def summarize_session(turns: list[dict], settings: dict, progress_cb=None) -> tuple[str, str]:
+    call = make_llm_caller(settings)
+    ctx_path = settings.get("world_context_path") or ""
+    if ctx_path and Path(ctx_path).exists():
+        world_context = Path(ctx_path).read_text(encoding="utf-8").strip()
+    else:
+        world_context = (settings.get("world_context") or "").strip()
+    world_block = f"\nCampaign context (use this to recognize names/places correctly):\n{world_context}\n" if world_context else ""
+
+    chunks = chunk_turns(turns, int(settings.get("chunk_word_target", 2500)))
+    dm_summaries, player_summaries = [], []
+
+    for i, chunk in enumerate(chunks):
+        if progress_cb:
+            progress_cb(f"Summarizing chunk {i+1}/{len(chunks)}")
+        time_range = f"{fmt_time(chunk[0]['start'])}-{fmt_time(chunk[-1]['end'])}"
+        chunk_text = turns_to_text(chunk)
+        dm_summaries.append(call(DM_CHUNK_PROMPT.format(world_context_block=world_block, time_range=time_range, chunk_text=chunk_text)))
+        player_summaries.append(call(PLAYER_CHUNK_PROMPT.format(world_context_block=world_block, time_range=time_range, chunk_text=chunk_text)))
+
+    if progress_cb:
+        progress_cb("Writing final notes...")
+    dm_final = call(DM_FINAL_PROMPT.format(summaries="\n\n".join(dm_summaries)))
+    player_final = call(PLAYER_FINAL_PROMPT.format(summaries="\n\n".join(player_summaries)))
+    return dm_final, player_final

backend/app/pipeline/transcribe.py

@@ -0,0 +1,104 @@
+"""
+Transcription + diarization using WhisperX as a Python library.
+"""
+import gc
+import json
+from pathlib import Path
+
+import torch
+import whisperx
+
+from ..errors import TranscriptionError, DiarizationError
+from ..logging_config import get_logger
+
+log = get_logger(__name__)
+
+
+def transcribe_and_diarize(
+    audio_path: Path,
+    output_json: Path,
+    model_size: str,
+    compute_type: str,
+    hf_token: str,
+    progress_cb=None,
+) -> dict:
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    if device == "cpu":
+        log.warning("No CUDA GPU available - running on CPU, this will be much slower")
+
+    def report(msg):
+        log.info(msg)
+        if progress_cb:
+            progress_cb(msg)
+
+    report("Loading transcription model...")
+    try:
+        model = whisperx.load_model(model_size, device, compute_type=compute_type)
+    except RuntimeError as e:
+        if "out of memory" in str(e).lower():
+            raise TranscriptionError(
+                f"Ran out of GPU memory loading the '{model_size}' model. "
+                f"Try a smaller model size in Settings (e.g. 'small' or 'medium'), "
+                f"or check nothing else is using the GPU right now.", cause=e
+            )
+        raise TranscriptionError(f"Failed to load Whisper model '{model_size}': {e}", cause=e)
+    except Exception as e:
+        raise TranscriptionError(f"Failed to load Whisper model '{model_size}': {e}", cause=e)
+
+    try:
+        report("Loading audio...")
+        audio = whisperx.load_audio(str(audio_path))
+
+        report("Transcribing...")
+        result = model.transcribe(audio, batch_size=16)
+    except RuntimeError as e:
+        if "out of memory" in str(e).lower():
+            raise TranscriptionError(
+                f"Ran out of GPU memory during transcription. Try a smaller Whisper model size in Settings.", cause=e
+            )
+        raise TranscriptionError(f"Transcription failed: {e}", cause=e)
+    except Exception as e:
+        raise TranscriptionError(f"Transcription failed: {e}", cause=e)
+    finally:
+        del model
+        gc.collect()
+        if device == "cuda":
+            torch.cuda.empty_cache()
+
+    try:
+        report("Aligning...")
+        align_model, metadata = whisperx.load_align_model(language_code=result["language"], device=device)
+        result = whisperx.align(result["segments"], align_model, metadata, audio, device, return_char_alignments=False)
+        del align_model
+        gc.collect()
+        if device == "cuda":
+            torch.cuda.empty_cache()
+    except Exception as e:
+        raise TranscriptionError(f"Alignment step failed: {e}", cause=e)
+
+    if hf_token:
+        try:
+            report("Diarizing...")
+            diarize_model = whisperx.diarize.DiarizationPipeline(use_auth_token=hf_token, device=device)
+            diarize_segments = diarize_model(audio)
+            result = whisperx.assign_word_speakers(diarize_segments, result)
+        except Exception as e:
+            msg = str(e)
+            if "gated" in msg.lower() or "403" in msg or "access" in msg.lower():
+                raise DiarizationError(
+                    "HuggingFace rejected access to the diarization model. Make sure you've accepted the "
+                    "terms on the gated model page (linked in the error below) using the SAME account "
+                    f"your HF token belongs to.\n{msg}", cause=e
+                )
+            raise DiarizationError(f"Diarization failed: {msg}", cause=e)
+    else:
+        report("No HF token set - skipping speaker diarization (all speech will be unattributed)")
+
+    try:
+        output_json.parent.mkdir(parents=True, exist_ok=True)
+        output_json.write_text(json.dumps(result, ensure_ascii=False), encoding='utf-8')
+    except OSError as e:
+        raise TranscriptionError(f"Transcribed successfully, but failed to write output to {output_json}: {e}", cause=e)
+
+    report("Done")
+    return result

backend/app/pipeline/turns.py

@@ -0,0 +1,60 @@
+import json
+from pathlib import Path
+
+
+def fmt_time(seconds: float) -> str:
+    h, rem = divmod(int(seconds), 3600)
+    m, s = divmod(rem, 60)
+    return f"{h:02d}:{m:02d}:{s:02d}"
+
+
+def load_turns(transcript_json: Path, speaker_map: dict[str, str] | None = None) -> list[dict]:
+    """Parse whisperx json into merged (start, end, speaker, text) turns.
+    speaker_map maps raw labels (SPEAKER_00...) to display names; unmapped labels pass through raw."""
+    data = json.loads(transcript_json.read_text())
+    speaker_map = speaker_map or {}
+    turns = []
+    for seg in data.get("segments", []):
+        raw = seg.get("speaker", "UNKNOWN")
+        speaker = speaker_map.get(raw, raw)
+        text = (seg.get("text") or "").strip()
+        if not text:
+            continue
+        if turns and turns[-1]["speaker"] == speaker:
+            turns[-1]["text"] += " " + text
+            turns[-1]["end"] = seg["end"]
+        else:
+            turns.append({"start": seg["start"], "end": seg["end"], "speaker": speaker, "text": text, "raw_speaker": raw})
+    return turns
+
+
+def distinct_speakers(transcript_json: Path) -> list[dict]:
+    """Return each raw speaker label with a couple of sample lines, for the naming UI."""
+    turns = load_turns(transcript_json)  # no map -> raw labels
+    seen: dict[str, list[dict]] = {}
+    for t in turns:
+        seen.setdefault(t["raw_speaker"], []).append(t)
+    return [
+        {
+            "raw_label": label,
+            "samples": [{"start": t["start"], "text": t["text"][:140]} for t in items[:3]],
+        }
+        for label, items in sorted(seen.items())
+    ]
+
+
+def chunk_turns(turns: list[dict], word_target: int) -> list[list[dict]]:
+    chunks, current, word_count = [], [], 0
+    for turn in turns:
+        current.append(turn)
+        word_count += len(turn["text"].split())
+        if word_count >= word_target:
+            chunks.append(current)
+            current, word_count = [], 0
+    if current:
+        chunks.append(current)
+    return chunks
+
+
+def turns_to_text(turns: list[dict]) -> str:
+    return "\n".join(f"[{fmt_time(t['start'])}] {t['speaker']}: {t['text']}" for t in turns)