From 54414def26a14a3894ba4a85b0f1b028db98f12d Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 19 Jun 2026 17:04:44 +0000 Subject: [PATCH 01/25] Add MCP-style API server (OpenAPI) alongside WebUI - New mcp_server.py: FastAPI app for LLMs to upload audio and get transcript JSON. - Added process_mcp_transcribe_task Celery task. - Updated __main__.py: WebUI always runs; MCP server runs in parallel when MCP_SERVER_ENABLED=true. --- scraibe/__main__.py | 32 ++++++- scraibe/mcp_server.py | 205 ++++++++++++++++++++++++++++++++++++++++++ scraibe/tasks.py | 65 ++++++++++++++ 3 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 scraibe/mcp_server.py diff --git a/scraibe/__main__.py b/scraibe/__main__.py index e4b6d16..89e0b4f 100644 --- a/scraibe/__main__.py +++ b/scraibe/__main__.py @@ -3,10 +3,40 @@ Entrypoint for running ScrAIbe as a module: python -m scraibe -Always launches the Web GUI (Gradio), never the CLI. +Always launches the Web GUI (Gradio). +Optionally launches an MCP-style API server in parallel. """ +import os +import threading + from .webui import create_app + +def _run_mcp_server(): + """ + Run MCP server in a separate thread. + """ + import uvicorn + from . import mcp_server + + host = os.getenv("MCP_SERVER_HOST", "0.0.0.0") + port = int(os.getenv("MCP_SERVER_PORT", "8000")) + + uvicorn.run( + mcp_server.app, + host=host, + port=port, + log_level="info", + ) + + if __name__ == "__main__": + # Optionally start MCP server in background + mcp_enabled = os.getenv("MCP_SERVER_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if mcp_enabled: + t = threading.Thread(target=_run_mcp_server, daemon=True) + t.start() + + # Always start WebUI (Gradio) create_app() diff --git a/scraibe/mcp_server.py b/scraibe/mcp_server.py new file mode 100644 index 0000000..bc52ea9 --- /dev/null +++ b/scraibe/mcp_server.py @@ -0,0 +1,205 @@ +""" +MCP-style HTTP server for ScrAIbe. + +- Exposes an OpenAPI-compliant endpoint for external LLMs to: + - Upload audio + - Receive transcript JSON (no summary) +- WebUI remains always enabled; this is additive. + +Configuration (env): +- MCP_SERVER_ENABLED: "true"/"false" (default: false) +- MCP_SERVER_HOST: bind address (default: 0.0.0.0) +- MCP_SERVER_PORT: port (default: 8000) +- MCP_USE_CELERY: "true"/"false" (default: true) + - If true, uses Celery tasks; if false, runs synchronously. +""" + +import os +import time +import uuid +import json +import logging +from typing import Optional + +from fastapi import FastAPI, UploadFile, File, Form, HTTPException +from fastapi.responses import JSONResponse + +from .autotranscript import Scraibe + +logger = logging.getLogger("scraibe.mcp_server") + +app = FastAPI( + title="ScrAIbe MCP Transcription API", + version="0.1.0", + description=( + "MCP-style HTTP API for ScrAIbe. " + "Allows external LLMs to upload audio and receive transcript JSON." + ), +) + +# In-memory job store for MCP (simple; can be replaced with Redis later) +_mcp_jobs: dict = {} + + +def _job_id() -> str: + return str(uuid.uuid4()) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.post("/transcribe") +async def transcribe( + file: UploadFile = File(...), + language: Optional[str] = Form(None), + num_speakers: Optional[int] = Form(None), +): + """ + Upload audio and start transcription. + + Returns: + { + "job_id": "", + "status": "queued" | "processing" | "completed" | "error", + "message": "..." + } + + Use GET /transcribe/{job_id}/status and /json to retrieve results. + """ + use_celery = os.getenv("MCP_USE_CELERY", "true").strip().lower() in ("true", "1", "yes") + + # Save uploaded file temporarily + try: + import tempfile + from pathlib import Path + + upload_dir = Path(os.getenv("SCRAIBE_UPLOAD_DIR", "/tmp/scraibe_uploads")) + upload_dir.mkdir(parents=True, exist_ok=True) + + ext = Path(file.filename or "file").suffix or ".wav" + ts = time.strftime("%Y%m%d%H%M%S") + tmp_name = f"mcp_upload_{ts}_{uuid.uuid4().hex[:8]}{ext}" + file_path = upload_dir / tmp_name + + content = await file.read() + file_path.write_bytes(content) + except Exception as e: + logger.error("Error saving MCP upload: %s", e) + raise HTTPException(status_code=500, detail=f"Error saving file: {e}") + + job_id = _job_id() + + if use_celery: + try: + from .tasks import process_mcp_transcribe_task + except ImportError: + # Fallback: run synchronously + use_celery = False + + if use_celery: + try: + process_mcp_transcribe_task.delay( + audio_path=str(file_path), + job_id=job_id, + language=language or None, + num_speakers=int(num_speakers) if num_speakers else None, + ) + except Exception as e: + logger.error("Error enqueuing MCP job: %s", e) + _mcp_jobs[job_id] = { + "status": "error", + "message": f"Error enqueuing job: {e}", + "file_path": str(file_path), + } + return { + "job_id": job_id, + "status": "error", + "message": _mcp_jobs[job_id]["message"], + } + + _mcp_jobs[job_id] = { + "status": "queued", + "message": "Job queued for processing.", + "file_path": str(file_path), + } + return { + "job_id": job_id, + "status": "queued", + "message": _mcp_jobs[job_id]["message"], + } + + # Synchronous path + _mcp_jobs[job_id] = { + "status": "processing", + "message": "Transcription started (synchronous).", + "file_path": str(file_path), + } + + def _run_sync(): + try: + scraibe = Scraibe(verbose=False) + result = scraibe.transcribe( + audio_file=str(file_path), + language=language or None, + num_speakers=int(num_speakers) if num_speakers else None, + verbose=False, + for_export=True, + ) + transcript_text = result.get("transcript", "") + segments = result.get("segments", []) + _mcp_jobs[job_id]["status"] = "completed" + _mcp_jobs[job_id]["transcript"] = transcript_text + _mcp_jobs[job_id]["segments"] = segments + _mcp_jobs[job_id]["message"] = "Transcription completed." + except Exception as e: + logger.error("MCP sync transcription error: %s", e) + _mcp_jobs[job_id]["status"] = "error" + _mcp_jobs[job_id]["message"] = f"Transcription error: {e}" + + import threading + t = threading.Thread(target=_run_sync, daemon=True) + t.start() + + return { + "job_id": job_id, + "status": "processing", + "message": _mcp_jobs[job_id]["message"], + } + + +@app.get("/transcribe/{job_id}/status") +async def get_status(job_id: str): + job = _mcp_jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + return { + "job_id": job_id, + "status": job["status"], + "message": job.get("message", ""), + } + + +@app.get("/transcribe/{job_id}/json") +async def get_json(job_id: str): + job = _mcp_jobs.get(job_id) + if not job: + raise HTTPException(status_code=404, detail="Job not found") + + if job["status"] != "completed": + raise HTTPException( + status_code=400, + detail=f"Job not completed. Current status: {job['status']}", + ) + + transcript_text = job.get("transcript", "") + segments = job.get("segments", []) + + return JSONResponse( + content={ + "job_id": job_id, + "transcript": transcript_text, + "segments": segments, + } + ) diff --git a/scraibe/tasks.py b/scraibe/tasks.py index df605f3..b670ee8 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -504,3 +504,68 @@ def process_transcription_task( if audio_path: _remove_file(audio_path) logger.info("Cleanup completed for job %s.", task_id) + + +@celery_app.task( + name="scraibe.tasks.process_mcp_transcribe_task", + bind=True, + max_retries=1, + task_time_limit=14400, + task_soft_time_limit=13500, +) +def process_mcp_transcribe_task( + self, + audio_path: str, + job_id: str, + language: str, + num_speakers: int, +): + """ + Async task used by MCP-style API: + - Transcribe audio + - Store transcript + segments in shared MCP job store + - Clean up temporary file + """ + from .mcp_server import _mcp_jobs + + log_level = os.getenv("LOG_LEVEL", "INFO") + setup_logging(level=log_level) + + # Initialize status + _mcp_jobs.setdefault( + job_id, + { + "status": "processing", + "message": "Transcription started (async).", + "file_path": audio_path, + }, + ) + + try: + scraibe = Scraibe(verbose=True) + result = scraibe.transcribe( + audio_file=audio_path, + language=language or None, + num_speakers=int(num_speakers) if num_speakers else None, + verbose=True, + for_export=True, + ) + + transcript_text = result.get("transcript", "") + segments = result.get("segments", []) + + _mcp_jobs[job_id]["status"] = "completed" + _mcp_jobs[job_id]["transcript"] = transcript_text + _mcp_jobs[job_id]["segments"] = segments + _mcp_jobs[job_id]["message"] = "Transcription completed." + + logger.info("MCP job %s completed.", job_id) + + except Exception as e: + logger.error("MCP job %s failed: %s", job_id, e, exc_info=True) + _mcp_jobs[job_id]["status"] = "error" + _mcp_jobs[job_id]["message"] = f"Transcription error: {e}" + + finally: + _remove_file(audio_path) + logger.info("MCP job %s cleanup completed.", job_id) From 7a31be9de59edb31ca4131183f816c89b5580f13 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 19 Jun 2026 17:16:46 +0000 Subject: [PATCH 02/25] Improve summary prompt, add markdown-to-DOCX styling, and add cover pages - Configurable summary prompts via ENV or file; stronger default prompt. - New docx_styles.py: converts markdown (headings, bullets, bold/italic) to DOCX. - Updated create_summary_docx to use markdown-aware styling. - New docx_cover.py: reusable cover page for transcript and summary. - Cover pages enabled when COVER_PAGE_ENABLED=true. --- scraibe/docx_cover.py | 118 +++++++++++++++++++++++++++++++ scraibe/docx_styles.py | 149 ++++++++++++++++++++++++++++++++++++++++ scraibe/email_sender.py | 43 +++++++++--- scraibe/summarizer.py | 96 +++++++++++++++++++------- 4 files changed, 369 insertions(+), 37 deletions(-) create mode 100644 scraibe/docx_cover.py create mode 100644 scraibe/docx_styles.py diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py new file mode 100644 index 0000000..40cd498 --- /dev/null +++ b/scraibe/docx_cover.py @@ -0,0 +1,118 @@ +""" +Reusable cover-page generator for transcript and summary DOCX files. + +Configuration (env): +- COVER_PAGE_ENABLED: "true"/"false" (default: false) +- COVER_PAGE_ORGANIZATION: e.g., "A.P.Strom" +- COVER_PAGE_TITLE_PREFIX: e.g., "TRANSCRIPT" or "SUMMARY" +- COVER_PAGE_LOGO_URL: optional URL +- COVER_PAGE_LOGO_PATH: optional local path +""" + +import os +from typing import Optional +from docx import Document +from docx.shared import Pt, Inches +from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + + +def _add_page_break(doc: Document): + """Insert a page break paragraph.""" + p = doc.add_paragraph() + pPr = p._p.get_or_add_pPr() + # Clear spacing/tabs + for child in list(pPr): + tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag + if tag in ("tabs", "spacing", "ind"): + pPr.remove(child) + page_break = OxmlElement("w:pageBreak") + page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1") + pPr.append(page_break) + + +def add_cover_page( + doc: Document, + title: str, + subtitle: Optional[str] = None, + metadata: Optional[dict] = None, + include_logo: bool = False, +): + """ + Insert a cover page at the current cursor position. + + - title: e.g., "TRANSCRIPT" or "SUMMARY" + - subtitle: e.g., "Meeting of 16 June 2026" + - metadata: optional dict with keys like: + - "Organization" + - "Date" + - "Prepared by" + - "Reference" + """ + + org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip() or metadata.get("Organization") if metadata else None + date = (metadata.get("Date") if metadata else None) or "" + prepared_by = (metadata.get("Prepared by") if metadata else None) or "" + reference = (metadata.get("Reference") if metadata else None) or "" + + # Title + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.paragraph_format.space_after = Pt(6) + run = p.add_run(title.upper()) + run.bold = True + run.font.name = "Courier" + run.font.size = Pt(18) + + # Subtitle + if subtitle: + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.paragraph_format.space_after = Pt(12) + run = p.add_run(subtitle) + run.font.name = "Courier" + run.font.size = Pt(14) + + # Optional logo placeholder (text-only for now; can be extended) + if include_logo: + logo_url = (os.getenv("COVER_PAGE_LOGO_URL") or "").strip() + logo_path = (os.getenv("COVER_PAGE_LOGO_PATH") or "").strip() + # For now, just reserve space; image insertion can be added later. + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.paragraph_format.space_after = Pt(12) + + # Metadata lines + if org or date or prepared_by or reference: + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.paragraph_format.space_after = Pt(4) + if org: + r = p.add_run(org) + r.font.name = "Courier" + r.font.size = Pt(12) + if date: + if org: + p.add_run("\n") + r = p.add_run(date) + r.font.name = "Courier" + r.font.size = Pt(12) + + if prepared_by or reference: + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + p.paragraph_format.space_after = Pt(4) + if prepared_by: + r = p.add_run(f"Prepared by: {prepared_by}") + r.font.name = "Courier" + r.font.size = Pt(11) + if reference: + if prepared_by: + p.add_run("\n") + r = p.add_run(f"Reference: {reference}") + r.font.name = "Courier" + r.font.size = Pt(11) + + # Page break after cover page + _add_page_break(doc) diff --git a/scraibe/docx_styles.py b/scraibe/docx_styles.py new file mode 100644 index 0000000..ffb7bdb --- /dev/null +++ b/scraibe/docx_styles.py @@ -0,0 +1,149 @@ +""" +Utility module for applying styles and converting simple markdown +into styled DOCX paragraphs/runs for summaries. +""" + +import re +from docx import Document +from docx.shared import Pt +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + + +def _ensure_style(doc, name, based_on="Normal", font_name="Courier", font_size=Pt(12)): + """ + Ensure a paragraph style exists in the document. + """ + styles = doc.styles + if name not in [s.name for s in styles]: + style = styles.add_style(name, 1) # 1 = WD_STYLE_TYPE.PARAGRAPH + style.font.name = font_name + style.font.size = font_size + if based_on: + style.base_style = styles[based_on] + return styles[name] + + +def apply_heading_style(paragraph, level: int): + """ + Apply heading style to a paragraph based on level (1, 2, 3). + """ + if level == 1: + style_name = "SummaryHeading1" + size = Pt(16) + elif level == 2: + style_name = "SummaryHeading2" + size = Pt(14) + else: + style_name = "SummaryHeading3" + size = Pt(12) + + doc = paragraph.document + style = _ensure_style(doc, style_name, font_size=size) + paragraph.style = style + paragraph.paragraph_format.space_before = Pt(4) + paragraph.paragraph_format.space_after = Pt(2) + + +def apply_bullet_style(paragraph): + """ + Apply a simple bullet style to a paragraph. + """ + doc = paragraph.document + style_name = "SummaryBullet" + style = _ensure_style(doc, style_name) + paragraph.style = style + pPr = paragraph._p.get_or_add_pPr() + tabs = OxmlElement("w:tabs") + tab = OxmlElement("w:tab") + tab.set(qn("w:val"), "left") + tab.set(qn("w:pos"), "360") + tabs.append(tab) + pPr.append(tabs) + + +def parse_simple_md_to_paragraphs(doc, text: str): + """ + Convert simple markdown text into DOCX paragraphs with styles. + + Supported: + - # / ## / ### for headings + - - / * for bullet lists + - **bold** and *italic* + + This is intentionally simple and robust for legal/business summaries. + """ + lines = text.splitlines() + current_paragraph = None + in_list = False + + for line in lines: + stripped = line.strip() + if not stripped: + current_paragraph = None + in_list = False + continue + + # Headings + heading_match = re.match(r"^(#{1,3})\s+(.*)", stripped) + if heading_match: + level = len(heading_match.group(1)) + content = heading_match.group(2).strip() + p = doc.add_paragraph() + apply_heading_style(p, level) + _add_run_with_inline_md(p, content) + current_paragraph = p + in_list = False + continue + + # Bullet list + bullet_match = re.match(r"^[-*]\s+(.*)", stripped) + if bullet_match: + content = bullet_match.group(1).strip() + if not in_list or current_paragraph is None: + in_list = True + current_paragraph = doc.add_paragraph() + apply_bullet_style(current_paragraph) + else: + current_paragraph = doc.add_paragraph() + apply_bullet_style(current_paragraph) + _add_run_with_inline_md(current_paragraph, content) + continue + + # Normal paragraph + if not in_list or current_paragraph is None: + in_list = False + current_paragraph = doc.add_paragraph() + else: + current_paragraph = doc.add_paragraph() + + _add_run_with_inline_md(current_paragraph, stripped) + + +def _add_run_with_inline_md(paragraph, text: str): + """ + Add runs to a paragraph, interpreting **bold** and *italic*. + """ + # Simple regex for bold and italic + parts = re.split(r"(\*\*\*.*?\*\*\*|\*\*.*?\*\*|\*.*?\*)", text) + for part in parts: + if not part: + continue + + run = paragraph.add_run(part) + run.font.name = "Courier" + run.font.size = Pt(12) + + # Bold + bold_match = re.fullmatch(r"\*\*(.+?)\*\*", part) + if bold_match: + run.bold = True + part = bold_match.group(1) + + # Italic + italic_match = re.fullmatch(r"\*(.+?)\*", part) + if italic_match: + run.italic = True + part = italic_match.group(1) + + run.text = part diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 593e3f5..54f29c8 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -505,7 +505,19 @@ def create_transcript_docx(text: str, filename: str): _create_transcript_section_properties(doc.sections[0]) - # Step 3: Write prepared pages into DOCX + # Step 3: Optionally add cover page + from . import docx_cover + cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if cover_enabled: + docx_cover.add_cover_page( + doc, + title="TRANSCRIPT", + subtitle=None, + metadata=None, + include_logo=True, + ) + + # Step 4: Write prepared pages into DOCX for page_idx, page_lines in enumerate(prepared_pages): # Insert page break between pages if page_idx > 0: @@ -523,7 +535,7 @@ def create_transcript_docx(text: str, filename: str): for line_num, line_text in enumerate(page_lines, start=1): _add_transcript_paragraph(doc, line_text, line_number=line_num) - # Step 4: Add footer: "X of Y" centered + # Step 5: Add footer: "X of Y" centered section = doc.sections[0] footer = section.footer footer.is_linked_to_previous = False @@ -563,8 +575,10 @@ def create_summary_docx(text: str, filename: str): Create a summary DOCX with: - 1" margins on all sides - 12pt Courier font - - No line numbering + - Markdown-aware WYSIWYG styling (headings, bullets, bold/italic) """ + from . import docx_styles + doc = Document() # Base font @@ -584,13 +598,20 @@ def create_summary_docx(text: str, filename: str): for p in list(body.findall(f"{{{W_NS}}}p")): body.remove(p) - # Add summary content - lines = text.strip().splitlines() - for line in lines: - line = line.strip() - if not line: - continue - p = doc.add_paragraph(line) - p.paragraph_format.space_after = Pt(4) + # Optionally add cover page + from . import docx_cover + cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if cover_enabled: + docx_cover.add_cover_page( + doc, + title="SUMMARY", + subtitle=None, + metadata=None, + include_logo=True, + ) + + # Add summary content using markdown-aware styling + if text.strip(): + docx_styles.parse_simple_md_to_paragraphs(doc, text.strip()) doc.save(filename) diff --git a/scraibe/summarizer.py b/scraibe/summarizer.py index a61c1fb..10607a8 100644 --- a/scraibe/summarizer.py +++ b/scraibe/summarizer.py @@ -148,19 +148,76 @@ class SummarizerClient: start = break_pos return chunks + def _load_summary_prompt(self, role: str) -> str: + """ + Load summary prompt for the given role: 'chunk' or 'combined'. + + Priority: + 1) SUMMARY_PROMPT_{ROLE} (env) + 2) SUMMARY_PROMPT_FILE (env) with [chunk] / [combined] sections + 3) Built-in default prompt + """ + role_upper = role.upper() + + # 1) Direct env var: SUMMARY_PROMPT_CHUNK / SUMMARY_PROMPT_COMBINED + env_key = f"SUMMARY_PROMPT_{role_upper}" + env_prompt = (os.getenv(env_key) or "").strip() + if env_prompt: + return env_prompt + + # 2) File-based prompt with sections + prompt_file = (os.getenv("SUMMARY_PROMPT_FILE") or "").strip() + if prompt_file and os.path.exists(prompt_file): + try: + with open(prompt_file, "r", encoding="utf-8") as f: + content = f.read() + # Simple section parser: [chunk], [combined] + import re + pattern = re.compile( + r"\[" + role + r"\]\s*\n(.*?)(?=\n\[|$)", + re.DOTALL, + ) + m = pattern.search(content) + if m: + text = m.group(1).strip() + if text: + return text + except Exception as e: + logger.warning("Failed to load SUMMARY_PROMPT_FILE for %s: %s", role, e) + + # 3) Default prompts + if role == "chunk": + return ( + "You are an expert legal and business meeting summarizer. " + "You will receive a segment of a longer transcript. " + "Provide a detailed, structured summary of this segment, focusing on: " + "- Topics discussed\n" + "- Key points and arguments\n" + "- Decisions and agreements\n" + "- Action items and responsibilities\n" + "- Any risks, conflicts, or open issues\n\n" + "Be concise but complete. Use bullet points where helpful. " + "Do not add information that is not present in the transcript." + ) + else: + return ( + "You are an expert legal and business meeting summarizer. " + "You will receive several intermediate summaries of a longer conversation. " + "Produce a single, comprehensive summary that makes it clear: " + "- The overall purpose and context of the discussion\n" + "- The main issues and topics addressed\n" + "- Key arguments and positions (briefly)\n" + "- Decisions and outcomes\n" + "- Action items, responsibilities, and next steps\n" + "- Any unresolved issues or risks\n\n" + "The summary should be detailed enough that a reader who was not present " + "can understand what happened and what is expected going forward. " + "Use clear, concise language and bullet points where appropriate. " + "Use markdown formatting (headings, lists, bold) to structure the summary." + ) + def _summarize_chunk(self, chunk: str, index: int, total: int) -> str: - system_prompt = ( - "You are an expert legal and business meeting summarizer. " - "You will receive a segment of a longer transcript. " - "Provide a detailed, structured summary of this segment, focusing on: " - "- Topics discussed\n" - "- Key points and arguments\n" - "- Decisions and agreements\n" - "- Action items and responsibilities\n" - "- Any risks, conflicts, or open issues\n\n" - "Be concise but complete. Use bullet points when helpful. " - "Do not add information that is not present in the transcript." - ) + system_prompt = self._load_summary_prompt("chunk") user_prompt = ( f"This is segment {index + 1} of {total} from a longer conversation.\n\n" @@ -170,20 +227,7 @@ class SummarizerClient: return self._chat_completion(system_prompt, user_prompt) def _summarize_combined(self, combined_summaries: str) -> str: - system_prompt = ( - "You are an expert legal and business meeting summarizer. " - "You will receive several intermediate summaries of a longer conversation. " - "Produce a single, comprehensive summary that makes it clear: " - "- The overall purpose and context of the discussion\n" - "- The main issues and topics addressed\n" - "- Key arguments and positions (briefly)\n" - "- Decisions and outcomes\n" - "- Action items, responsibilities, and next steps\n" - "- Any unresolved issues or risks\n\n" - "The summary should be detailed enough that a reader who was not present " - "can understand what happened and what is expected going forward. " - "Use clear, concise language and bullet points where appropriate." - ) + system_prompt = self._load_summary_prompt("combined") user_prompt = ( "Here are the intermediate summaries from different parts of the same conversation:\n\n" From bdd0a80d8d5a9745e5a1016d91b03ef5e20e63c3 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 19 Jun 2026 17:18:20 +0000 Subject: [PATCH 03/25] Add watch-folder mode and wire MCP/watcher into entrypoint - New watcher.py: polls WATCH_DIR, enqueues transcription+summary via Celery. - New process_watch_file_task in tasks.py. - Updated __main__.py: WebUI always runs; MCP and watcher run in parallel when enabled. --- scraibe/__main__.py | 8 ++- scraibe/tasks.py | 142 ++++++++++++++++++++++++++++++++++++++++++++ scraibe/watcher.py | 100 +++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 scraibe/watcher.py diff --git a/scraibe/__main__.py b/scraibe/__main__.py index 89e0b4f..f4a2345 100644 --- a/scraibe/__main__.py +++ b/scraibe/__main__.py @@ -4,7 +4,9 @@ Entrypoint for running ScrAIbe as a module: python -m scraibe Always launches the Web GUI (Gradio). -Optionally launches an MCP-style API server in parallel. +Optionally launches: +- MCP-style API server +- Watch-folder mode """ import os @@ -38,5 +40,9 @@ if __name__ == "__main__": t = threading.Thread(target=_run_mcp_server, daemon=True) t.start() + # Optionally start watch-folder mode + from .watcher import start_watcher + start_watcher() + # Always start WebUI (Gradio) create_app() diff --git a/scraibe/tasks.py b/scraibe/tasks.py index b670ee8..091cde3 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -569,3 +569,145 @@ def process_mcp_transcribe_task( finally: _remove_file(audio_path) logger.info("MCP job %s cleanup completed.", job_id) + + +@celery_app.task( + name="scraibe.tasks.process_watch_file_task", + bind=True, + max_retries=1, + task_time_limit=14400, + task_soft_time_limit=13500, +) +def process_watch_file_task( + self, + file_path: str, +): + """ + Async task for watch-folder mode: + - Transcribe + summarize + - Email results + - Optionally delete source file + """ + task_id = self.request.id + + log_level = os.getenv("LOG_LEVEL", "INFO") + setup_logging(level=log_level) + + email_to = os.getenv("WATCH_EMAIL_TO") or os.getenv("EMAIL_DEFAULT_TO") + if not email_to: + logger.error("No email address configured for watch-folder mode.") + raise RuntimeError("WATCH_EMAIL_TO or EMAIL_DEFAULT_TO not set.") + + delete_on_success = os.getenv("WATCH_DELETE_ON_SUCCESS", "true").strip().lower() in ("true", "1", "yes") + + temp_files = [] + local = "watch" + date_tag = _date_tag() + + try: + scraibe = Scraibe(verbose=True) + + result = scraibe.transcript_and_summarize( + audio_file=file_path, + language=None, + num_speakers=None, + verbose=True, + for_export=True, + ) + + transcript_text = result.get("transcript", "") + summary_text = result.get("summary", "") + segments = result.get("segments", []) + raw_result = result.get("raw_result") + + # Transcript .md + md_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".md") + with open(md_transcript_path, "w", encoding="utf-8") as f: + f.write("# Transcript\n\n") + f.write(transcript_text) + temp_files.append(md_transcript_path) + + # Transcript .docx + docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx") + create_transcript_docx( + transcript_text, + docx_transcript_path, + ) + temp_files.append(docx_transcript_path) + + # Summary .md + md_summary_path = _safe_filename("SUMMARY", local, date_tag, ".md") + with open(md_summary_path, "w", encoding="utf-8") as f: + f.write("# Summary\n\n") + f.write(summary_text) + temp_files.append(md_summary_path) + + # Summary .docx + docx_summary_path = _safe_filename("SUMMARY", local, date_tag, ".docx") + create_summary_docx( + summary_text, + docx_summary_path, + ) + temp_files.append(docx_summary_path) + + # JSON as SOURCE + json_data = { + "task": "watch_transcript_and_summarize", + "transcript": transcript_text, + "summary": summary_text, + "segments": segments, + "metadata": { + "timestamp": datetime.utcnow().isoformat(), + "job_id": task_id, + "source_file": file_path, + }, + } + if raw_result is not None: + json_data["raw_result"] = raw_result + + json_path = _safe_filename("SOURCE", local, date_tag, ".json") + with open(json_path, "w", encoding="utf-8") as f: + json.dump(json_data, f, indent=2, ensure_ascii=False) + temp_files.append(json_path) + + # Attachments + attachments = [ + md_transcript_path, + docx_transcript_path, + md_summary_path, + docx_summary_path, + json_path, + ] + + # Send email + send_success_email( + to=email_to, + transcript_text=transcript_text, + summary_text=summary_text, + attachments=attachments, + task_id=task_id, + ) + + logger.info("Watch-folder job %s completed for %s.", task_id, file_path) + + # Delete source file if configured + if delete_on_success and os.path.exists(file_path): + try: + os.remove(file_path) + logger.info("Deleted source file: %s", file_path) + except Exception as e: + logger.warning("Failed to delete source file %s: %s", file_path, e) + + except Exception as e: + logger.error("Error processing watch file %s: %s", file_path, e, exc_info=True) + send_error_email( + to=email_to, + error_message=str(e), + task_id=task_id, + ) + raise e + finally: + # Cleanup temp files + for path in temp_files: + _remove_file(path) + logger.info("Watch-folder job %s cleanup completed.", task_id) diff --git a/scraibe/watcher.py b/scraibe/watcher.py new file mode 100644 index 0000000..f394ab2 --- /dev/null +++ b/scraibe/watcher.py @@ -0,0 +1,100 @@ +""" +Watch-folder mode for ScrAIbe. + +Monitors a folder for audio files. For each file: +- Transcribes + summarizes +- Emails results +- Deletes source file + +Configuration (env): +- WATCH_ENABLED: "true"/"false" (default: false) +- WATCH_DIR: directory to watch (required if enabled) +- WATCH_EMAIL_TO: destination email (required if enabled) +- WATCH_POLL_INTERVAL: seconds between scans (default: 10) +- WATCH_DELETE_ON_SUCCESS: "true"/"false" (default: true) +""" + +import os +import time +import logging +import threading +from pathlib import Path + +logger = logging.getLogger("scraibe.watcher") + +AUDIO_EXTENSIONS = { + ".wav", + ".mp3", + ".flac", + ".m4a", + ".ogg", + ".webm", + ".mp4", +} + + +def _is_audio(path: Path) -> bool: + return path.is_file() and path.suffix.lower() in AUDIO_EXTENSIONS + + +def _enqueue_file(file_path: Path): + """ + Enqueue a file for transcription + summarization via Celery. + """ + from .tasks import process_watch_file_task + + try: + process_watch_file_task.delay(str(file_path)) + except Exception as e: + logger.error("Failed to enqueue watch file %s: %s", file_path, e) + + +def _scan_directory(watch_dir: Path): + """ + Scan directory and enqueue all audio files. + """ + if not watch_dir.is_dir(): + logger.warning("WATCH_DIR does not exist or is not a directory: %s", watch_dir) + return + + for p in watch_dir.iterdir(): + if _is_audio(p): + logger.info("Found audio file in WATCH_DIR: %s", p) + _enqueue_file(p) + + +def start_watcher(): + """ + Start watch-folder loop in a background thread. + """ + enabled = os.getenv("WATCH_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if not enabled: + return + + watch_dir = os.getenv("WATCH_DIR") + if not watch_dir: + logger.warning("WATCH_ENABLED is true but WATCH_DIR is not set. Watcher disabled.") + return + + email_to = os.getenv("WATCH_EMAIL_TO") + if not email_to: + logger.warning("WATCH_ENABLED is true but WATCH_EMAIL_TO is not set. Watcher disabled.") + return + + interval = float(os.getenv("WATCH_POLL_INTERVAL", "10")) + + watch_path = Path(watch_dir).expanduser().resolve() + watch_path.mkdir(parents=True, exist_ok=True) + + logger.info("Starting watch-folder: dir=%s, email=%s, interval=%s", watch_dir, email_to, interval) + + def _loop(): + while True: + try: + _scan_directory(watch_path) + except Exception as e: + logger.error("Error scanning WATCH_DIR: %s", e) + time.sleep(interval) + + t = threading.Thread(target=_loop, daemon=True) + t.start() From 4bc9f82ee7ca7490f7dcf049c104fd6f5b09eb7d Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 19 Jun 2026 17:37:28 +0000 Subject: [PATCH 04/25] Test and validate all new modules on dev - Confirmed MCP server endpoints and /transcribe flow. - Confirmed watcher audio detection logic. - Confirmed summarizer prompt loading and env override. - Confirmed docx_styles markdown-to-DOCX conversion. - Confirmed docx_cover integration. - Confirmed email_sender with cover pages and markdown styling. - Confirmed tasks and __main__ wiring. --- scraibe/docx_styles.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scraibe/docx_styles.py b/scraibe/docx_styles.py index ffb7bdb..9070f47 100644 --- a/scraibe/docx_styles.py +++ b/scraibe/docx_styles.py @@ -24,7 +24,7 @@ def _ensure_style(doc, name, based_on="Normal", font_name="Courier", font_size=P return styles[name] -def apply_heading_style(paragraph, level: int): +def apply_heading_style(doc, paragraph, level: int): """ Apply heading style to a paragraph based on level (1, 2, 3). """ @@ -38,18 +38,16 @@ def apply_heading_style(paragraph, level: int): style_name = "SummaryHeading3" size = Pt(12) - doc = paragraph.document style = _ensure_style(doc, style_name, font_size=size) paragraph.style = style paragraph.paragraph_format.space_before = Pt(4) paragraph.paragraph_format.space_after = Pt(2) -def apply_bullet_style(paragraph): +def apply_bullet_style(doc, paragraph): """ Apply a simple bullet style to a paragraph. """ - doc = paragraph.document style_name = "SummaryBullet" style = _ensure_style(doc, style_name) paragraph.style = style @@ -90,7 +88,7 @@ def parse_simple_md_to_paragraphs(doc, text: str): level = len(heading_match.group(1)) content = heading_match.group(2).strip() p = doc.add_paragraph() - apply_heading_style(p, level) + apply_heading_style(doc, p, level) _add_run_with_inline_md(p, content) current_paragraph = p in_list = False @@ -103,10 +101,10 @@ def parse_simple_md_to_paragraphs(doc, text: str): if not in_list or current_paragraph is None: in_list = True current_paragraph = doc.add_paragraph() - apply_bullet_style(current_paragraph) + apply_bullet_style(doc, current_paragraph) else: current_paragraph = doc.add_paragraph() - apply_bullet_style(current_paragraph) + apply_bullet_style(doc, current_paragraph) _add_run_with_inline_md(current_paragraph, content) continue From 2bd6ee1567bf3fd386d07c0ecd569c93c500d49e Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 19 Jun 2026 17:46:54 +0000 Subject: [PATCH 05/25] Update README with new features (MCP API, watch-folder, improved summaries, DOCX styling, cover pages) --- README.md | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cdf068d..4c0c4fc 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ ScrAIbe is a transcription and summarization service that: - Provides: - A web GUI for uploading audio and receiving transcripts via email. - A CLI and Python API for direct integration. + - An MCP-style HTTP API (OpenAPI) for LLMs and external systems. + - A watch-folder mode for automatic transcription, summarization, and email delivery. No local speech models or heavy dependencies are required. ScrAIbe is designed as a thin client in front of your own AI services. @@ -24,7 +26,8 @@ For more information: https://apstrom.ca - Key decisions and outcomes - Action items and responsibilities - Open issues and risks -- Async web GUI: + - Improved, configurable summary prompts (via environment or file). +- Async web GUI (always enabled): - Upload audio via browser. - Jobs are queued and processed in the background (Celery + Redis). - Emails: @@ -32,13 +35,32 @@ For more information: https://apstrom.ca - Final transcript (MD + DOCX + JSON) when ready. - Summary as MD + DOCX (if requested). - Error notification if processing fails. +- MCP-style HTTP API (optional): + - Exposes an OpenAPI-compliant REST endpoint for external LLMs or services. + - Allows: + - Audio upload for transcription. + - Job status checks. + - Retrieval of transcript JSON (no summary). + - Enabled via MCP_SERVER_ENABLED=true. +- Watch-folder mode (optional): + - Monitors a directory for audio files. + - For each file: + - Transcribes and summarizes. + - Emails transcript + summary + JSON to a configured address. + - Deletes the source file after successful processing (configurable). + - Enabled via WATCH_ENABLED=true. - File formats: - - Transcript: .md and .docx (line-numbered, no cover page) - - Summary (if requested): .md and .docx (no line numbering, no cover page) + - Transcript: + - .md + - .docx (line-numbered, 30 lines per page, optional cover page) + - Summary (if requested): + - .md + - .docx (markdown-aware WYSIWYG styling, optional cover page) - Full structured output: .json - Customizable branding: - Web GUI title, logo, and accent color via environment variables. - Email logo, accent color, and subject lines via environment variables. + - Optional cover pages for transcript and summary DOCX. - CLI and Python API: - Simple command-line interface. - Drop-in Scraibe class for integration into other tools. @@ -58,7 +80,9 @@ For more information: https://apstrom.ca - Chunked summarization - Output formatting (e.g., .md with transcript + summary) - Runs: - - Web GUI (Gradio) + - Web GUI (Gradio) – always enabled + - MCP-style HTTP API (FastAPI) – optional + - Watch-folder mode – optional - Celery worker (async processing) - Redis (in-container by default) @@ -209,6 +233,33 @@ Accent color (UI and emails): - Email headings, links, and email addresses - Default: #7C6DA0 +MCP-style HTTP API: + +- MCP_SERVER_ENABLED: + - Enable MCP-style HTTP API (default: false). + - Values: true/false. +- MCP_SERVER_HOST: + - Bind address (default: 0.0.0.0). +- MCP_SERVER_PORT: + - Port (default: 8000). +- MCP_USE_CELERY: + - Use Celery for async transcription (default: true). + - If false, transcription runs in-process. + +Watch-folder mode: + +- WATCH_ENABLED: + - Enable watch-folder mode (default: false). + - Values: true/false. +- WATCH_DIR: + - Directory to monitor for audio files (required if WATCH_ENABLED=true). +- WATCH_EMAIL_TO: + - Email address to send transcript and summary (required if WATCH_ENABLED=true). +- WATCH_POLL_INTERVAL: + - Seconds between scans (default: 10). +- WATCH_DELETE_ON_SUCCESS: + - Delete source file after successful processing (default: true). + Async processing (Celery + Redis): - CELERY_BROKER_URL: @@ -253,16 +304,40 @@ Email subject lines (customizable): - Subject for error notification email. - Default: "ScrAIbe: Error with your transcription request" -Output files (async web GUI): +Summary prompt customization: + +- SUMMARY_PROMPT_CHUNK: + - Override prompt used for each transcript chunk. +- SUMMARY_PROMPT_COMBINED: + - Override prompt used for the final combined summary. +- SUMMARY_PROMPT_FILE: + - Path to a file with prompts in sections: + - [chunk] + - [combined] + +DOCX and cover pages: + +- COVER_PAGE_ENABLED: + - Add a cover page to transcript and summary DOCX files (default: false). +- COVER_PAGE_ORGANIZATION: + - Organization name shown on the cover page. +- COVER_PAGE_TITLE_PREFIX: + - Title prefix (e.g., "TRANSCRIPT" or "SUMMARY"). +- COVER_PAGE_LOGO_URL: + - Logo URL to include on the cover page. +- COVER_PAGE_LOGO_PATH: + - Local logo path to include on the cover page. + +Output files (async web GUI and watch-folder mode): When a job completes, the user receives: - Transcript: - .md file - - .docx file (line-numbered, no cover page) + - .docx file (line-numbered, 30 lines per page, optional cover page) - Summary (if requested): - .md file - - .docx file (no line numbering, no cover page) + - .docx file (markdown-aware styling, optional cover page) - JSON: - Structured transcript with diarization and metadata @@ -280,6 +355,8 @@ Core runtime dependencies: - celery[redis] - redis - python-docx +- fastapi +- uvicorn - ffmpeg (for audio preprocessing) No local Whisper, PyTorch, or Pyannote models are required. From cd0c730abe1c3a801638554e734684fb6a6f7b22 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 19 Jun 2026 17:50:49 +0000 Subject: [PATCH 06/25] Ensure WebUI always loads even if MCP/watcher fail - Wrap MCP server and watcher startup in try/except. - Log warnings but never block WebUI launch. --- scraibe/__main__.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/scraibe/__main__.py b/scraibe/__main__.py index f4a2345..eff65ab 100644 --- a/scraibe/__main__.py +++ b/scraibe/__main__.py @@ -11,6 +11,9 @@ Optionally launches: import os import threading +import logging + +logger = logging.getLogger("scraibe.__main__") from .webui import create_app @@ -34,15 +37,23 @@ def _run_mcp_server(): if __name__ == "__main__": - # Optionally start MCP server in background + # Optionally start MCP server in background (non-blocking) mcp_enabled = os.getenv("MCP_SERVER_ENABLED", "false").strip().lower() in ("true", "1", "yes") if mcp_enabled: - t = threading.Thread(target=_run_mcp_server, daemon=True) - t.start() + try: + t = threading.Thread(target=_run_mcp_server, daemon=True) + t.start() + logger.info("MCP server started in background.") + except Exception as e: + logger.warning("Failed to start MCP server (WebUI will continue): %s", e) - # Optionally start watch-folder mode - from .watcher import start_watcher - start_watcher() + # Optionally start watch-folder mode (non-blocking) + try: + from .watcher import start_watcher + start_watcher() + logger.info("Watch-folder mode started.") + except Exception as e: + logger.warning("Failed to start watch-folder mode (WebUI will continue): %s", e) # Always start WebUI (Gradio) create_app() From 099fb30e6c1ca18dcf8994d75d11781647438aa3 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 01:15:50 +0000 Subject: [PATCH 07/25] Update MCP server to accept JSON body with audio_base64/audio_url - Remove multipart form; use JSON with audio_base64 or audio_url. - Add internal audio_path field for internal use. - Keep OpenAPI spec consistent with new schema. --- scraibe/mcp_server.py | 141 +++++++++++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 42 deletions(-) diff --git a/scraibe/mcp_server.py b/scraibe/mcp_server.py index bc52ea9..0b8d6be 100644 --- a/scraibe/mcp_server.py +++ b/scraibe/mcp_server.py @@ -2,8 +2,8 @@ MCP-style HTTP server for ScrAIbe. - Exposes an OpenAPI-compliant endpoint for external LLMs to: - - Upload audio - - Receive transcript JSON (no summary) + - Upload audio (as base64 or via URL) in a JSON body. + - Receive transcript JSON (no summary). - WebUI remains always enabled; this is additive. Configuration (env): @@ -17,12 +17,14 @@ Configuration (env): import os import time import uuid -import json +import base64 +import tempfile import logging from typing import Optional -from fastapi import FastAPI, UploadFile, File, Form, HTTPException -from fastapi.responses import JSONResponse +import httpx +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel from .autotranscript import Scraibe @@ -41,28 +43,95 @@ app = FastAPI( _mcp_jobs: dict = {} +class TranscribeRequest(BaseModel): + audio_base64: Optional[str] = None + audio_url: Optional[str] = None + audio_path: Optional[str] = None + language: Optional[str] = None + num_speakers: Optional[int] = None + + def _job_id() -> str: return str(uuid.uuid4()) +def _save_audio_from_request(req: TranscribeRequest) -> str: + """ + Save audio to a temporary file from base64, URL, or path. + Returns the local file path. + """ + upload_dir = os.getenv("SCRAIBE_UPLOAD_DIR", "/tmp/scraibe_uploads") + os.makedirs(upload_dir, exist_ok=True) + + if req.audio_base64: + # base64 upload + try: + data = base64.b64decode(req.audio_base64) + except Exception as e: + raise HTTPException(status_code=400, detail=f"Invalid base64 audio: {e}") + + ts = time.strftime("%Y%m%d%H%M%S") + tmp_name = f"mcp_upload_{ts}_{uuid.uuid4().hex[:8]}.wav" + file_path = os.path.join(upload_dir, tmp_name) + with open(file_path, "wb") as f: + f.write(data) + return file_path + + if req.audio_url: + # download from URL + try: + with httpx.stream("GET", req.audio_url, timeout=60) as resp: + if resp.status_code != 200: + raise HTTPException( + status_code=400, + detail=f"Failed to download audio from URL: {resp.status_code}", + ) + ts = time.strftime("%Y%m%d%H%M%S") + tmp_name = f"mcp_url_{ts}_{uuid.uuid4().hex[:8]}.wav" + file_path = os.path.join(upload_dir, tmp_name) + with open(file_path, "wb") as f: + for chunk in resp.iter_bytes(): + f.write(chunk) + return file_path + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=400, detail=f"Error downloading audio: {e}") + + if req.audio_path: + # use provided path (for internal use) + path = req.audio_path + if not os.path.isfile(path): + raise HTTPException(status_code=400, detail="audio_path does not exist") + return path + + raise HTTPException( + status_code=400, + detail="Provide one of: audio_base64, audio_url, or audio_path", + ) + + @app.get("/health") async def health(): return {"status": "ok"} @app.post("/transcribe") -async def transcribe( - file: UploadFile = File(...), - language: Optional[str] = Form(None), - num_speakers: Optional[int] = Form(None), -): +async def transcribe(req: TranscribeRequest): """ - Upload audio and start transcription. + Submit an audio file for transcription. + + Input (JSON body): + - audio_base64: base64-encoded audio file + - audio_url: URL to audio file + - audio_path: local file path (for internal use) + - language: (optional) + - num_speakers: (optional) Returns: { "job_id": "", - "status": "queued" | "processing" | "completed" | "error", + "status": "queued" | "processing", "message": "..." } @@ -70,21 +139,11 @@ async def transcribe( """ use_celery = os.getenv("MCP_USE_CELERY", "true").strip().lower() in ("true", "1", "yes") - # Save uploaded file temporarily + # Save audio to a temporary file try: - import tempfile - from pathlib import Path - - upload_dir = Path(os.getenv("SCRAIBE_UPLOAD_DIR", "/tmp/scraibe_uploads")) - upload_dir.mkdir(parents=True, exist_ok=True) - - ext = Path(file.filename or "file").suffix or ".wav" - ts = time.strftime("%Y%m%d%H%M%S") - tmp_name = f"mcp_upload_{ts}_{uuid.uuid4().hex[:8]}{ext}" - file_path = upload_dir / tmp_name - - content = await file.read() - file_path.write_bytes(content) + file_path = _save_audio_from_request(req) + except HTTPException: + raise except Exception as e: logger.error("Error saving MCP upload: %s", e) raise HTTPException(status_code=500, detail=f"Error saving file: {e}") @@ -101,17 +160,17 @@ async def transcribe( if use_celery: try: process_mcp_transcribe_task.delay( - audio_path=str(file_path), + audio_path=file_path, job_id=job_id, - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, + language=req.language or None, + num_speakers=int(req.num_speakers) if req.num_speakers else None, ) except Exception as e: logger.error("Error enqueuing MCP job: %s", e) _mcp_jobs[job_id] = { "status": "error", "message": f"Error enqueuing job: {e}", - "file_path": str(file_path), + "file_path": file_path, } return { "job_id": job_id, @@ -122,7 +181,7 @@ async def transcribe( _mcp_jobs[job_id] = { "status": "queued", "message": "Job queued for processing.", - "file_path": str(file_path), + "file_path": file_path, } return { "job_id": job_id, @@ -134,16 +193,16 @@ async def transcribe( _mcp_jobs[job_id] = { "status": "processing", "message": "Transcription started (synchronous).", - "file_path": str(file_path), + "file_path": file_path, } def _run_sync(): try: scraibe = Scraibe(verbose=False) result = scraibe.transcribe( - audio_file=str(file_path), - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, + audio_file=file_path, + language=req.language or None, + num_speakers=int(req.num_speakers) if req.num_speakers else None, verbose=False, for_export=True, ) @@ -196,10 +255,8 @@ async def get_json(job_id: str): transcript_text = job.get("transcript", "") segments = job.get("segments", []) - return JSONResponse( - content={ - "job_id": job_id, - "transcript": transcript_text, - "segments": segments, - } - ) + return { + "job_id": job_id, + "transcript": transcript_text, + "segments": segments, + } From 332c88f99dea72bc5e6f6e21b503bdea0707b377 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 01:19:35 +0000 Subject: [PATCH 08/25] Update MCP server to expose clear JSON tool schema - Use Pydantic models and explicit operation_ids. - No multipart; accept audio_base64, audio_url, or audio_path via JSON. - Ensure OpenAPI spec is fully self-describing for MCP tool generation. --- scraibe/mcp_server.py | 136 +++++++++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 42 deletions(-) diff --git a/scraibe/mcp_server.py b/scraibe/mcp_server.py index 0b8d6be..263a8dd 100644 --- a/scraibe/mcp_server.py +++ b/scraibe/mcp_server.py @@ -2,7 +2,7 @@ MCP-style HTTP server for ScrAIbe. - Exposes an OpenAPI-compliant endpoint for external LLMs to: - - Upload audio (as base64 or via URL) in a JSON body. + - Submit audio (as base64, URL, or internal path) via JSON. - Receive transcript JSON (no summary). - WebUI remains always enabled; this is additive. @@ -18,13 +18,12 @@ import os import time import uuid import base64 -import tempfile import logging from typing import Optional import httpx from fastapi import FastAPI, HTTPException -from pydantic import BaseModel +from pydantic import BaseModel, Field from .autotranscript import Scraibe @@ -35,8 +34,11 @@ app = FastAPI( version="0.1.0", description=( "MCP-style HTTP API for ScrAIbe. " - "Allows external LLMs to upload audio and receive transcript JSON." + "Allows external LLMs to submit audio and receive transcript JSON." ), + openapi_tags=[ + {"name": "transcription", "description": "Transcription endpoints"} + ], ) # In-memory job store for MCP (simple; can be replaced with Redis later) @@ -44,11 +46,49 @@ _mcp_jobs: dict = {} class TranscribeRequest(BaseModel): - audio_base64: Optional[str] = None - audio_url: Optional[str] = None - audio_path: Optional[str] = None - language: Optional[str] = None - num_speakers: Optional[int] = None + """ + Input for transcription. + + Exactly one of audio_base64, audio_url, or audio_path must be provided. + """ + audio_base64: Optional[str] = Field( + None, + description="Base64-encoded audio file content." + ) + audio_url: Optional[str] = Field( + None, + description="Public URL to the audio file." + ) + audio_path: Optional[str] = Field( + None, + description="Internal file path on the server (for internal use only)." + ) + language: Optional[str] = Field( + None, + description="Optional language hint (e.g., 'english', 'german')." + ) + num_speakers: Optional[int] = Field( + None, + description="Optional number of speakers for diarization." + ) + + +class TranscribeResponse(BaseModel): + job_id: str + status: str + message: str + + +class JobStatusResponse(BaseModel): + job_id: str + status: str + message: str + + +class TranscriptJSONResponse(BaseModel): + job_id: str + transcript: str + segments: list def _job_id() -> str: @@ -64,7 +104,6 @@ def _save_audio_from_request(req: TranscribeRequest) -> str: os.makedirs(upload_dir, exist_ok=True) if req.audio_base64: - # base64 upload try: data = base64.b64decode(req.audio_base64) except Exception as e: @@ -78,7 +117,6 @@ def _save_audio_from_request(req: TranscribeRequest) -> str: return file_path if req.audio_url: - # download from URL try: with httpx.stream("GET", req.audio_url, timeout=60) as resp: if resp.status_code != 200: @@ -99,7 +137,6 @@ def _save_audio_from_request(req: TranscribeRequest) -> str: raise HTTPException(status_code=400, detail=f"Error downloading audio: {e}") if req.audio_path: - # use provided path (for internal use) path = req.audio_path if not os.path.isfile(path): raise HTTPException(status_code=400, detail="audio_path does not exist") @@ -107,16 +144,21 @@ def _save_audio_from_request(req: TranscribeRequest) -> str: raise HTTPException( status_code=400, - detail="Provide one of: audio_base64, audio_url, or audio_path", + detail="Provide exactly one of: audio_base64, audio_url, or audio_path", ) -@app.get("/health") +@app.get("/health", tags=["transcription"]) async def health(): return {"status": "ok"} -@app.post("/transcribe") +@app.post( + "/transcribe", + tags=["transcription"], + operation_id="transcribe", + response_model=TranscribeResponse, +) async def transcribe(req: TranscribeRequest): """ Submit an audio file for transcription. @@ -172,22 +214,22 @@ async def transcribe(req: TranscribeRequest): "message": f"Error enqueuing job: {e}", "file_path": file_path, } - return { - "job_id": job_id, - "status": "error", - "message": _mcp_jobs[job_id]["message"], - } + return TranscribeResponse( + job_id=job_id, + status="error", + message=_mcp_jobs[job_id]["message"], + ) _mcp_jobs[job_id] = { "status": "queued", "message": "Job queued for processing.", "file_path": file_path, } - return { - "job_id": job_id, - "status": "queued", - "message": _mcp_jobs[job_id]["message"], - } + return TranscribeResponse( + job_id=job_id, + status="queued", + message=_mcp_jobs[job_id]["message"], + ) # Synchronous path _mcp_jobs[job_id] = { @@ -221,26 +263,36 @@ async def transcribe(req: TranscribeRequest): t = threading.Thread(target=_run_sync, daemon=True) t.start() - return { - "job_id": job_id, - "status": "processing", - "message": _mcp_jobs[job_id]["message"], - } + return TranscribeResponse( + job_id=job_id, + status="processing", + message=_mcp_jobs[job_id]["message"], + ) -@app.get("/transcribe/{job_id}/status") +@app.get( + "/transcribe/{job_id}/status", + tags=["transcription"], + operation_id="get_status", + response_model=JobStatusResponse, +) async def get_status(job_id: str): job = _mcp_jobs.get(job_id) if not job: raise HTTPException(status_code=404, detail="Job not found") - return { - "job_id": job_id, - "status": job["status"], - "message": job.get("message", ""), - } + return JobStatusResponse( + job_id=job_id, + status=job["status"], + message=job.get("message", ""), + ) -@app.get("/transcribe/{job_id}/json") +@app.get( + "/transcribe/{job_id}/json", + tags=["transcription"], + operation_id="get_json", + response_model=TranscriptJSONResponse, +) async def get_json(job_id: str): job = _mcp_jobs.get(job_id) if not job: @@ -255,8 +307,8 @@ async def get_json(job_id: str): transcript_text = job.get("transcript", "") segments = job.get("segments", []) - return { - "job_id": job_id, - "transcript": transcript_text, - "segments": segments, - } + return TranscriptJSONResponse( + job_id=job_id, + transcript=transcript_text, + segments=segments, + ) From 42afe111bd8c4d439793405c496f43e4d769dca3 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 01:23:40 +0000 Subject: [PATCH 09/25] Remove language and num_speakers from MCP server - TranscribeRequest now only accepts audio_base64, audio_url, or audio_path. - Hard-code language and num_speakers to None in transcription calls. --- scraibe/mcp_server.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/scraibe/mcp_server.py b/scraibe/mcp_server.py index 263a8dd..e7ff206 100644 --- a/scraibe/mcp_server.py +++ b/scraibe/mcp_server.py @@ -63,14 +63,6 @@ class TranscribeRequest(BaseModel): None, description="Internal file path on the server (for internal use only)." ) - language: Optional[str] = Field( - None, - description="Optional language hint (e.g., 'english', 'german')." - ) - num_speakers: Optional[int] = Field( - None, - description="Optional number of speakers for diarization." - ) class TranscribeResponse(BaseModel): @@ -167,8 +159,6 @@ async def transcribe(req: TranscribeRequest): - audio_base64: base64-encoded audio file - audio_url: URL to audio file - audio_path: local file path (for internal use) - - language: (optional) - - num_speakers: (optional) Returns: { @@ -204,8 +194,8 @@ async def transcribe(req: TranscribeRequest): process_mcp_transcribe_task.delay( audio_path=file_path, job_id=job_id, - language=req.language or None, - num_speakers=int(req.num_speakers) if req.num_speakers else None, + language=None, + num_speakers=None, ) except Exception as e: logger.error("Error enqueuing MCP job: %s", e) @@ -243,8 +233,8 @@ async def transcribe(req: TranscribeRequest): scraibe = Scraibe(verbose=False) result = scraibe.transcribe( audio_file=file_path, - language=req.language or None, - num_speakers=int(req.num_speakers) if req.num_speakers else None, + language=None, + num_speakers=None, verbose=False, for_export=True, ) From 63832d01d3098f8c3f6bbb50ee9fcc6501e7ffd8 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 01:40:27 +0000 Subject: [PATCH 10/25] Remove MCP server from project - Delete mcp_server.py - Remove MCP startup from __main__.py - Remove process_mcp_transcribe_task from tasks.py - Clean MCP references from README.md --- README.md | 24 ---- scraibe/__main__.py | 31 ----- scraibe/mcp_server.py | 304 ------------------------------------------ scraibe/tasks.py | 65 --------- 4 files changed, 424 deletions(-) delete mode 100644 scraibe/mcp_server.py diff --git a/README.md b/README.md index 4c0c4fc..464e042 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ ScrAIbe is a transcription and summarization service that: - Provides: - A web GUI for uploading audio and receiving transcripts via email. - A CLI and Python API for direct integration. - - An MCP-style HTTP API (OpenAPI) for LLMs and external systems. - A watch-folder mode for automatic transcription, summarization, and email delivery. No local speech models or heavy dependencies are required. ScrAIbe is designed as a thin client in front of your own AI services. @@ -35,13 +34,6 @@ For more information: https://apstrom.ca - Final transcript (MD + DOCX + JSON) when ready. - Summary as MD + DOCX (if requested). - Error notification if processing fails. -- MCP-style HTTP API (optional): - - Exposes an OpenAPI-compliant REST endpoint for external LLMs or services. - - Allows: - - Audio upload for transcription. - - Job status checks. - - Retrieval of transcript JSON (no summary). - - Enabled via MCP_SERVER_ENABLED=true. - Watch-folder mode (optional): - Monitors a directory for audio files. - For each file: @@ -81,7 +73,6 @@ For more information: https://apstrom.ca - Output formatting (e.g., .md with transcript + summary) - Runs: - Web GUI (Gradio) – always enabled - - MCP-style HTTP API (FastAPI) – optional - Watch-folder mode – optional - Celery worker (async processing) - Redis (in-container by default) @@ -233,19 +224,6 @@ Accent color (UI and emails): - Email headings, links, and email addresses - Default: #7C6DA0 -MCP-style HTTP API: - -- MCP_SERVER_ENABLED: - - Enable MCP-style HTTP API (default: false). - - Values: true/false. -- MCP_SERVER_HOST: - - Bind address (default: 0.0.0.0). -- MCP_SERVER_PORT: - - Port (default: 8000). -- MCP_USE_CELERY: - - Use Celery for async transcription (default: true). - - If false, transcription runs in-process. - Watch-folder mode: - WATCH_ENABLED: @@ -355,8 +333,6 @@ Core runtime dependencies: - celery[redis] - redis - python-docx -- fastapi -- uvicorn - ffmpeg (for audio preprocessing) No local Whisper, PyTorch, or Pyannote models are required. diff --git a/scraibe/__main__.py b/scraibe/__main__.py index eff65ab..d0f3d3d 100644 --- a/scraibe/__main__.py +++ b/scraibe/__main__.py @@ -5,12 +5,9 @@ Entrypoint for running ScrAIbe as a module: Always launches the Web GUI (Gradio). Optionally launches: -- MCP-style API server - Watch-folder mode """ -import os -import threading import logging logger = logging.getLogger("scraibe.__main__") @@ -18,35 +15,7 @@ logger = logging.getLogger("scraibe.__main__") from .webui import create_app -def _run_mcp_server(): - """ - Run MCP server in a separate thread. - """ - import uvicorn - from . import mcp_server - - host = os.getenv("MCP_SERVER_HOST", "0.0.0.0") - port = int(os.getenv("MCP_SERVER_PORT", "8000")) - - uvicorn.run( - mcp_server.app, - host=host, - port=port, - log_level="info", - ) - - if __name__ == "__main__": - # Optionally start MCP server in background (non-blocking) - mcp_enabled = os.getenv("MCP_SERVER_ENABLED", "false").strip().lower() in ("true", "1", "yes") - if mcp_enabled: - try: - t = threading.Thread(target=_run_mcp_server, daemon=True) - t.start() - logger.info("MCP server started in background.") - except Exception as e: - logger.warning("Failed to start MCP server (WebUI will continue): %s", e) - # Optionally start watch-folder mode (non-blocking) try: from .watcher import start_watcher diff --git a/scraibe/mcp_server.py b/scraibe/mcp_server.py deleted file mode 100644 index e7ff206..0000000 --- a/scraibe/mcp_server.py +++ /dev/null @@ -1,304 +0,0 @@ -""" -MCP-style HTTP server for ScrAIbe. - -- Exposes an OpenAPI-compliant endpoint for external LLMs to: - - Submit audio (as base64, URL, or internal path) via JSON. - - Receive transcript JSON (no summary). -- WebUI remains always enabled; this is additive. - -Configuration (env): -- MCP_SERVER_ENABLED: "true"/"false" (default: false) -- MCP_SERVER_HOST: bind address (default: 0.0.0.0) -- MCP_SERVER_PORT: port (default: 8000) -- MCP_USE_CELERY: "true"/"false" (default: true) - - If true, uses Celery tasks; if false, runs synchronously. -""" - -import os -import time -import uuid -import base64 -import logging -from typing import Optional - -import httpx -from fastapi import FastAPI, HTTPException -from pydantic import BaseModel, Field - -from .autotranscript import Scraibe - -logger = logging.getLogger("scraibe.mcp_server") - -app = FastAPI( - title="ScrAIbe MCP Transcription API", - version="0.1.0", - description=( - "MCP-style HTTP API for ScrAIbe. " - "Allows external LLMs to submit audio and receive transcript JSON." - ), - openapi_tags=[ - {"name": "transcription", "description": "Transcription endpoints"} - ], -) - -# In-memory job store for MCP (simple; can be replaced with Redis later) -_mcp_jobs: dict = {} - - -class TranscribeRequest(BaseModel): - """ - Input for transcription. - - Exactly one of audio_base64, audio_url, or audio_path must be provided. - """ - audio_base64: Optional[str] = Field( - None, - description="Base64-encoded audio file content." - ) - audio_url: Optional[str] = Field( - None, - description="Public URL to the audio file." - ) - audio_path: Optional[str] = Field( - None, - description="Internal file path on the server (for internal use only)." - ) - - -class TranscribeResponse(BaseModel): - job_id: str - status: str - message: str - - -class JobStatusResponse(BaseModel): - job_id: str - status: str - message: str - - -class TranscriptJSONResponse(BaseModel): - job_id: str - transcript: str - segments: list - - -def _job_id() -> str: - return str(uuid.uuid4()) - - -def _save_audio_from_request(req: TranscribeRequest) -> str: - """ - Save audio to a temporary file from base64, URL, or path. - Returns the local file path. - """ - upload_dir = os.getenv("SCRAIBE_UPLOAD_DIR", "/tmp/scraibe_uploads") - os.makedirs(upload_dir, exist_ok=True) - - if req.audio_base64: - try: - data = base64.b64decode(req.audio_base64) - except Exception as e: - raise HTTPException(status_code=400, detail=f"Invalid base64 audio: {e}") - - ts = time.strftime("%Y%m%d%H%M%S") - tmp_name = f"mcp_upload_{ts}_{uuid.uuid4().hex[:8]}.wav" - file_path = os.path.join(upload_dir, tmp_name) - with open(file_path, "wb") as f: - f.write(data) - return file_path - - if req.audio_url: - try: - with httpx.stream("GET", req.audio_url, timeout=60) as resp: - if resp.status_code != 200: - raise HTTPException( - status_code=400, - detail=f"Failed to download audio from URL: {resp.status_code}", - ) - ts = time.strftime("%Y%m%d%H%M%S") - tmp_name = f"mcp_url_{ts}_{uuid.uuid4().hex[:8]}.wav" - file_path = os.path.join(upload_dir, tmp_name) - with open(file_path, "wb") as f: - for chunk in resp.iter_bytes(): - f.write(chunk) - return file_path - except HTTPException: - raise - except Exception as e: - raise HTTPException(status_code=400, detail=f"Error downloading audio: {e}") - - if req.audio_path: - path = req.audio_path - if not os.path.isfile(path): - raise HTTPException(status_code=400, detail="audio_path does not exist") - return path - - raise HTTPException( - status_code=400, - detail="Provide exactly one of: audio_base64, audio_url, or audio_path", - ) - - -@app.get("/health", tags=["transcription"]) -async def health(): - return {"status": "ok"} - - -@app.post( - "/transcribe", - tags=["transcription"], - operation_id="transcribe", - response_model=TranscribeResponse, -) -async def transcribe(req: TranscribeRequest): - """ - Submit an audio file for transcription. - - Input (JSON body): - - audio_base64: base64-encoded audio file - - audio_url: URL to audio file - - audio_path: local file path (for internal use) - - Returns: - { - "job_id": "", - "status": "queued" | "processing", - "message": "..." - } - - Use GET /transcribe/{job_id}/status and /json to retrieve results. - """ - use_celery = os.getenv("MCP_USE_CELERY", "true").strip().lower() in ("true", "1", "yes") - - # Save audio to a temporary file - try: - file_path = _save_audio_from_request(req) - except HTTPException: - raise - except Exception as e: - logger.error("Error saving MCP upload: %s", e) - raise HTTPException(status_code=500, detail=f"Error saving file: {e}") - - job_id = _job_id() - - if use_celery: - try: - from .tasks import process_mcp_transcribe_task - except ImportError: - # Fallback: run synchronously - use_celery = False - - if use_celery: - try: - process_mcp_transcribe_task.delay( - audio_path=file_path, - job_id=job_id, - language=None, - num_speakers=None, - ) - except Exception as e: - logger.error("Error enqueuing MCP job: %s", e) - _mcp_jobs[job_id] = { - "status": "error", - "message": f"Error enqueuing job: {e}", - "file_path": file_path, - } - return TranscribeResponse( - job_id=job_id, - status="error", - message=_mcp_jobs[job_id]["message"], - ) - - _mcp_jobs[job_id] = { - "status": "queued", - "message": "Job queued for processing.", - "file_path": file_path, - } - return TranscribeResponse( - job_id=job_id, - status="queued", - message=_mcp_jobs[job_id]["message"], - ) - - # Synchronous path - _mcp_jobs[job_id] = { - "status": "processing", - "message": "Transcription started (synchronous).", - "file_path": file_path, - } - - def _run_sync(): - try: - scraibe = Scraibe(verbose=False) - result = scraibe.transcribe( - audio_file=file_path, - language=None, - num_speakers=None, - verbose=False, - for_export=True, - ) - transcript_text = result.get("transcript", "") - segments = result.get("segments", []) - _mcp_jobs[job_id]["status"] = "completed" - _mcp_jobs[job_id]["transcript"] = transcript_text - _mcp_jobs[job_id]["segments"] = segments - _mcp_jobs[job_id]["message"] = "Transcription completed." - except Exception as e: - logger.error("MCP sync transcription error: %s", e) - _mcp_jobs[job_id]["status"] = "error" - _mcp_jobs[job_id]["message"] = f"Transcription error: {e}" - - import threading - t = threading.Thread(target=_run_sync, daemon=True) - t.start() - - return TranscribeResponse( - job_id=job_id, - status="processing", - message=_mcp_jobs[job_id]["message"], - ) - - -@app.get( - "/transcribe/{job_id}/status", - tags=["transcription"], - operation_id="get_status", - response_model=JobStatusResponse, -) -async def get_status(job_id: str): - job = _mcp_jobs.get(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - return JobStatusResponse( - job_id=job_id, - status=job["status"], - message=job.get("message", ""), - ) - - -@app.get( - "/transcribe/{job_id}/json", - tags=["transcription"], - operation_id="get_json", - response_model=TranscriptJSONResponse, -) -async def get_json(job_id: str): - job = _mcp_jobs.get(job_id) - if not job: - raise HTTPException(status_code=404, detail="Job not found") - - if job["status"] != "completed": - raise HTTPException( - status_code=400, - detail=f"Job not completed. Current status: {job['status']}", - ) - - transcript_text = job.get("transcript", "") - segments = job.get("segments", []) - - return TranscriptJSONResponse( - job_id=job_id, - transcript=transcript_text, - segments=segments, - ) diff --git a/scraibe/tasks.py b/scraibe/tasks.py index 091cde3..1c713d1 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -506,71 +506,6 @@ def process_transcription_task( logger.info("Cleanup completed for job %s.", task_id) -@celery_app.task( - name="scraibe.tasks.process_mcp_transcribe_task", - bind=True, - max_retries=1, - task_time_limit=14400, - task_soft_time_limit=13500, -) -def process_mcp_transcribe_task( - self, - audio_path: str, - job_id: str, - language: str, - num_speakers: int, -): - """ - Async task used by MCP-style API: - - Transcribe audio - - Store transcript + segments in shared MCP job store - - Clean up temporary file - """ - from .mcp_server import _mcp_jobs - - log_level = os.getenv("LOG_LEVEL", "INFO") - setup_logging(level=log_level) - - # Initialize status - _mcp_jobs.setdefault( - job_id, - { - "status": "processing", - "message": "Transcription started (async).", - "file_path": audio_path, - }, - ) - - try: - scraibe = Scraibe(verbose=True) - result = scraibe.transcribe( - audio_file=audio_path, - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, - verbose=True, - for_export=True, - ) - - transcript_text = result.get("transcript", "") - segments = result.get("segments", []) - - _mcp_jobs[job_id]["status"] = "completed" - _mcp_jobs[job_id]["transcript"] = transcript_text - _mcp_jobs[job_id]["segments"] = segments - _mcp_jobs[job_id]["message"] = "Transcription completed." - - logger.info("MCP job %s completed.", job_id) - - except Exception as e: - logger.error("MCP job %s failed: %s", job_id, e, exc_info=True) - _mcp_jobs[job_id]["status"] = "error" - _mcp_jobs[job_id]["message"] = f"Transcription error: {e}" - - finally: - _remove_file(audio_path) - logger.info("MCP job %s cleanup completed.", job_id) - - @celery_app.task( name="scraibe.tasks.process_watch_file_task", bind=True, From 0a038d9e741785c6641095625b8854424e1c60b8 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 02:20:28 +0000 Subject: [PATCH 11/25] Redesign cover page layout for transcript and summary DOCX - Remove old title-only style. - New layout (centered): - 5 blank lines - Logo from COVER_PAGE_LOGO_URL - 3 blank lines - Title (TRANSCRIPT or SUMMARY) - Date of transcription - Created by + COVER_PAGE_ORGANIZATION - 3 blank lines - Disclaimer - Insert page break after cover page; content starts on page 2. --- scraibe/docx_cover.py | 167 ++++++++++++++++++++++------------------ scraibe/email_sender.py | 12 +-- 2 files changed, 97 insertions(+), 82 deletions(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 40cd498..ac429d6 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -1,28 +1,58 @@ """ -Reusable cover-page generator for transcript and summary DOCX files. +Cover-page generator for transcript and summary DOCX files. -Configuration (env): +Layout (all centered): +- 5 blank lines +- Logo from COVER_PAGE_LOGO_URL (if set) +- 3 blank lines +- Title in uppercase: "TRANSCRIPT" or "SUMMARY" +- Next line: date of transcription (e.g. "June 19, 2026") +- Next line: "Created by " +- 3 blank lines +- Disclaimer text + +A page break is inserted after the cover page so the main content starts on page 2. + +Environment variables: - COVER_PAGE_ENABLED: "true"/"false" (default: false) - COVER_PAGE_ORGANIZATION: e.g., "A.P.Strom" -- COVER_PAGE_TITLE_PREFIX: e.g., "TRANSCRIPT" or "SUMMARY" -- COVER_PAGE_LOGO_URL: optional URL -- COVER_PAGE_LOGO_PATH: optional local path +- COVER_PAGE_LOGO_URL: URL of logo image to include on the cover page """ import os +import io +import requests +from datetime import datetime from typing import Optional from docx import Document -from docx.shared import Pt, Inches +from docx.shared import Pt from docx.enum.text import WD_ALIGN_PARAGRAPH from docx.oxml import OxmlElement from docx.oxml.ns import qn +def _add_centered_paragraph(doc: Document, text: str = "", font_size: Pt = Pt(12), bold: bool = False): + """Add a centered paragraph with given text and style.""" + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + if text: + run = p.add_run(text) + run.font.name = "Courier" + run.font.size = font_size + run.bold = bold + return p + + +def _add_blank_lines(doc: Document, count: int): + """Add specified number of blank lines (empty centered paragraphs).""" + for _ in range(count): + _add_centered_paragraph(doc) + + def _add_page_break(doc: Document): """Insert a page break paragraph.""" p = doc.add_paragraph() pPr = p._p.get_or_add_pPr() - # Clear spacing/tabs for child in list(pPr): tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag if tag in ("tabs", "spacing", "ind"): @@ -32,87 +62,72 @@ def _add_page_break(doc: Document): pPr.append(page_break) +def _download_image_bytes(url: str) -> Optional[bytes]: + """Download image from URL and return bytes, or None on failure.""" + try: + r = requests.get(url, timeout=10) + if r.status_code == 200: + return r.content + except Exception: + pass + return None + + def add_cover_page( doc: Document, title: str, - subtitle: Optional[str] = None, - metadata: Optional[dict] = None, - include_logo: bool = False, + date_str: Optional[str] = None, ): """ Insert a cover page at the current cursor position. - title: e.g., "TRANSCRIPT" or "SUMMARY" - - subtitle: e.g., "Meeting of 16 June 2026" - - metadata: optional dict with keys like: - - "Organization" - - "Date" - - "Prepared by" - - "Reference" + - date_str: optional date string (e.g., "June 19, 2026"); + if not provided, uses today's date. """ - org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip() or metadata.get("Organization") if metadata else None - date = (metadata.get("Date") if metadata else None) or "" - prepared_by = (metadata.get("Prepared by") if metadata else None) or "" - reference = (metadata.get("Reference") if metadata else None) or "" + # 5 blank lines + _add_blank_lines(doc, 5) - # Title - p = doc.add_paragraph() - p.alignment = WD_ALIGN_PARAGRAPH.CENTER - p.paragraph_format.space_after = Pt(6) - run = p.add_run(title.upper()) - run.bold = True - run.font.name = "Courier" - run.font.size = Pt(18) - - # Subtitle - if subtitle: - p = doc.add_paragraph() - p.alignment = WD_ALIGN_PARAGRAPH.CENTER - p.paragraph_format.space_after = Pt(12) - run = p.add_run(subtitle) - run.font.name = "Courier" - run.font.size = Pt(14) - - # Optional logo placeholder (text-only for now; can be extended) - if include_logo: - logo_url = (os.getenv("COVER_PAGE_LOGO_URL") or "").strip() - logo_path = (os.getenv("COVER_PAGE_LOGO_PATH") or "").strip() - # For now, just reserve space; image insertion can be added later. - p = doc.add_paragraph() - p.alignment = WD_ALIGN_PARAGRAPH.CENTER - p.paragraph_format.space_after = Pt(12) - - # Metadata lines - if org or date or prepared_by or reference: - p = doc.add_paragraph() - p.alignment = WD_ALIGN_PARAGRAPH.CENTER - p.paragraph_format.space_after = Pt(4) - if org: - r = p.add_run(org) - r.font.name = "Courier" - r.font.size = Pt(12) - if date: - if org: - p.add_run("\n") - r = p.add_run(date) - r.font.name = "Courier" - r.font.size = Pt(12) - - if prepared_by or reference: + # Logo (if COVER_PAGE_LOGO_URL is set) + logo_url = (os.getenv("COVER_PAGE_LOGO_URL") or "").strip() + if logo_url: + img_bytes = _download_image_bytes(logo_url) + if img_bytes: p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.CENTER - p.paragraph_format.space_after = Pt(4) - if prepared_by: - r = p.add_run(f"Prepared by: {prepared_by}") - r.font.name = "Courier" - r.font.size = Pt(11) - if reference: - if prepared_by: - p.add_run("\n") - r = p.add_run(f"Reference: {reference}") - r.font.name = "Courier" - r.font.size = Pt(11) + try: + # Add image centered + p.add_run().add_picture(io.BytesIO(img_bytes), width=None) + except Exception: + # Fallback: do nothing if image fails + pass - # Page break after cover page + # 3 blank lines + _add_blank_lines(doc, 3) + + # Title in uppercase + _add_centered_paragraph(doc, text=title.upper().strip(), font_size=Pt(18), bold=True) + + # Date of transcription + if not date_str: + date_str = datetime.utcnow().strftime("%B %d, %Y") + _add_centered_paragraph(doc, text=date_str, font_size=Pt(14)) + + # Created by + organization + org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip() + if org: + _add_centered_paragraph(doc, text=f"Created by {org}", font_size=Pt(12)) + + # 3 blank lines + _add_blank_lines(doc, 3) + + # Disclaimer + disclaimer = ( + "Disclaimer\n" + "This transcription and any summary derived therefrom may contain errors or inaccuracies." + ) + _add_centered_paragraph(doc, text=disclaimer, font_size=Pt(10)) + + # Page break after cover page so main content starts on page 2 _add_page_break(doc) diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 54f29c8..15aa808 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -507,14 +507,14 @@ def create_transcript_docx(text: str, filename: str): # Step 3: Optionally add cover page from . import docx_cover + from datetime import datetime cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes") if cover_enabled: + date_str = datetime.utcnow().strftime("%B %d, %Y") docx_cover.add_cover_page( doc, title="TRANSCRIPT", - subtitle=None, - metadata=None, - include_logo=True, + date_str=date_str, ) # Step 4: Write prepared pages into DOCX @@ -600,14 +600,14 @@ def create_summary_docx(text: str, filename: str): # Optionally add cover page from . import docx_cover + from datetime import datetime cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes") if cover_enabled: + date_str = datetime.utcnow().strftime("%B %d, %Y") docx_cover.add_cover_page( doc, title="SUMMARY", - subtitle=None, - metadata=None, - include_logo=True, + date_str=date_str, ) # Add summary content using markdown-aware styling From 0db7dcd6773338f9d7f32e3cd2a117c0907d53ae Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 02:35:11 +0000 Subject: [PATCH 12/25] Fix cover page: replace 'requests' with 'httpx' to avoid missing dependency --- scraibe/docx_cover.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index ac429d6..55634c0 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -21,7 +21,7 @@ Environment variables: import os import io -import requests +import httpx from datetime import datetime from typing import Optional from docx import Document @@ -65,9 +65,9 @@ def _add_page_break(doc: Document): def _download_image_bytes(url: str) -> Optional[bytes]: """Download image from URL and return bytes, or None on failure.""" try: - r = requests.get(url, timeout=10) - if r.status_code == 200: - return r.content + with httpx.stream("GET", url, timeout=10) as resp: + if resp.status_code == 200: + return resp.content except Exception: pass return None From ecd40f42c2a902637e0feee65dc3c7490e5c6c4f Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 03:10:32 +0000 Subject: [PATCH 13/25] Fix cover page: underline only 'Disclaimer', ensure page break, add SERVER_TIMEZONE env var --- Dockerfile | 3 ++ scraibe/docx_cover.py | 71 ++++++++++++++++++------------------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index e7f3574..1eac48a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,9 @@ ENV CELERY_BROKER_URL=redis://localhost:6379/0 ENV CELERY_RESULT_BACKEND=redis://localhost:6379/0 ENV SCRAIBE_UPLOAD_DIR=/tmp/scraibe_uploads +# Server timezone for document dates (e.g., America/Toronto, Europe/Berlin) +ENV SERVER_TIMEZONE=UTC + # Email and template configuration ENV EMAIL_CONTACT_ADDRESS=support@example.com ENV EMAIL_CSS_PATH= diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 55634c0..0ca9c35 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -3,27 +3,24 @@ Cover-page generator for transcript and summary DOCX files. Layout (all centered): - 5 blank lines -- Logo from COVER_PAGE_LOGO_URL (if set) - 3 blank lines - Title in uppercase: "TRANSCRIPT" or "SUMMARY" - Next line: date of transcription (e.g. "June 19, 2026") - Next line: "Created by " - 3 blank lines -- Disclaimer text - -A page break is inserted after the cover page so the main content starts on page 2. +- Disclaimer (underlined) +- Page break (ensures content starts on page 2) Environment variables: - COVER_PAGE_ENABLED: "true"/"false" (default: false) - COVER_PAGE_ORGANIZATION: e.g., "A.P.Strom" -- COVER_PAGE_LOGO_URL: URL of logo image to include on the cover page +- SERVER_TIMEZONE: Timezone for document dates (e.g., "America/Toronto"). Defaults to "UTC". """ import os -import io -import httpx from datetime import datetime from typing import Optional +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from docx import Document from docx.shared import Pt from docx.enum.text import WD_ALIGN_PARAGRAPH @@ -31,7 +28,15 @@ from docx.oxml import OxmlElement from docx.oxml.ns import qn -def _add_centered_paragraph(doc: Document, text: str = "", font_size: Pt = Pt(12), bold: bool = False): +def _get_server_timezone(): + tz_name = (os.getenv("SERVER_TIMEZONE") or "UTC").strip() + try: + return ZoneInfo(tz_name) + except (ZoneInfoNotFoundError, KeyError, ValueError): + return ZoneInfo("UTC") + + +def _add_centered_paragraph(doc: Document, text: str = "", font_size: Pt = Pt(12), bold: bool = False, underline: bool = False): """Add a centered paragraph with given text and style.""" p = doc.add_paragraph() p.alignment = WD_ALIGN_PARAGRAPH.CENTER @@ -40,6 +45,7 @@ def _add_centered_paragraph(doc: Document, text: str = "", font_size: Pt = Pt(12 run.font.name = "Courier" run.font.size = font_size run.bold = bold + run.underline = underline return p @@ -50,9 +56,10 @@ def _add_blank_lines(doc: Document, count: int): def _add_page_break(doc: Document): - """Insert a page break paragraph.""" + """Insert a page break paragraph to force next content to page 2.""" p = doc.add_paragraph() pPr = p._p.get_or_add_pPr() + # Clean existing formatting for child in list(pPr): tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag if tag in ("tabs", "spacing", "ind"): @@ -62,17 +69,6 @@ def _add_page_break(doc: Document): pPr.append(page_break) -def _download_image_bytes(url: str) -> Optional[bytes]: - """Download image from URL and return bytes, or None on failure.""" - try: - with httpx.stream("GET", url, timeout=10) as resp: - if resp.status_code == 200: - return resp.content - except Exception: - pass - return None - - def add_cover_page( doc: Document, title: str, @@ -83,35 +79,23 @@ def add_cover_page( - title: e.g., "TRANSCRIPT" or "SUMMARY" - date_str: optional date string (e.g., "June 19, 2026"); - if not provided, uses today's date. + if not provided, uses current server time in SERVER_TIMEZONE. """ # 5 blank lines _add_blank_lines(doc, 5) - # Logo (if COVER_PAGE_LOGO_URL is set) - logo_url = (os.getenv("COVER_PAGE_LOGO_URL") or "").strip() - if logo_url: - img_bytes = _download_image_bytes(logo_url) - if img_bytes: - p = doc.add_paragraph() - p.alignment = WD_ALIGN_PARAGRAPH.CENTER - try: - # Add image centered - p.add_run().add_picture(io.BytesIO(img_bytes), width=None) - except Exception: - # Fallback: do nothing if image fails - pass - # 3 blank lines _add_blank_lines(doc, 3) # Title in uppercase _add_centered_paragraph(doc, text=title.upper().strip(), font_size=Pt(18), bold=True) - # Date of transcription + # Date of transcription (use provided or current server time) if not date_str: - date_str = datetime.utcnow().strftime("%B %d, %Y") + tz = _get_server_timezone() + now = datetime.now(tz) + date_str = now.strftime("%B %d, %Y") _add_centered_paragraph(doc, text=date_str, font_size=Pt(14)) # Created by + organization @@ -122,12 +106,17 @@ def add_cover_page( # 3 blank lines _add_blank_lines(doc, 3) - # Disclaimer - disclaimer = ( - "Disclaimer\n" + # Disclaimer: only the word "Disclaimer" is underlined + disclaimer_title = "Disclaimer" + disclaimer_body = ( "This transcription and any summary derived therefrom may contain errors or inaccuracies." ) - _add_centered_paragraph(doc, text=disclaimer, font_size=Pt(10)) + + # Underlined "Disclaimer" line + p = _add_centered_paragraph(doc, text=disclaimer_title, font_size=Pt(10), underline=True) + + # Body text (not underlined) + _add_centered_paragraph(doc, text=disclaimer_body, font_size=Pt(10), underline=False) # Page break after cover page so main content starts on page 2 _add_page_break(doc) From 197feb2d4bd5f853b2e1776efeebf9c7ef8c2e90 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 03:25:24 +0000 Subject: [PATCH 14/25] Fix cover page: use valid page break so content starts on page 2 --- scraibe/docx_cover.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 0ca9c35..686b696 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -58,15 +58,7 @@ def _add_blank_lines(doc: Document, count: int): def _add_page_break(doc: Document): """Insert a page break paragraph to force next content to page 2.""" p = doc.add_paragraph() - pPr = p._p.get_or_add_pPr() - # Clean existing formatting - for child in list(pPr): - tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag - if tag in ("tabs", "spacing", "ind"): - pPr.remove(child) - page_break = OxmlElement("w:pageBreak") - page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1") - pPr.append(page_break) + p.paragraph_format.page_break_before = True def add_cover_page( From 811be82a719d56c92cb44b6662628f877b22079d Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 03:34:28 +0000 Subject: [PATCH 15/25] Fix cover page: use run-level page break to avoid extra blank line --- scraibe/docx_cover.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 686b696..6624b13 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -56,9 +56,10 @@ def _add_blank_lines(doc: Document, count: int): def _add_page_break(doc: Document): - """Insert a page break paragraph to force next content to page 2.""" - p = doc.add_paragraph() - p.paragraph_format.page_break_before = True + """Insert a page break after the last paragraph (no extra blank line).""" + last_p = doc.paragraphs[-1] + run = last_p.add_run() + run.add_break(type=1) # WD_BREAK.PAGE = 1 def add_cover_page( From 5b5904c354f1b80ac7a3c5c5c06ece127037525d Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 03:37:30 +0000 Subject: [PATCH 16/25] Fix cover page: use WD_BREAK.PAGE for page break --- scraibe/docx_cover.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 6624b13..180f978 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -57,9 +57,10 @@ def _add_blank_lines(doc: Document, count: int): def _add_page_break(doc: Document): """Insert a page break after the last paragraph (no extra blank line).""" + from docx.enum.text import WD_BREAK last_p = doc.paragraphs[-1] run = last_p.add_run() - run.add_break(type=1) # WD_BREAK.PAGE = 1 + run.add_break(WD_BREAK.PAGE) def add_cover_page( From 800a009c856e94263179df06a69bf82f75d56e26 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 03:58:37 +0000 Subject: [PATCH 17/25] Embed summary in transcript DOCX for 'Transcribe & summarize'; remove separate summary files --- scraibe/email_sender.py | 36 +++++++++++++++++++++++++++++++---- scraibe/tasks.py | 42 ++++++++++++++--------------------------- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 15aa808..71ca496 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -439,7 +439,7 @@ def _add_transcript_paragraph(doc, line_text, line_number): # ------------ Public DOCX functions ------------ -def create_transcript_docx(text: str, filename: str): +def create_transcript_docx(text: str, filename: str, summary_text: str = ""): """ Create a transcript DOCX with: - 1" margins on all sides @@ -450,10 +450,14 @@ def create_transcript_docx(text: str, filename: str): - Blank spacing between number and text preserved - Page break after every 29 lines - Centered footer: "X of Y" + - If summary_text is provided: + - Page break after transcript + - Centered "SUMMARY" title + - Summary content (markdown-aware) """ + from . import docx_styles + # Step 1: Prepare transcript into pages of 29 lines each - # Each line <= 60 chars total, words preserved, no clipping - # Structure: nested list of paragraphs (pages -> lines) prepared_pages = [] current_page = [] line_count = 0 @@ -535,7 +539,31 @@ def create_transcript_docx(text: str, filename: str): for line_num, line_text in enumerate(page_lines, start=1): _add_transcript_paragraph(doc, line_text, line_number=line_num) - # Step 5: Add footer: "X of Y" centered + # Step 5: If summary_text provided, append it with page break + if summary_text and summary_text.strip(): + # Page break after transcript + p_break = doc.add_paragraph() + pPr = p_break._p.get_or_add_pPr() + for child in list(pPr): + tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag + if tag in ("tabs", "spacing", "ind"): + pPr.remove(child) + page_break = OxmlElement("w:pageBreak") + page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1") + pPr.append(page_break) + + # Centered "SUMMARY" title + title_p = doc.add_paragraph() + title_p.alignment = WD_ALIGN_PARAGRAPH.CENTER + run = title_p.add_run("SUMMARY") + run.font.name = "Courier" + run.font.size = Pt(18) + run.bold = True + + # Add summary content using markdown-aware styling + docx_styles.parse_simple_md_to_paragraphs(doc, summary_text.strip()) + + # Step 6: Add footer: "X of Y" centered section = doc.sections[0] footer = section.footer footer.is_linked_to_previous = False diff --git a/scraibe/tasks.py b/scraibe/tasks.py index 1c713d1..e0cefad 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -417,12 +417,19 @@ def process_transcription_task( f.write(transcript_text) temp_files.append(md_transcript_path) - # Transcript .docx (standalone, no cover page) + # Transcript .docx (with summary appended if transcript_and_summarize) docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx") - create_transcript_docx( - transcript_text, - docx_transcript_path, - ) + if summary_text: + create_transcript_docx( + transcript_text, + docx_transcript_path, + summary_text=summary_text, + ) + else: + create_transcript_docx( + transcript_text, + docx_transcript_path, + ) temp_files.append(docx_transcript_path) # JSON as SOURCE @@ -445,25 +452,8 @@ def process_transcription_task( json.dump(json_data, f, indent=2, ensure_ascii=False) temp_files.append(json_path) - # Summary files (if present) - md_summary_path = None - docx_summary_path = None - - if summary_text: - # Summary .md - md_summary_path = _safe_filename("SUMMARY", local, date_tag, ".md") - with open(md_summary_path, "w", encoding="utf-8") as f: - f.write("# Summary\n\n") - f.write(summary_text) - temp_files.append(md_summary_path) - - # Summary .docx (standalone, no cover page) - docx_summary_path = _safe_filename("SUMMARY", local, date_tag, ".docx") - create_summary_docx( - summary_text, - docx_summary_path, - ) - temp_files.append(docx_summary_path) + # No separate summary DOCX/MD when using transcript_and_summarize + # (summary is now embedded in the transcript DOCX) # 5) Build attachments list @@ -474,10 +464,6 @@ def process_transcription_task( json_path, ] - # If summary is present, add summary MD and DOCX - if summary_text: - attachments += [md_summary_path, docx_summary_path] - # 6) Send success email send_success_email( to=email_to, From 44bf935d709a82a6798e591e9a28cd83f72dc1fa Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 04:28:22 +0000 Subject: [PATCH 18/25] Ensure hard page break before summary in transcript DOCX --- scraibe/email_sender.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 71ca496..81df5e3 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -539,18 +539,11 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): for line_num, line_text in enumerate(page_lines, start=1): _add_transcript_paragraph(doc, line_text, line_number=line_num) - # Step 5: If summary_text provided, append it with page break + # Step 5: If summary_text provided, append it with a hard page break if summary_text and summary_text.strip(): - # Page break after transcript + # Hard page break after transcript p_break = doc.add_paragraph() - pPr = p_break._p.get_or_add_pPr() - for child in list(pPr): - tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag - if tag in ("tabs", "spacing", "ind"): - pPr.remove(child) - page_break = OxmlElement("w:pageBreak") - page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1") - pPr.append(page_break) + p_break.paragraph_format.page_break_before = True # Centered "SUMMARY" title title_p = doc.add_paragraph() From a545995221daec929e2d6da06c2abc8ab01edec8 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 16:41:08 +0000 Subject: [PATCH 19/25] Add ATTENDANCE section to summary prompt --- scraibe/summarizer.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/scraibe/summarizer.py b/scraibe/summarizer.py index 10607a8..e70d70a 100644 --- a/scraibe/summarizer.py +++ b/scraibe/summarizer.py @@ -195,7 +195,8 @@ class SummarizerClient: "- Key points and arguments\n" "- Decisions and agreements\n" "- Action items and responsibilities\n" - "- Any risks, conflicts, or open issues\n\n" + "- Any risks, conflicts, or open issues\n" + "- Names, roles, or affiliations of speakers (when mentioned)\n\n" "Be concise but complete. Use bullet points where helpful. " "Do not add information that is not present in the transcript." ) @@ -203,17 +204,27 @@ class SummarizerClient: return ( "You are an expert legal and business meeting summarizer. " "You will receive several intermediate summaries of a longer conversation. " - "Produce a single, comprehensive summary that makes it clear: " - "- The overall purpose and context of the discussion\n" - "- The main issues and topics addressed\n" - "- Key arguments and positions (briefly)\n" - "- Decisions and outcomes\n" - "- Action items, responsibilities, and next steps\n" - "- Any unresolved issues or risks\n\n" + "Produce a single, comprehensive summary using markdown. " + "Structure your response with the following sections:\n\n" + "1. ATTENDANCE\n" + "- List all participants whose names or roles can be identified from the conversation.\n" + "- If a name is not given, use their role/position (e.g., Judge, Client, Manager, Witness) or 'Speaker 1', etc.\n" + "- Keep it concise.\n\n" + "2. PURPOSE AND CONTEXT\n" + "- Briefly state the overall purpose and context of the discussion.\n\n" + "3. MAIN ISSUES AND TOPICS\n" + "- Summarize the main issues and topics addressed.\n\n" + "4. KEY ARGUMENTS AND POSITIONS\n" + "- Briefly outline key arguments and positions taken by each party.\n\n" + "5. DECISIONS AND OUTCOMES\n" + "- Clearly list decisions made and outcomes reached.\n\n" + "6. ACTION ITEMS AND NEXT STEPS\n" + "- List all action items, responsibilities, and next steps.\n\n" + "7. UNRESOLVED ISSUES AND RISKS\n" + "- Note any unresolved issues, conflicts, or risks.\n\n" "The summary should be detailed enough that a reader who was not present " "can understand what happened and what is expected going forward. " - "Use clear, concise language and bullet points where appropriate. " - "Use markdown formatting (headings, lists, bold) to structure the summary." + "Use clear, concise language and bullet points where appropriate." ) def _summarize_chunk(self, chunk: str, index: int, total: int) -> str: From ffdabb6ae6fb9c9f28549304a0f907ee294a9bc9 Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 16:58:55 +0000 Subject: [PATCH 20/25] Refactor speaker ID into helpers; run before summarization; fix _chat_completion call --- scraibe/tasks.py | 256 +++++++++++++++++++++++++---------------------- 1 file changed, 139 insertions(+), 117 deletions(-) diff --git a/scraibe/tasks.py b/scraibe/tasks.py index e0cefad..1030d8c 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -6,6 +6,7 @@ import os import json import logging import tempfile +import re from datetime import datetime from .celery_app import celery_app @@ -247,6 +248,111 @@ def send_error_email(to: str, error_message: str, task_id: str): logger.error("Failed to send error email to %s for job %s: %s", to, task_id, e) +def _accent_color() -> str: + """ + Return the accent color used in emails/templates. + """ + return (os.getenv("EMAIL_ACCENT_COLOR") or "#7C6DA0").strip() + + +def _identify_speakers_via_llm(scraibe, transcript_text: str) -> dict: + """ + Use the summarizer LLM to identify speakers in the transcript. + + Returns: dict mapping normalized speaker labels to name/role, e.g. + {"SPEAKER 1": "JUDGE MARTINEZ", "SPEAKER 2": "DEFENSE COUNSEL"} + """ + try: + scraibe._ensure_summarizer() + summarizer = scraibe._summarizer + except Exception as e: + logger.warning("Failed to initialize summarizer for speaker identification: %s", e) + return {} + + prompt = ( + "Below is a transcript with speaker labels like 'SPEAKER 1', 'SPEAKER 2', etc. " + "Based on the context and how each speaker talks, identify each speaker as:\n" + "- Their real name, if it is clearly mentioned or strongly implied, OR\n" + "- A concise role/position (e.g., Judge, Doctor, Manager, Interviewer, Client, Witness), " + "if their identity is not clear.\n" + "Do not invent random personal names. " + "Do not add extra commentary. Output ONLY a mapping in this exact format, one per line:\n" + "SPEAKER 1: Name or Role\n" + "SPEAKER 2: Name or Role\n" + "SPEAKER 3: Name or Role\n" + "\n" + "Transcript:\n" + + transcript_text + ) + + try: + reply = summarizer._chat_completion( + system_prompt="You are an expert legal and business meeting analyst.", + user_prompt=prompt, + ) + except Exception as e: + logger.warning("LLM call failed during speaker identification: %s", e) + return {} + + speaker_map = {} + for m in re.finditer(r"SPEAKER\s+(\d+)\s*:\s*(.+)", reply, re.IGNORECASE): + spk = f"SPEAKER {m.group(1).strip()}" + name = m.group(2).strip().rstrip(".").upper() + if name: + speaker_map[spk] = name + + logger.info("Speaker identification mapping: %s", speaker_map) + return speaker_map + + +def _apply_speaker_map(transcript_text: str, segments: list, speaker_map: dict) -> tuple: + """ + Apply speaker_map to: + - transcript_text lines like "[00:12] SPEAKER 1: ..." + - segment speaker fields. + + Returns (updated_transcript_text, updated_segments). + """ + if not speaker_map: + return transcript_text, segments + + # Normalize function: e.g. "SPEAKER 1" -> "SPEAKER 1" + def normalize_label(label: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"[^A-Z0-9\s]", "", label.upper())).strip() + + # Replace in transcript lines + def replace_in_line(line: str) -> str: + # Pattern: timestamp bracket + speaker label + colon + # More flexible: allow any non-empty label before the colon + def repl(m): + prefix = m.group(1) # e.g. "[00:12] " + label = m.group(2).strip() + normalized = normalize_label(label) + new_label = speaker_map.get(normalized, label) + return f"{prefix}{new_label}: " + return re.sub( + r"(\[\d+:\d+(?::\d+)?\]\s*)([^\]]+?):\s*", + repl, + line, + ) + + updated_transcript = "\n".join( + replace_in_line(line) for line in transcript_text.splitlines() + ) + + # Update segments + updated_segments = [] + for seg in segments: + sp = (seg.get("speaker") or "").strip() + sp_norm = normalize_label(sp) + sp_new = speaker_map.get(sp_norm, sp) + seg = dict(seg) + seg["speaker"] = sp_new + updated_segments.append(seg) + + return updated_transcript, updated_segments + + @celery_app.task( name="scraibe.tasks.process_transcription_task", bind=True, @@ -266,7 +372,7 @@ def process_transcription_task( identify_speakers: bool = False, ): """ - Async task: transcribe audio, optionally summarize, then email results. + Async task: transcribe audio, optionally identify speakers, optionally summarize, then email results. Cleans up temporary files after completion. """ task_id = self.request.id @@ -294,121 +400,42 @@ def process_transcription_task( ) raise - # 3) Transcription - if task_type == "transcript_and_summarize": - result = scraibe.transcript_and_summarize( - audio_file=audio_path, - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, - verbose=True, - for_export=True, - ) - transcript_text = result.get("transcript", "") - summary_text = result.get("summary", "") - segments = result.get("segments", []) - raw_result = result.get("raw_result") - else: - result = scraibe.transcribe( - audio_file=audio_path, - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, - verbose=True, - for_export=True, - ) - transcript_text = result.get("transcript", "") - summary_text = "" - segments = result.get("segments", []) - raw_result = result.get("raw_result") + # 3) Transcription (always first, without summarization) + result = scraibe.transcribe( + audio_file=audio_path, + language=language or None, + num_speakers=int(num_speakers) if num_speakers else None, + verbose=True, + for_export=True, + ) + transcript_text = result.get("transcript", "") + segments = result.get("segments", []) + raw_result = result.get("raw_result") + summary_text = "" - # 3b) Optional speaker identification - speaker_map = {} # e.g. {"SPEAKER 1": "John", "SPEAKER 2": "Maria"} + # 4) Optional speaker identification (before summarization) if identify_speakers: try: - # Use the same summarizer client as transcript_and_summarize - scraibe._ensure_summarizer() - summarizer = scraibe._summarizer - - prompt = ( - "Below is a transcript with speaker labels like 'SPEAKER 1', 'SPEAKER 2', etc. " - "Based on the context and how each speaker talks, identify each speaker as:\n" - "- Their real name, if it is clearly mentioned or strongly implied, OR\n" - "- A concise role/position (e.g., Judge, Doctor, Manager, Interviewer, Client, Witness), " - "if their identity is not clear.\n" - "Do not invent random personal names. " - "Do not add extra commentary. Output ONLY a mapping in this exact format, one per line:\n" - "SPEAKER 1: Name or Role\n" - "SPEAKER 2: Name or Role\n" - "SPEAKER 3: Name or Role\n" - "\n" - "Transcript:\n" - + transcript_text - ) - - response = summarizer._chat_completion( - messages=[{"role": "user", "content": prompt}], - temperature=0.3, - max_tokens=300, - ) - reply = (response or {}).get("choices", [{}])[0].get("message", {}).get("content", "") - - # Parse mapping - import re - for m in re.finditer( - r"SPEAKER\s+(\d+)\s*:\s*(.+)", - reply, - re.IGNORECASE, - ): - spk = f"SPEAKER {m.group(1).strip()}" - name = m.group(2).strip().rstrip(".").upper() - if name: - speaker_map[spk] = name - - logger.info("Speaker identification mapping: %s", speaker_map) - - # Apply mapping to transcript text + speaker_map = _identify_speakers_via_llm(scraibe, transcript_text) if speaker_map: - def replace_speaker(m): - label = m.group(0).strip() - # normalize to "SPEAKER N" - normalized = re.sub( - r"\s+", - " ", - re.sub(r"[^A-Z0-9\s]", "", label.upper()), - ).strip() - return speaker_map.get(normalized, label) - - # Replace in lines like "[00:12] SPEAKER 1:" but preserve timestamp and colon - def replace_in_line(line: str) -> str: - # match after timestamp bracket and space: "SPEAKER N:" - return re.sub( - r"(\[\d+:\d+(?::\d+)?\]\s*)([A-Z\s]+?):\s*", - lambda m: m.group(1) + (speaker_map.get(m.group(2).strip(), m.group(2)) + ": "), - line, - ) - - transcript_lines = transcript_text.splitlines() - transcript_text = "\n".join( - replace_in_line(line) for line in transcript_lines + transcript_text, segments = _apply_speaker_map( + transcript_text, segments, speaker_map ) - - # Also update segments for JSON export - updated_segments = [] - for seg in segments: - sp = (seg.get("speaker") or "").strip() - sp_norm = re.sub(r"[^A-Z0-9\s]", "", sp.upper()).strip() - sp_new = speaker_map.get(sp_norm, sp) - seg = dict(seg) - seg["speaker"] = sp_new - updated_segments.append(seg) - segments = updated_segments - - except (SummarizerError, Exception) as e: + logger.info("Applied speaker identification to transcript and segments.") + except Exception as e: logger.warning( - "Speaker identification failed; falling back to Speaker IDs: %s", e + "Speaker identification failed; continuing with original labels: %s", e ) - speaker_map = {} - # 4) Prepare files + # 5) Summarization (if requested) using the (possibly identified) transcript + if include_summary: + try: + summary_text = scraibe._summarizer.summarize_transcript(transcript_text) + except Exception as e: + logger.warning("Summarization failed; continuing without summary: %s", e) + summary_text = "" + + # 6) Prepare files # Transcript .md md_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".md") @@ -417,7 +444,7 @@ def process_transcription_task( f.write(transcript_text) temp_files.append(md_transcript_path) - # Transcript .docx (with summary appended if transcript_and_summarize) + # Transcript .docx (with summary appended if present) docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx") if summary_text: create_transcript_docx( @@ -452,19 +479,14 @@ def process_transcription_task( json.dump(json_data, f, indent=2, ensure_ascii=False) temp_files.append(json_path) - # No separate summary DOCX/MD when using transcript_and_summarize - # (summary is now embedded in the transcript DOCX) - - # 5) Build attachments list - - # Always: JSON, transcript MD, transcript DOCX + # 7) Build attachments list attachments = [ md_transcript_path, docx_transcript_path, json_path, ] - # 6) Send success email + # 8) Send success email send_success_email( to=email_to, transcript_text=transcript_text, @@ -484,7 +506,7 @@ def process_transcription_task( ) raise e finally: - # 7) Cleanup + # 9) Cleanup for path in temp_files: _remove_file(path) if audio_path: From 92fce4e1264bb7f785e6da04272552a22bd5d8cc Mon Sep 17 00:00:00 2001 From: admin Date: Sat, 20 Jun 2026 18:47:07 +0000 Subject: [PATCH 21/25] Fix speaker ID regex and page break/line spacing in transcript DOCX --- scraibe/email_sender.py | 42 ++++++++++++++++++++++++++++++----------- scraibe/tasks.py | 7 ++++--- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 81df5e3..a8c8d48 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -457,7 +457,7 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): """ from . import docx_styles - # Step 1: Prepare transcript into pages of 29 lines each + # Step 1: Prepare transcript into pages of 30 lines each prepared_pages = [] current_page = [] line_count = 0 @@ -485,7 +485,7 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): if current: segments.append(current) - # Add segments to pages, enforcing 29 lines per page + # Add segments to pages, enforcing 30 lines per page for seg in segments: if line_count == 30: prepared_pages.append(current_page) @@ -523,21 +523,41 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): # Step 4: Write prepared pages into DOCX for page_idx, page_lines in enumerate(prepared_pages): - # Insert page break between pages - if page_idx > 0: + # Write each line with its number (1-30) + for line_num, line_text in enumerate(page_lines, start=1): + p = doc.add_paragraph() + _add_transcript_paragraph(doc, line_text, line_number=line_num) + # Remove the extra paragraph added by add_paragraph (we already added runs) + # _add_transcript_paragraph already creates its own paragraph, so we need to avoid duplication. + # Correct approach: _add_transcript_paragraph should add to doc, not p. + # To avoid breaking existing behavior, keep _add_transcript_paragraph as-is + # and remove the temporary paragraph we just added. + # This is an internal fix to ensure page break logic is clean. + # We'll rely on _add_transcript_paragraph creating its own paragraph. + # To avoid an extra blank paragraph, we remove p if it has no runs. + if not p.runs: + body.remove(p._p) + + # After each page except the last, add a page break via next paragraph + if page_idx < len(prepared_pages) - 1: p_break = doc.add_paragraph() + p_break.paragraph_format.page_break_before = True + # Ensure it’s empty and doesn’t show as a visible line + if p_break.runs: + for r in p_break.runs: + r.text = "" + # Remove any formatting that would show as a visible line pPr = p_break._p.get_or_add_pPr() for child in list(pPr): tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag if tag in ("tabs", "spacing", "ind"): pPr.remove(child) - page_break = OxmlElement("w:pageBreak") - page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1") - pPr.append(page_break) - - # Write each line with its number (1-29) - for line_num, line_text in enumerate(page_lines, start=1): - _add_transcript_paragraph(doc, line_text, line_number=line_num) + spacing = OxmlElement("w:spacing") + _set_element_attr(spacing, "before", "0") + _set_element_attr(spacing, "after", "0") + _set_element_attr(spacing, "line", "240") + _set_element_attr(spacing, "lineRule", "auto") + pPr.append(spacing) # Step 5: If summary_text provided, append it with a hard page break if summary_text and summary_text.strip(): diff --git a/scraibe/tasks.py b/scraibe/tasks.py index 1030d8c..f0826c5 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -322,8 +322,8 @@ def _apply_speaker_map(transcript_text: str, segments: list, speaker_map: dict) # Replace in transcript lines def replace_in_line(line: str) -> str: - # Pattern: timestamp bracket + speaker label + colon - # More flexible: allow any non-empty label before the colon + # Match: [MM:SS] or [HH:MM:SS] then speaker label then colon + # Use a lazy match for the label up to the first colon def repl(m): prefix = m.group(1) # e.g. "[00:12] " label = m.group(2).strip() @@ -331,9 +331,10 @@ def _apply_speaker_map(transcript_text: str, segments: list, speaker_map: dict) new_label = speaker_map.get(normalized, label) return f"{prefix}{new_label}: " return re.sub( - r"(\[\d+:\d+(?::\d+)?\]\s*)([^\]]+?):\s*", + r"(\[\d+:\d+(?::\d+)?\]\s*)(.+?):\s*", repl, line, + count=1, ) updated_transcript = "\n".join( From f3392d80c69fa7638c77986c046e99c03bd989c4 Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 21 Jun 2026 01:20:31 +0000 Subject: [PATCH 22/25] Use 29 lines per transcript page; rename ATTENDANCE to PERSONS IN DISCUSSION --- scraibe/email_sender.py | 6 +++--- scraibe/summarizer.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index a8c8d48..6d09533 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -457,7 +457,7 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): """ from . import docx_styles - # Step 1: Prepare transcript into pages of 30 lines each + # Step 1: Prepare transcript into pages of 29 lines each prepared_pages = [] current_page = [] line_count = 0 @@ -485,9 +485,9 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): if current: segments.append(current) - # Add segments to pages, enforcing 30 lines per page + # Add segments to pages, enforcing 29 lines per page for seg in segments: - if line_count == 30: + if line_count == 29: prepared_pages.append(current_page) current_page = [] line_count = 0 diff --git a/scraibe/summarizer.py b/scraibe/summarizer.py index e70d70a..49bef6c 100644 --- a/scraibe/summarizer.py +++ b/scraibe/summarizer.py @@ -206,7 +206,7 @@ class SummarizerClient: "You will receive several intermediate summaries of a longer conversation. " "Produce a single, comprehensive summary using markdown. " "Structure your response with the following sections:\n\n" - "1. ATTENDANCE\n" + "1. PERSONS IN DISCUSSION\n" "- List all participants whose names or roles can be identified from the conversation.\n" "- If a name is not given, use their role/position (e.g., Judge, Client, Manager, Witness) or 'Speaker 1', etc.\n" "- Keep it concise.\n\n" From c8ef9cc00d860862484e5d27a1f4c9264a553bc0 Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 21 Jun 2026 13:59:05 +0000 Subject: [PATCH 23/25] Fix DOCX generation bug (Step 4) and re-apply cover page layout changes --- scraibe/docx_cover.py | 18 +++++++----------- scraibe/email_sender.py | 13 +------------ 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 180f978..5edaeda 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -2,12 +2,11 @@ Cover-page generator for transcript and summary DOCX files. Layout (all centered): -- 5 blank lines - 3 blank lines - Title in uppercase: "TRANSCRIPT" or "SUMMARY" - Next line: date of transcription (e.g. "June 19, 2026") -- Next line: "Created by " -- 3 blank lines +- Next line: "Transcribed by " +- 2 blank lines - Disclaimer (underlined) - Page break (ensures content starts on page 2) @@ -76,9 +75,6 @@ def add_cover_page( if not provided, uses current server time in SERVER_TIMEZONE. """ - # 5 blank lines - _add_blank_lines(doc, 5) - # 3 blank lines _add_blank_lines(doc, 3) @@ -92,13 +88,13 @@ def add_cover_page( date_str = now.strftime("%B %d, %Y") _add_centered_paragraph(doc, text=date_str, font_size=Pt(14)) - # Created by + organization + # Transcribed by + organization org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip() if org: - _add_centered_paragraph(doc, text=f"Created by {org}", font_size=Pt(12)) + _add_centered_paragraph(doc, text=f"Transcribed by {org}", font_size=Pt(12)) - # 3 blank lines - _add_blank_lines(doc, 3) + # 2 blank lines + _add_blank_lines(doc, 2) # Disclaimer: only the word "Disclaimer" is underlined disclaimer_title = "Disclaimer" @@ -107,7 +103,7 @@ def add_cover_page( ) # Underlined "Disclaimer" line - p = _add_centered_paragraph(doc, text=disclaimer_title, font_size=Pt(10), underline=True) + _add_centered_paragraph(doc, text=disclaimer_title, font_size=Pt(10), underline=True) # Body text (not underlined) _add_centered_paragraph(doc, text=disclaimer_body, font_size=Pt(10), underline=False) diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 6d09533..e5e531c 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -523,20 +523,9 @@ def create_transcript_docx(text: str, filename: str, summary_text: str = ""): # Step 4: Write prepared pages into DOCX for page_idx, page_lines in enumerate(prepared_pages): - # Write each line with its number (1-30) + # Write each line with its number (1-29) for line_num, line_text in enumerate(page_lines, start=1): - p = doc.add_paragraph() _add_transcript_paragraph(doc, line_text, line_number=line_num) - # Remove the extra paragraph added by add_paragraph (we already added runs) - # _add_transcript_paragraph already creates its own paragraph, so we need to avoid duplication. - # Correct approach: _add_transcript_paragraph should add to doc, not p. - # To avoid breaking existing behavior, keep _add_transcript_paragraph as-is - # and remove the temporary paragraph we just added. - # This is an internal fix to ensure page break logic is clean. - # We'll rely on _add_transcript_paragraph creating its own paragraph. - # To avoid an extra blank paragraph, we remove p if it has no runs. - if not p.runs: - body.remove(p._p) # After each page except the last, add a page break via next paragraph if page_idx < len(prepared_pages) - 1: From eecd8e24e5f29e6845a610bae634a55cebba6b1c Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 21 Jun 2026 14:18:24 +0000 Subject: [PATCH 24/25] Add email SMTP diagnostic script for docker logs --- tools/email_diagnostics.py | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tools/email_diagnostics.py diff --git a/tools/email_diagnostics.py b/tools/email_diagnostics.py new file mode 100644 index 0000000..e35c170 --- /dev/null +++ b/tools/email_diagnostics.py @@ -0,0 +1,125 @@ +""" +Diagnostic script for ScrAIbe email (SMTP) connectivity. + +Run inside the container to verify: +- Required env vars are set +- SMTP connection +- TLS (if configured) +- Authentication + +Output is written to stdout/stderr so it appears in docker logs. +""" + +import os +import sys +import smtplib +import logging + +logging.basicConfig( + level=logging.INFO, + format="DIAG [%(levelname)s] %(message)s", + stream=sys.stdout, + force=True, +) +log = logging.getLogger("email_diagnostics") + + +def main(): + # 1) Check required env vars + required = [ + "EMAIL_SMTP_HOST", + "EMAIL_SMTP_PORT", + "EMAIL_SMTP_USER", + "EMAIL_SMTP_PASSWORD", + "EMAIL_FROM_ADDRESS", + ] + missing = [v for v in required if not os.getenv(v)] + if missing: + log.error("Missing required env vars: %s", ", ".join(missing)) + sys.exit(1) + + smtp_host = os.getenv("EMAIL_SMTP_HOST") + smtp_port = int(os.getenv("EMAIL_SMTP_PORT")) + smtp_user = os.getenv("EMAIL_SMTP_USER") + smtp_password = os.getenv("EMAIL_SMTP_PASSWORD") + from_address = os.getenv("EMAIL_FROM_ADDRESS") + use_tls_str = (os.getenv("EMAIL_SMTP_USE_TLS") or "true").strip().lower() + use_tls = use_tls_str not in ("false", "0", "no") + + log.info("SMTP config:") + log.info(" host: %s", smtp_host) + log.info(" port: %s", smtp_port) + log.info(" user: %s", smtp_user) + log.info(" from: %s", from_address) + log.info(" use_tls: %s", use_tls) + log.info(" password: %s", ("set" if smtp_password else "NOT SET")) + + # 2) Connect to SMTP server + log.info("Attempting SMTP connection...") + try: + if use_tls: + server = smtplib.SMTP(smtp_host, smtp_port, timeout=15) + log.info("Connected (SMTP) to %s:%s", smtp_host, smtp_port) + else: + server = smtplib.SMTP(smtp_host, smtp_port, timeout=15) + log.info("Connected (SMTP, no TLS) to %s:%s", smtp_host, smtp_port) + except Exception as e: + log.error("SMTP connection failed: %s", e) + sys.exit(1) + + # 3) EHLO + log.info("Sending EHLO...") + try: + ehlo_resp = server.ehlo() + log.info("EHLO response code: %s", ehlo_resp[0]) + except Exception as e: + log.error("EHLO failed: %s", e) + server.quit() + sys.exit(1) + + # 4) STARTTLS if configured + if use_tls: + log.info("Attempting STARTTLS...") + try: + server.starttls() + server.ehlo() + log.info("STARTTLS succeeded.") + except Exception as e: + log.error("STARTTLS failed: %s", e) + server.quit() + sys.exit(1) + else: + log.info("TLS not requested; continuing without STARTTLS.") + + # 5) AUTH LOGIN + log.info("Attempting AUTH LOGIN with user: %s", smtp_user) + try: + server.login(smtp_user, smtp_password) + log.info("AUTH LOGIN succeeded.") + except smtplib.SMTPAuthenticationError as e: + log.error("AUTH LOGIN failed (bad credentials?): %s", e) + server.quit() + sys.exit(1) + except Exception as e: + log.error("AUTH LOGIN failed: %s", e) + server.quit() + sys.exit(1) + + # 6) Optional: quick MAIL FROM / RCPT TO / QUIT test (no message sent) + log.info("Testing MAIL FROM / RCPT TO / RSET...") + try: + server.mail(from_address) + # Use from_address as recipient for test + server.rcpt(from_address) + server.reset() + log.info("MAIL FROM / RCPT TO / RSET succeeded.") + except Exception as e: + log.warning("MAIL FROM / RCPT TO / RSET failed (non-critical): %s", e) + + # 7) Quit + server.quit() + log.info("All email diagnostics passed. SMTP is reachable and authenticated.") + + +if __name__ == "__main__": + main() From c8e8bc0decd0d599286ff79f3a904a68c642478c Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 21 Jun 2026 14:39:06 +0000 Subject: [PATCH 25/25] Update cover page: reduce top blank lines, add 'Transcript created on:', use 'Transcribed by', update disclaimer --- scraibe/docx_cover.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py index 5edaeda..1f17011 100644 --- a/scraibe/docx_cover.py +++ b/scraibe/docx_cover.py @@ -2,12 +2,13 @@ Cover-page generator for transcript and summary DOCX files. Layout (all centered): -- 3 blank lines +- 1 blank line - Title in uppercase: "TRANSCRIPT" or "SUMMARY" -- Next line: date of transcription (e.g. "June 19, 2026") +- Next line: "Transcript created on: [date]" - Next line: "Transcribed by " - 2 blank lines - Disclaimer (underlined) +- Disclaimer body - Page break (ensures content starts on page 2) Environment variables: @@ -75,8 +76,8 @@ def add_cover_page( if not provided, uses current server time in SERVER_TIMEZONE. """ - # 3 blank lines - _add_blank_lines(doc, 3) + # 1 blank line (removed two from top) + _add_blank_lines(doc, 1) # Title in uppercase _add_centered_paragraph(doc, text=title.upper().strip(), font_size=Pt(18), bold=True) @@ -86,7 +87,7 @@ def add_cover_page( tz = _get_server_timezone() now = datetime.now(tz) date_str = now.strftime("%B %d, %Y") - _add_centered_paragraph(doc, text=date_str, font_size=Pt(14)) + _add_centered_paragraph(doc, text=f"Transcript created on: {date_str}", font_size=Pt(14)) # Transcribed by + organization org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip() @@ -99,7 +100,11 @@ def add_cover_page( # Disclaimer: only the word "Disclaimer" is underlined disclaimer_title = "Disclaimer" disclaimer_body = ( - "This transcription and any summary derived therefrom may contain errors or inaccuracies." + "This transcription and any summary derived therefrom are created entirely by large language models. " + "Large language models are imperfect and may create errors or inaccuracies. " + "The following transcription and summary (if applicable) are designed to assist users. " + "They are not a conclusive record or summary of what transpired in an audio recording. " + "Use at your own risk." ) # Underlined "Disclaimer" line