diff --git a/Dockerfile b/Dockerfile index e7f3574..1eac48a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,9 @@ ENV CELERY_BROKER_URL=redis://localhost:6379/0 ENV CELERY_RESULT_BACKEND=redis://localhost:6379/0 ENV SCRAIBE_UPLOAD_DIR=/tmp/scraibe_uploads +# Server timezone for document dates (e.g., America/Toronto, Europe/Berlin) +ENV SERVER_TIMEZONE=UTC + # Email and template configuration ENV EMAIL_CONTACT_ADDRESS=support@example.com ENV EMAIL_CSS_PATH= diff --git a/README.md b/README.md index cdf068d..464e042 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ ScrAIbe is a transcription and summarization service that: - Provides: - A web GUI for uploading audio and receiving transcripts via email. - A CLI and Python API for direct integration. + - A watch-folder mode for automatic transcription, summarization, and email delivery. No local speech models or heavy dependencies are required. ScrAIbe is designed as a thin client in front of your own AI services. @@ -24,7 +25,8 @@ For more information: https://apstrom.ca - Key decisions and outcomes - Action items and responsibilities - Open issues and risks -- Async web GUI: + - Improved, configurable summary prompts (via environment or file). +- Async web GUI (always enabled): - Upload audio via browser. - Jobs are queued and processed in the background (Celery + Redis). - Emails: @@ -32,13 +34,25 @@ For more information: https://apstrom.ca - Final transcript (MD + DOCX + JSON) when ready. - Summary as MD + DOCX (if requested). - Error notification if processing fails. +- Watch-folder mode (optional): + - Monitors a directory for audio files. + - For each file: + - Transcribes and summarizes. + - Emails transcript + summary + JSON to a configured address. + - Deletes the source file after successful processing (configurable). + - Enabled via WATCH_ENABLED=true. - File formats: - - Transcript: .md and .docx (line-numbered, no cover page) - - Summary (if requested): .md and .docx (no line numbering, no cover page) + - Transcript: + - .md + - .docx (line-numbered, 30 lines per page, optional cover page) + - Summary (if requested): + - .md + - .docx (markdown-aware WYSIWYG styling, optional cover page) - Full structured output: .json - Customizable branding: - Web GUI title, logo, and accent color via environment variables. - Email logo, accent color, and subject lines via environment variables. + - Optional cover pages for transcript and summary DOCX. - CLI and Python API: - Simple command-line interface. - Drop-in Scraibe class for integration into other tools. @@ -58,7 +72,8 @@ For more information: https://apstrom.ca - Chunked summarization - Output formatting (e.g., .md with transcript + summary) - Runs: - - Web GUI (Gradio) + - Web GUI (Gradio) – always enabled + - Watch-folder mode – optional - Celery worker (async processing) - Redis (in-container by default) @@ -209,6 +224,20 @@ Accent color (UI and emails): - Email headings, links, and email addresses - Default: #7C6DA0 +Watch-folder mode: + +- WATCH_ENABLED: + - Enable watch-folder mode (default: false). + - Values: true/false. +- WATCH_DIR: + - Directory to monitor for audio files (required if WATCH_ENABLED=true). +- WATCH_EMAIL_TO: + - Email address to send transcript and summary (required if WATCH_ENABLED=true). +- WATCH_POLL_INTERVAL: + - Seconds between scans (default: 10). +- WATCH_DELETE_ON_SUCCESS: + - Delete source file after successful processing (default: true). + Async processing (Celery + Redis): - CELERY_BROKER_URL: @@ -253,16 +282,40 @@ Email subject lines (customizable): - Subject for error notification email. - Default: "ScrAIbe: Error with your transcription request" -Output files (async web GUI): +Summary prompt customization: + +- SUMMARY_PROMPT_CHUNK: + - Override prompt used for each transcript chunk. +- SUMMARY_PROMPT_COMBINED: + - Override prompt used for the final combined summary. +- SUMMARY_PROMPT_FILE: + - Path to a file with prompts in sections: + - [chunk] + - [combined] + +DOCX and cover pages: + +- COVER_PAGE_ENABLED: + - Add a cover page to transcript and summary DOCX files (default: false). +- COVER_PAGE_ORGANIZATION: + - Organization name shown on the cover page. +- COVER_PAGE_TITLE_PREFIX: + - Title prefix (e.g., "TRANSCRIPT" or "SUMMARY"). +- COVER_PAGE_LOGO_URL: + - Logo URL to include on the cover page. +- COVER_PAGE_LOGO_PATH: + - Local logo path to include on the cover page. + +Output files (async web GUI and watch-folder mode): When a job completes, the user receives: - Transcript: - .md file - - .docx file (line-numbered, no cover page) + - .docx file (line-numbered, 30 lines per page, optional cover page) - Summary (if requested): - .md file - - .docx file (no line numbering, no cover page) + - .docx file (markdown-aware styling, optional cover page) - JSON: - Structured transcript with diarization and metadata diff --git a/scraibe/__main__.py b/scraibe/__main__.py index e4b6d16..d0f3d3d 100644 --- a/scraibe/__main__.py +++ b/scraibe/__main__.py @@ -3,10 +3,26 @@ Entrypoint for running ScrAIbe as a module: python -m scraibe -Always launches the Web GUI (Gradio), never the CLI. +Always launches the Web GUI (Gradio). +Optionally launches: +- Watch-folder mode """ +import logging + +logger = logging.getLogger("scraibe.__main__") + from .webui import create_app + if __name__ == "__main__": + # Optionally start watch-folder mode (non-blocking) + try: + from .watcher import start_watcher + start_watcher() + logger.info("Watch-folder mode started.") + except Exception as e: + logger.warning("Failed to start watch-folder mode (WebUI will continue): %s", e) + + # Always start WebUI (Gradio) create_app() diff --git a/scraibe/docx_cover.py b/scraibe/docx_cover.py new file mode 100644 index 0000000..1f17011 --- /dev/null +++ b/scraibe/docx_cover.py @@ -0,0 +1,117 @@ +""" +Cover-page generator for transcript and summary DOCX files. + +Layout (all centered): +- 1 blank line +- Title in uppercase: "TRANSCRIPT" or "SUMMARY" +- Next line: "Transcript created on: [date]" +- Next line: "Transcribed by " +- 2 blank lines +- Disclaimer (underlined) +- Disclaimer body +- Page break (ensures content starts on page 2) + +Environment variables: +- COVER_PAGE_ENABLED: "true"/"false" (default: false) +- COVER_PAGE_ORGANIZATION: e.g., "A.P.Strom" +- SERVER_TIMEZONE: Timezone for document dates (e.g., "America/Toronto"). Defaults to "UTC". +""" + +import os +from datetime import datetime +from typing import Optional +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from docx import Document +from docx.shared import Pt +from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + + +def _get_server_timezone(): + tz_name = (os.getenv("SERVER_TIMEZONE") or "UTC").strip() + try: + return ZoneInfo(tz_name) + except (ZoneInfoNotFoundError, KeyError, ValueError): + return ZoneInfo("UTC") + + +def _add_centered_paragraph(doc: Document, text: str = "", font_size: Pt = Pt(12), bold: bool = False, underline: bool = False): + """Add a centered paragraph with given text and style.""" + p = doc.add_paragraph() + p.alignment = WD_ALIGN_PARAGRAPH.CENTER + if text: + run = p.add_run(text) + run.font.name = "Courier" + run.font.size = font_size + run.bold = bold + run.underline = underline + return p + + +def _add_blank_lines(doc: Document, count: int): + """Add specified number of blank lines (empty centered paragraphs).""" + for _ in range(count): + _add_centered_paragraph(doc) + + +def _add_page_break(doc: Document): + """Insert a page break after the last paragraph (no extra blank line).""" + from docx.enum.text import WD_BREAK + last_p = doc.paragraphs[-1] + run = last_p.add_run() + run.add_break(WD_BREAK.PAGE) + + +def add_cover_page( + doc: Document, + title: str, + date_str: Optional[str] = None, +): + """ + Insert a cover page at the current cursor position. + + - title: e.g., "TRANSCRIPT" or "SUMMARY" + - date_str: optional date string (e.g., "June 19, 2026"); + if not provided, uses current server time in SERVER_TIMEZONE. + """ + + # 1 blank line (removed two from top) + _add_blank_lines(doc, 1) + + # Title in uppercase + _add_centered_paragraph(doc, text=title.upper().strip(), font_size=Pt(18), bold=True) + + # Date of transcription (use provided or current server time) + if not date_str: + tz = _get_server_timezone() + now = datetime.now(tz) + date_str = now.strftime("%B %d, %Y") + _add_centered_paragraph(doc, text=f"Transcript created on: {date_str}", font_size=Pt(14)) + + # Transcribed by + organization + org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip() + if org: + _add_centered_paragraph(doc, text=f"Transcribed by {org}", font_size=Pt(12)) + + # 2 blank lines + _add_blank_lines(doc, 2) + + # Disclaimer: only the word "Disclaimer" is underlined + disclaimer_title = "Disclaimer" + disclaimer_body = ( + "This transcription and any summary derived therefrom are created entirely by large language models. " + "Large language models are imperfect and may create errors or inaccuracies. " + "The following transcription and summary (if applicable) are designed to assist users. " + "They are not a conclusive record or summary of what transpired in an audio recording. " + "Use at your own risk." + ) + + # Underlined "Disclaimer" line + _add_centered_paragraph(doc, text=disclaimer_title, font_size=Pt(10), underline=True) + + # Body text (not underlined) + _add_centered_paragraph(doc, text=disclaimer_body, font_size=Pt(10), underline=False) + + # Page break after cover page so main content starts on page 2 + _add_page_break(doc) diff --git a/scraibe/docx_styles.py b/scraibe/docx_styles.py new file mode 100644 index 0000000..9070f47 --- /dev/null +++ b/scraibe/docx_styles.py @@ -0,0 +1,147 @@ +""" +Utility module for applying styles and converting simple markdown +into styled DOCX paragraphs/runs for summaries. +""" + +import re +from docx import Document +from docx.shared import Pt +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + + +def _ensure_style(doc, name, based_on="Normal", font_name="Courier", font_size=Pt(12)): + """ + Ensure a paragraph style exists in the document. + """ + styles = doc.styles + if name not in [s.name for s in styles]: + style = styles.add_style(name, 1) # 1 = WD_STYLE_TYPE.PARAGRAPH + style.font.name = font_name + style.font.size = font_size + if based_on: + style.base_style = styles[based_on] + return styles[name] + + +def apply_heading_style(doc, paragraph, level: int): + """ + Apply heading style to a paragraph based on level (1, 2, 3). + """ + if level == 1: + style_name = "SummaryHeading1" + size = Pt(16) + elif level == 2: + style_name = "SummaryHeading2" + size = Pt(14) + else: + style_name = "SummaryHeading3" + size = Pt(12) + + style = _ensure_style(doc, style_name, font_size=size) + paragraph.style = style + paragraph.paragraph_format.space_before = Pt(4) + paragraph.paragraph_format.space_after = Pt(2) + + +def apply_bullet_style(doc, paragraph): + """ + Apply a simple bullet style to a paragraph. + """ + style_name = "SummaryBullet" + style = _ensure_style(doc, style_name) + paragraph.style = style + pPr = paragraph._p.get_or_add_pPr() + tabs = OxmlElement("w:tabs") + tab = OxmlElement("w:tab") + tab.set(qn("w:val"), "left") + tab.set(qn("w:pos"), "360") + tabs.append(tab) + pPr.append(tabs) + + +def parse_simple_md_to_paragraphs(doc, text: str): + """ + Convert simple markdown text into DOCX paragraphs with styles. + + Supported: + - # / ## / ### for headings + - - / * for bullet lists + - **bold** and *italic* + + This is intentionally simple and robust for legal/business summaries. + """ + lines = text.splitlines() + current_paragraph = None + in_list = False + + for line in lines: + stripped = line.strip() + if not stripped: + current_paragraph = None + in_list = False + continue + + # Headings + heading_match = re.match(r"^(#{1,3})\s+(.*)", stripped) + if heading_match: + level = len(heading_match.group(1)) + content = heading_match.group(2).strip() + p = doc.add_paragraph() + apply_heading_style(doc, p, level) + _add_run_with_inline_md(p, content) + current_paragraph = p + in_list = False + continue + + # Bullet list + bullet_match = re.match(r"^[-*]\s+(.*)", stripped) + if bullet_match: + content = bullet_match.group(1).strip() + if not in_list or current_paragraph is None: + in_list = True + current_paragraph = doc.add_paragraph() + apply_bullet_style(doc, current_paragraph) + else: + current_paragraph = doc.add_paragraph() + apply_bullet_style(doc, current_paragraph) + _add_run_with_inline_md(current_paragraph, content) + continue + + # Normal paragraph + if not in_list or current_paragraph is None: + in_list = False + current_paragraph = doc.add_paragraph() + else: + current_paragraph = doc.add_paragraph() + + _add_run_with_inline_md(current_paragraph, stripped) + + +def _add_run_with_inline_md(paragraph, text: str): + """ + Add runs to a paragraph, interpreting **bold** and *italic*. + """ + # Simple regex for bold and italic + parts = re.split(r"(\*\*\*.*?\*\*\*|\*\*.*?\*\*|\*.*?\*)", text) + for part in parts: + if not part: + continue + + run = paragraph.add_run(part) + run.font.name = "Courier" + run.font.size = Pt(12) + + # Bold + bold_match = re.fullmatch(r"\*\*(.+?)\*\*", part) + if bold_match: + run.bold = True + part = bold_match.group(1) + + # Italic + italic_match = re.fullmatch(r"\*(.+?)\*", part) + if italic_match: + run.italic = True + part = italic_match.group(1) + + run.text = part diff --git a/scraibe/email_sender.py b/scraibe/email_sender.py index 593e3f5..e5e531c 100644 --- a/scraibe/email_sender.py +++ b/scraibe/email_sender.py @@ -439,7 +439,7 @@ def _add_transcript_paragraph(doc, line_text, line_number): # ------------ Public DOCX functions ------------ -def create_transcript_docx(text: str, filename: str): +def create_transcript_docx(text: str, filename: str, summary_text: str = ""): """ Create a transcript DOCX with: - 1" margins on all sides @@ -450,10 +450,14 @@ def create_transcript_docx(text: str, filename: str): - Blank spacing between number and text preserved - Page break after every 29 lines - Centered footer: "X of Y" + - If summary_text is provided: + - Page break after transcript + - Centered "SUMMARY" title + - Summary content (markdown-aware) """ + from . import docx_styles + # Step 1: Prepare transcript into pages of 29 lines each - # Each line <= 60 chars total, words preserved, no clipping - # Structure: nested list of paragraphs (pages -> lines) prepared_pages = [] current_page = [] line_count = 0 @@ -483,7 +487,7 @@ def create_transcript_docx(text: str, filename: str): # Add segments to pages, enforcing 29 lines per page for seg in segments: - if line_count == 30: + if line_count == 29: prepared_pages.append(current_page) current_page = [] line_count = 0 @@ -505,25 +509,63 @@ def create_transcript_docx(text: str, filename: str): _create_transcript_section_properties(doc.sections[0]) - # Step 3: Write prepared pages into DOCX + # Step 3: Optionally add cover page + from . import docx_cover + from datetime import datetime + cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if cover_enabled: + date_str = datetime.utcnow().strftime("%B %d, %Y") + docx_cover.add_cover_page( + doc, + title="TRANSCRIPT", + date_str=date_str, + ) + + # Step 4: Write prepared pages into DOCX for page_idx, page_lines in enumerate(prepared_pages): - # Insert page break between pages - if page_idx > 0: + # Write each line with its number (1-29) + for line_num, line_text in enumerate(page_lines, start=1): + _add_transcript_paragraph(doc, line_text, line_number=line_num) + + # After each page except the last, add a page break via next paragraph + if page_idx < len(prepared_pages) - 1: p_break = doc.add_paragraph() + p_break.paragraph_format.page_break_before = True + # Ensure it’s empty and doesn’t show as a visible line + if p_break.runs: + for r in p_break.runs: + r.text = "" + # Remove any formatting that would show as a visible line pPr = p_break._p.get_or_add_pPr() for child in list(pPr): tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag if tag in ("tabs", "spacing", "ind"): pPr.remove(child) - page_break = OxmlElement("w:pageBreak") - page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1") - pPr.append(page_break) + spacing = OxmlElement("w:spacing") + _set_element_attr(spacing, "before", "0") + _set_element_attr(spacing, "after", "0") + _set_element_attr(spacing, "line", "240") + _set_element_attr(spacing, "lineRule", "auto") + pPr.append(spacing) - # Write each line with its number (1-29) - for line_num, line_text in enumerate(page_lines, start=1): - _add_transcript_paragraph(doc, line_text, line_number=line_num) + # Step 5: If summary_text provided, append it with a hard page break + if summary_text and summary_text.strip(): + # Hard page break after transcript + p_break = doc.add_paragraph() + p_break.paragraph_format.page_break_before = True - # Step 4: Add footer: "X of Y" centered + # Centered "SUMMARY" title + title_p = doc.add_paragraph() + title_p.alignment = WD_ALIGN_PARAGRAPH.CENTER + run = title_p.add_run("SUMMARY") + run.font.name = "Courier" + run.font.size = Pt(18) + run.bold = True + + # Add summary content using markdown-aware styling + docx_styles.parse_simple_md_to_paragraphs(doc, summary_text.strip()) + + # Step 6: Add footer: "X of Y" centered section = doc.sections[0] footer = section.footer footer.is_linked_to_previous = False @@ -563,8 +605,10 @@ def create_summary_docx(text: str, filename: str): Create a summary DOCX with: - 1" margins on all sides - 12pt Courier font - - No line numbering + - Markdown-aware WYSIWYG styling (headings, bullets, bold/italic) """ + from . import docx_styles + doc = Document() # Base font @@ -584,13 +628,20 @@ def create_summary_docx(text: str, filename: str): for p in list(body.findall(f"{{{W_NS}}}p")): body.remove(p) - # Add summary content - lines = text.strip().splitlines() - for line in lines: - line = line.strip() - if not line: - continue - p = doc.add_paragraph(line) - p.paragraph_format.space_after = Pt(4) + # Optionally add cover page + from . import docx_cover + from datetime import datetime + cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if cover_enabled: + date_str = datetime.utcnow().strftime("%B %d, %Y") + docx_cover.add_cover_page( + doc, + title="SUMMARY", + date_str=date_str, + ) + + # Add summary content using markdown-aware styling + if text.strip(): + docx_styles.parse_simple_md_to_paragraphs(doc, text.strip()) doc.save(filename) diff --git a/scraibe/summarizer.py b/scraibe/summarizer.py index a61c1fb..49bef6c 100644 --- a/scraibe/summarizer.py +++ b/scraibe/summarizer.py @@ -148,19 +148,87 @@ class SummarizerClient: start = break_pos return chunks + def _load_summary_prompt(self, role: str) -> str: + """ + Load summary prompt for the given role: 'chunk' or 'combined'. + + Priority: + 1) SUMMARY_PROMPT_{ROLE} (env) + 2) SUMMARY_PROMPT_FILE (env) with [chunk] / [combined] sections + 3) Built-in default prompt + """ + role_upper = role.upper() + + # 1) Direct env var: SUMMARY_PROMPT_CHUNK / SUMMARY_PROMPT_COMBINED + env_key = f"SUMMARY_PROMPT_{role_upper}" + env_prompt = (os.getenv(env_key) or "").strip() + if env_prompt: + return env_prompt + + # 2) File-based prompt with sections + prompt_file = (os.getenv("SUMMARY_PROMPT_FILE") or "").strip() + if prompt_file and os.path.exists(prompt_file): + try: + with open(prompt_file, "r", encoding="utf-8") as f: + content = f.read() + # Simple section parser: [chunk], [combined] + import re + pattern = re.compile( + r"\[" + role + r"\]\s*\n(.*?)(?=\n\[|$)", + re.DOTALL, + ) + m = pattern.search(content) + if m: + text = m.group(1).strip() + if text: + return text + except Exception as e: + logger.warning("Failed to load SUMMARY_PROMPT_FILE for %s: %s", role, e) + + # 3) Default prompts + if role == "chunk": + return ( + "You are an expert legal and business meeting summarizer. " + "You will receive a segment of a longer transcript. " + "Provide a detailed, structured summary of this segment, focusing on: " + "- Topics discussed\n" + "- Key points and arguments\n" + "- Decisions and agreements\n" + "- Action items and responsibilities\n" + "- Any risks, conflicts, or open issues\n" + "- Names, roles, or affiliations of speakers (when mentioned)\n\n" + "Be concise but complete. Use bullet points where helpful. " + "Do not add information that is not present in the transcript." + ) + else: + return ( + "You are an expert legal and business meeting summarizer. " + "You will receive several intermediate summaries of a longer conversation. " + "Produce a single, comprehensive summary using markdown. " + "Structure your response with the following sections:\n\n" + "1. PERSONS IN DISCUSSION\n" + "- List all participants whose names or roles can be identified from the conversation.\n" + "- If a name is not given, use their role/position (e.g., Judge, Client, Manager, Witness) or 'Speaker 1', etc.\n" + "- Keep it concise.\n\n" + "2. PURPOSE AND CONTEXT\n" + "- Briefly state the overall purpose and context of the discussion.\n\n" + "3. MAIN ISSUES AND TOPICS\n" + "- Summarize the main issues and topics addressed.\n\n" + "4. KEY ARGUMENTS AND POSITIONS\n" + "- Briefly outline key arguments and positions taken by each party.\n\n" + "5. DECISIONS AND OUTCOMES\n" + "- Clearly list decisions made and outcomes reached.\n\n" + "6. ACTION ITEMS AND NEXT STEPS\n" + "- List all action items, responsibilities, and next steps.\n\n" + "7. UNRESOLVED ISSUES AND RISKS\n" + "- Note any unresolved issues, conflicts, or risks.\n\n" + "The summary should be detailed enough that a reader who was not present " + "can understand what happened and what is expected going forward. " + "Use clear, concise language and bullet points where appropriate." + ) + def _summarize_chunk(self, chunk: str, index: int, total: int) -> str: - system_prompt = ( - "You are an expert legal and business meeting summarizer. " - "You will receive a segment of a longer transcript. " - "Provide a detailed, structured summary of this segment, focusing on: " - "- Topics discussed\n" - "- Key points and arguments\n" - "- Decisions and agreements\n" - "- Action items and responsibilities\n" - "- Any risks, conflicts, or open issues\n\n" - "Be concise but complete. Use bullet points when helpful. " - "Do not add information that is not present in the transcript." - ) + system_prompt = self._load_summary_prompt("chunk") user_prompt = ( f"This is segment {index + 1} of {total} from a longer conversation.\n\n" @@ -170,20 +238,7 @@ class SummarizerClient: return self._chat_completion(system_prompt, user_prompt) def _summarize_combined(self, combined_summaries: str) -> str: - system_prompt = ( - "You are an expert legal and business meeting summarizer. " - "You will receive several intermediate summaries of a longer conversation. " - "Produce a single, comprehensive summary that makes it clear: " - "- The overall purpose and context of the discussion\n" - "- The main issues and topics addressed\n" - "- Key arguments and positions (briefly)\n" - "- Decisions and outcomes\n" - "- Action items, responsibilities, and next steps\n" - "- Any unresolved issues or risks\n\n" - "The summary should be detailed enough that a reader who was not present " - "can understand what happened and what is expected going forward. " - "Use clear, concise language and bullet points where appropriate." - ) + system_prompt = self._load_summary_prompt("combined") user_prompt = ( "Here are the intermediate summaries from different parts of the same conversation:\n\n" diff --git a/scraibe/tasks.py b/scraibe/tasks.py index df605f3..f0826c5 100644 --- a/scraibe/tasks.py +++ b/scraibe/tasks.py @@ -6,6 +6,7 @@ import os import json import logging import tempfile +import re from datetime import datetime from .celery_app import celery_app @@ -247,6 +248,112 @@ def send_error_email(to: str, error_message: str, task_id: str): logger.error("Failed to send error email to %s for job %s: %s", to, task_id, e) +def _accent_color() -> str: + """ + Return the accent color used in emails/templates. + """ + return (os.getenv("EMAIL_ACCENT_COLOR") or "#7C6DA0").strip() + + +def _identify_speakers_via_llm(scraibe, transcript_text: str) -> dict: + """ + Use the summarizer LLM to identify speakers in the transcript. + + Returns: dict mapping normalized speaker labels to name/role, e.g. + {"SPEAKER 1": "JUDGE MARTINEZ", "SPEAKER 2": "DEFENSE COUNSEL"} + """ + try: + scraibe._ensure_summarizer() + summarizer = scraibe._summarizer + except Exception as e: + logger.warning("Failed to initialize summarizer for speaker identification: %s", e) + return {} + + prompt = ( + "Below is a transcript with speaker labels like 'SPEAKER 1', 'SPEAKER 2', etc. " + "Based on the context and how each speaker talks, identify each speaker as:\n" + "- Their real name, if it is clearly mentioned or strongly implied, OR\n" + "- A concise role/position (e.g., Judge, Doctor, Manager, Interviewer, Client, Witness), " + "if their identity is not clear.\n" + "Do not invent random personal names. " + "Do not add extra commentary. Output ONLY a mapping in this exact format, one per line:\n" + "SPEAKER 1: Name or Role\n" + "SPEAKER 2: Name or Role\n" + "SPEAKER 3: Name or Role\n" + "\n" + "Transcript:\n" + + transcript_text + ) + + try: + reply = summarizer._chat_completion( + system_prompt="You are an expert legal and business meeting analyst.", + user_prompt=prompt, + ) + except Exception as e: + logger.warning("LLM call failed during speaker identification: %s", e) + return {} + + speaker_map = {} + for m in re.finditer(r"SPEAKER\s+(\d+)\s*:\s*(.+)", reply, re.IGNORECASE): + spk = f"SPEAKER {m.group(1).strip()}" + name = m.group(2).strip().rstrip(".").upper() + if name: + speaker_map[spk] = name + + logger.info("Speaker identification mapping: %s", speaker_map) + return speaker_map + + +def _apply_speaker_map(transcript_text: str, segments: list, speaker_map: dict) -> tuple: + """ + Apply speaker_map to: + - transcript_text lines like "[00:12] SPEAKER 1: ..." + - segment speaker fields. + + Returns (updated_transcript_text, updated_segments). + """ + if not speaker_map: + return transcript_text, segments + + # Normalize function: e.g. "SPEAKER 1" -> "SPEAKER 1" + def normalize_label(label: str) -> str: + return re.sub(r"\s+", " ", re.sub(r"[^A-Z0-9\s]", "", label.upper())).strip() + + # Replace in transcript lines + def replace_in_line(line: str) -> str: + # Match: [MM:SS] or [HH:MM:SS] then speaker label then colon + # Use a lazy match for the label up to the first colon + def repl(m): + prefix = m.group(1) # e.g. "[00:12] " + label = m.group(2).strip() + normalized = normalize_label(label) + new_label = speaker_map.get(normalized, label) + return f"{prefix}{new_label}: " + return re.sub( + r"(\[\d+:\d+(?::\d+)?\]\s*)(.+?):\s*", + repl, + line, + count=1, + ) + + updated_transcript = "\n".join( + replace_in_line(line) for line in transcript_text.splitlines() + ) + + # Update segments + updated_segments = [] + for seg in segments: + sp = (seg.get("speaker") or "").strip() + sp_norm = normalize_label(sp) + sp_new = speaker_map.get(sp_norm, sp) + seg = dict(seg) + seg["speaker"] = sp_new + updated_segments.append(seg) + + return updated_transcript, updated_segments + + @celery_app.task( name="scraibe.tasks.process_transcription_task", bind=True, @@ -266,7 +373,7 @@ def process_transcription_task( identify_speakers: bool = False, ): """ - Async task: transcribe audio, optionally summarize, then email results. + Async task: transcribe audio, optionally identify speakers, optionally summarize, then email results. Cleans up temporary files after completion. """ task_id = self.request.id @@ -294,121 +401,42 @@ def process_transcription_task( ) raise - # 3) Transcription - if task_type == "transcript_and_summarize": - result = scraibe.transcript_and_summarize( - audio_file=audio_path, - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, - verbose=True, - for_export=True, - ) - transcript_text = result.get("transcript", "") - summary_text = result.get("summary", "") - segments = result.get("segments", []) - raw_result = result.get("raw_result") - else: - result = scraibe.transcribe( - audio_file=audio_path, - language=language or None, - num_speakers=int(num_speakers) if num_speakers else None, - verbose=True, - for_export=True, - ) - transcript_text = result.get("transcript", "") - summary_text = "" - segments = result.get("segments", []) - raw_result = result.get("raw_result") + # 3) Transcription (always first, without summarization) + result = scraibe.transcribe( + audio_file=audio_path, + language=language or None, + num_speakers=int(num_speakers) if num_speakers else None, + verbose=True, + for_export=True, + ) + transcript_text = result.get("transcript", "") + segments = result.get("segments", []) + raw_result = result.get("raw_result") + summary_text = "" - # 3b) Optional speaker identification - speaker_map = {} # e.g. {"SPEAKER 1": "John", "SPEAKER 2": "Maria"} + # 4) Optional speaker identification (before summarization) if identify_speakers: try: - # Use the same summarizer client as transcript_and_summarize - scraibe._ensure_summarizer() - summarizer = scraibe._summarizer - - prompt = ( - "Below is a transcript with speaker labels like 'SPEAKER 1', 'SPEAKER 2', etc. " - "Based on the context and how each speaker talks, identify each speaker as:\n" - "- Their real name, if it is clearly mentioned or strongly implied, OR\n" - "- A concise role/position (e.g., Judge, Doctor, Manager, Interviewer, Client, Witness), " - "if their identity is not clear.\n" - "Do not invent random personal names. " - "Do not add extra commentary. Output ONLY a mapping in this exact format, one per line:\n" - "SPEAKER 1: Name or Role\n" - "SPEAKER 2: Name or Role\n" - "SPEAKER 3: Name or Role\n" - "\n" - "Transcript:\n" - + transcript_text - ) - - response = summarizer._chat_completion( - messages=[{"role": "user", "content": prompt}], - temperature=0.3, - max_tokens=300, - ) - reply = (response or {}).get("choices", [{}])[0].get("message", {}).get("content", "") - - # Parse mapping - import re - for m in re.finditer( - r"SPEAKER\s+(\d+)\s*:\s*(.+)", - reply, - re.IGNORECASE, - ): - spk = f"SPEAKER {m.group(1).strip()}" - name = m.group(2).strip().rstrip(".").upper() - if name: - speaker_map[spk] = name - - logger.info("Speaker identification mapping: %s", speaker_map) - - # Apply mapping to transcript text + speaker_map = _identify_speakers_via_llm(scraibe, transcript_text) if speaker_map: - def replace_speaker(m): - label = m.group(0).strip() - # normalize to "SPEAKER N" - normalized = re.sub( - r"\s+", - " ", - re.sub(r"[^A-Z0-9\s]", "", label.upper()), - ).strip() - return speaker_map.get(normalized, label) - - # Replace in lines like "[00:12] SPEAKER 1:" but preserve timestamp and colon - def replace_in_line(line: str) -> str: - # match after timestamp bracket and space: "SPEAKER N:" - return re.sub( - r"(\[\d+:\d+(?::\d+)?\]\s*)([A-Z\s]+?):\s*", - lambda m: m.group(1) + (speaker_map.get(m.group(2).strip(), m.group(2)) + ": "), - line, - ) - - transcript_lines = transcript_text.splitlines() - transcript_text = "\n".join( - replace_in_line(line) for line in transcript_lines + transcript_text, segments = _apply_speaker_map( + transcript_text, segments, speaker_map ) - - # Also update segments for JSON export - updated_segments = [] - for seg in segments: - sp = (seg.get("speaker") or "").strip() - sp_norm = re.sub(r"[^A-Z0-9\s]", "", sp.upper()).strip() - sp_new = speaker_map.get(sp_norm, sp) - seg = dict(seg) - seg["speaker"] = sp_new - updated_segments.append(seg) - segments = updated_segments - - except (SummarizerError, Exception) as e: + logger.info("Applied speaker identification to transcript and segments.") + except Exception as e: logger.warning( - "Speaker identification failed; falling back to Speaker IDs: %s", e + "Speaker identification failed; continuing with original labels: %s", e ) - speaker_map = {} - # 4) Prepare files + # 5) Summarization (if requested) using the (possibly identified) transcript + if include_summary: + try: + summary_text = scraibe._summarizer.summarize_transcript(transcript_text) + except Exception as e: + logger.warning("Summarization failed; continuing without summary: %s", e) + summary_text = "" + + # 6) Prepare files # Transcript .md md_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".md") @@ -417,12 +445,19 @@ def process_transcription_task( f.write(transcript_text) temp_files.append(md_transcript_path) - # Transcript .docx (standalone, no cover page) + # Transcript .docx (with summary appended if present) docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx") - create_transcript_docx( - transcript_text, - docx_transcript_path, - ) + if summary_text: + create_transcript_docx( + transcript_text, + docx_transcript_path, + summary_text=summary_text, + ) + else: + create_transcript_docx( + transcript_text, + docx_transcript_path, + ) temp_files.append(docx_transcript_path) # JSON as SOURCE @@ -445,40 +480,14 @@ def process_transcription_task( json.dump(json_data, f, indent=2, ensure_ascii=False) temp_files.append(json_path) - # Summary files (if present) - md_summary_path = None - docx_summary_path = None - - if summary_text: - # Summary .md - md_summary_path = _safe_filename("SUMMARY", local, date_tag, ".md") - with open(md_summary_path, "w", encoding="utf-8") as f: - f.write("# Summary\n\n") - f.write(summary_text) - temp_files.append(md_summary_path) - - # Summary .docx (standalone, no cover page) - docx_summary_path = _safe_filename("SUMMARY", local, date_tag, ".docx") - create_summary_docx( - summary_text, - docx_summary_path, - ) - temp_files.append(docx_summary_path) - - # 5) Build attachments list - - # Always: JSON, transcript MD, transcript DOCX + # 7) Build attachments list attachments = [ md_transcript_path, docx_transcript_path, json_path, ] - # If summary is present, add summary MD and DOCX - if summary_text: - attachments += [md_summary_path, docx_summary_path] - - # 6) Send success email + # 8) Send success email send_success_email( to=email_to, transcript_text=transcript_text, @@ -498,9 +507,151 @@ def process_transcription_task( ) raise e finally: - # 7) Cleanup + # 9) Cleanup for path in temp_files: _remove_file(path) if audio_path: _remove_file(audio_path) logger.info("Cleanup completed for job %s.", task_id) + + +@celery_app.task( + name="scraibe.tasks.process_watch_file_task", + bind=True, + max_retries=1, + task_time_limit=14400, + task_soft_time_limit=13500, +) +def process_watch_file_task( + self, + file_path: str, +): + """ + Async task for watch-folder mode: + - Transcribe + summarize + - Email results + - Optionally delete source file + """ + task_id = self.request.id + + log_level = os.getenv("LOG_LEVEL", "INFO") + setup_logging(level=log_level) + + email_to = os.getenv("WATCH_EMAIL_TO") or os.getenv("EMAIL_DEFAULT_TO") + if not email_to: + logger.error("No email address configured for watch-folder mode.") + raise RuntimeError("WATCH_EMAIL_TO or EMAIL_DEFAULT_TO not set.") + + delete_on_success = os.getenv("WATCH_DELETE_ON_SUCCESS", "true").strip().lower() in ("true", "1", "yes") + + temp_files = [] + local = "watch" + date_tag = _date_tag() + + try: + scraibe = Scraibe(verbose=True) + + result = scraibe.transcript_and_summarize( + audio_file=file_path, + language=None, + num_speakers=None, + verbose=True, + for_export=True, + ) + + transcript_text = result.get("transcript", "") + summary_text = result.get("summary", "") + segments = result.get("segments", []) + raw_result = result.get("raw_result") + + # Transcript .md + md_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".md") + with open(md_transcript_path, "w", encoding="utf-8") as f: + f.write("# Transcript\n\n") + f.write(transcript_text) + temp_files.append(md_transcript_path) + + # Transcript .docx + docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx") + create_transcript_docx( + transcript_text, + docx_transcript_path, + ) + temp_files.append(docx_transcript_path) + + # Summary .md + md_summary_path = _safe_filename("SUMMARY", local, date_tag, ".md") + with open(md_summary_path, "w", encoding="utf-8") as f: + f.write("# Summary\n\n") + f.write(summary_text) + temp_files.append(md_summary_path) + + # Summary .docx + docx_summary_path = _safe_filename("SUMMARY", local, date_tag, ".docx") + create_summary_docx( + summary_text, + docx_summary_path, + ) + temp_files.append(docx_summary_path) + + # JSON as SOURCE + json_data = { + "task": "watch_transcript_and_summarize", + "transcript": transcript_text, + "summary": summary_text, + "segments": segments, + "metadata": { + "timestamp": datetime.utcnow().isoformat(), + "job_id": task_id, + "source_file": file_path, + }, + } + if raw_result is not None: + json_data["raw_result"] = raw_result + + json_path = _safe_filename("SOURCE", local, date_tag, ".json") + with open(json_path, "w", encoding="utf-8") as f: + json.dump(json_data, f, indent=2, ensure_ascii=False) + temp_files.append(json_path) + + # Attachments + attachments = [ + md_transcript_path, + docx_transcript_path, + md_summary_path, + docx_summary_path, + json_path, + ] + + # Send email + send_success_email( + to=email_to, + transcript_text=transcript_text, + summary_text=summary_text, + attachments=attachments, + task_id=task_id, + ) + + logger.info("Watch-folder job %s completed for %s.", task_id, file_path) + + # Delete source file if configured + if delete_on_success and os.path.exists(file_path): + try: + os.remove(file_path) + logger.info("Deleted source file: %s", file_path) + except Exception as e: + logger.warning("Failed to delete source file %s: %s", file_path, e) + + except Exception as e: + logger.error("Error processing watch file %s: %s", file_path, e, exc_info=True) + send_error_email( + to=email_to, + error_message=str(e), + task_id=task_id, + ) + raise e + finally: + # Cleanup temp files + for path in temp_files: + _remove_file(path) + logger.info("Watch-folder job %s cleanup completed.", task_id) diff --git a/scraibe/watcher.py b/scraibe/watcher.py new file mode 100644 index 0000000..f394ab2 --- /dev/null +++ b/scraibe/watcher.py @@ -0,0 +1,100 @@ +""" +Watch-folder mode for ScrAIbe. + +Monitors a folder for audio files. For each file: +- Transcribes + summarizes +- Emails results +- Deletes source file + +Configuration (env): +- WATCH_ENABLED: "true"/"false" (default: false) +- WATCH_DIR: directory to watch (required if enabled) +- WATCH_EMAIL_TO: destination email (required if enabled) +- WATCH_POLL_INTERVAL: seconds between scans (default: 10) +- WATCH_DELETE_ON_SUCCESS: "true"/"false" (default: true) +""" + +import os +import time +import logging +import threading +from pathlib import Path + +logger = logging.getLogger("scraibe.watcher") + +AUDIO_EXTENSIONS = { + ".wav", + ".mp3", + ".flac", + ".m4a", + ".ogg", + ".webm", + ".mp4", +} + + +def _is_audio(path: Path) -> bool: + return path.is_file() and path.suffix.lower() in AUDIO_EXTENSIONS + + +def _enqueue_file(file_path: Path): + """ + Enqueue a file for transcription + summarization via Celery. + """ + from .tasks import process_watch_file_task + + try: + process_watch_file_task.delay(str(file_path)) + except Exception as e: + logger.error("Failed to enqueue watch file %s: %s", file_path, e) + + +def _scan_directory(watch_dir: Path): + """ + Scan directory and enqueue all audio files. + """ + if not watch_dir.is_dir(): + logger.warning("WATCH_DIR does not exist or is not a directory: %s", watch_dir) + return + + for p in watch_dir.iterdir(): + if _is_audio(p): + logger.info("Found audio file in WATCH_DIR: %s", p) + _enqueue_file(p) + + +def start_watcher(): + """ + Start watch-folder loop in a background thread. + """ + enabled = os.getenv("WATCH_ENABLED", "false").strip().lower() in ("true", "1", "yes") + if not enabled: + return + + watch_dir = os.getenv("WATCH_DIR") + if not watch_dir: + logger.warning("WATCH_ENABLED is true but WATCH_DIR is not set. Watcher disabled.") + return + + email_to = os.getenv("WATCH_EMAIL_TO") + if not email_to: + logger.warning("WATCH_ENABLED is true but WATCH_EMAIL_TO is not set. Watcher disabled.") + return + + interval = float(os.getenv("WATCH_POLL_INTERVAL", "10")) + + watch_path = Path(watch_dir).expanduser().resolve() + watch_path.mkdir(parents=True, exist_ok=True) + + logger.info("Starting watch-folder: dir=%s, email=%s, interval=%s", watch_dir, email_to, interval) + + def _loop(): + while True: + try: + _scan_directory(watch_path) + except Exception as e: + logger.error("Error scanning WATCH_DIR: %s", e) + time.sleep(interval) + + t = threading.Thread(target=_loop, daemon=True) + t.start() diff --git a/tools/email_diagnostics.py b/tools/email_diagnostics.py new file mode 100644 index 0000000..e35c170 --- /dev/null +++ b/tools/email_diagnostics.py @@ -0,0 +1,125 @@ +""" +Diagnostic script for ScrAIbe email (SMTP) connectivity. + +Run inside the container to verify: +- Required env vars are set +- SMTP connection +- TLS (if configured) +- Authentication + +Output is written to stdout/stderr so it appears in docker logs. +""" + +import os +import sys +import smtplib +import logging + +logging.basicConfig( + level=logging.INFO, + format="DIAG [%(levelname)s] %(message)s", + stream=sys.stdout, + force=True, +) +log = logging.getLogger("email_diagnostics") + + +def main(): + # 1) Check required env vars + required = [ + "EMAIL_SMTP_HOST", + "EMAIL_SMTP_PORT", + "EMAIL_SMTP_USER", + "EMAIL_SMTP_PASSWORD", + "EMAIL_FROM_ADDRESS", + ] + missing = [v for v in required if not os.getenv(v)] + if missing: + log.error("Missing required env vars: %s", ", ".join(missing)) + sys.exit(1) + + smtp_host = os.getenv("EMAIL_SMTP_HOST") + smtp_port = int(os.getenv("EMAIL_SMTP_PORT")) + smtp_user = os.getenv("EMAIL_SMTP_USER") + smtp_password = os.getenv("EMAIL_SMTP_PASSWORD") + from_address = os.getenv("EMAIL_FROM_ADDRESS") + use_tls_str = (os.getenv("EMAIL_SMTP_USE_TLS") or "true").strip().lower() + use_tls = use_tls_str not in ("false", "0", "no") + + log.info("SMTP config:") + log.info(" host: %s", smtp_host) + log.info(" port: %s", smtp_port) + log.info(" user: %s", smtp_user) + log.info(" from: %s", from_address) + log.info(" use_tls: %s", use_tls) + log.info(" password: %s", ("set" if smtp_password else "NOT SET")) + + # 2) Connect to SMTP server + log.info("Attempting SMTP connection...") + try: + if use_tls: + server = smtplib.SMTP(smtp_host, smtp_port, timeout=15) + log.info("Connected (SMTP) to %s:%s", smtp_host, smtp_port) + else: + server = smtplib.SMTP(smtp_host, smtp_port, timeout=15) + log.info("Connected (SMTP, no TLS) to %s:%s", smtp_host, smtp_port) + except Exception as e: + log.error("SMTP connection failed: %s", e) + sys.exit(1) + + # 3) EHLO + log.info("Sending EHLO...") + try: + ehlo_resp = server.ehlo() + log.info("EHLO response code: %s", ehlo_resp[0]) + except Exception as e: + log.error("EHLO failed: %s", e) + server.quit() + sys.exit(1) + + # 4) STARTTLS if configured + if use_tls: + log.info("Attempting STARTTLS...") + try: + server.starttls() + server.ehlo() + log.info("STARTTLS succeeded.") + except Exception as e: + log.error("STARTTLS failed: %s", e) + server.quit() + sys.exit(1) + else: + log.info("TLS not requested; continuing without STARTTLS.") + + # 5) AUTH LOGIN + log.info("Attempting AUTH LOGIN with user: %s", smtp_user) + try: + server.login(smtp_user, smtp_password) + log.info("AUTH LOGIN succeeded.") + except smtplib.SMTPAuthenticationError as e: + log.error("AUTH LOGIN failed (bad credentials?): %s", e) + server.quit() + sys.exit(1) + except Exception as e: + log.error("AUTH LOGIN failed: %s", e) + server.quit() + sys.exit(1) + + # 6) Optional: quick MAIL FROM / RCPT TO / QUIT test (no message sent) + log.info("Testing MAIL FROM / RCPT TO / RSET...") + try: + server.mail(from_address) + # Use from_address as recipient for test + server.rcpt(from_address) + server.reset() + log.info("MAIL FROM / RCPT TO / RSET succeeded.") + except Exception as e: + log.warning("MAIL FROM / RCPT TO / RSET failed (non-critical): %s", e) + + # 7) Quit + server.quit() + log.info("All email diagnostics passed. SMTP is reachable and authenticated.") + + +if __name__ == "__main__": + main()