Merge branch 'dev' into main
This commit is contained in:
@@ -36,6 +36,9 @@ ENV CELERY_BROKER_URL=redis://localhost:6379/0
|
||||
ENV CELERY_RESULT_BACKEND=redis://localhost:6379/0
|
||||
ENV SCRAIBE_UPLOAD_DIR=/tmp/scraibe_uploads
|
||||
|
||||
# Server timezone for document dates (e.g., America/Toronto, Europe/Berlin)
|
||||
ENV SERVER_TIMEZONE=UTC
|
||||
|
||||
# Email and template configuration
|
||||
ENV EMAIL_CONTACT_ADDRESS=support@example.com
|
||||
ENV EMAIL_CSS_PATH=
|
||||
|
||||
@@ -7,6 +7,7 @@ ScrAIbe is a transcription and summarization service that:
|
||||
- Provides:
|
||||
- A web GUI for uploading audio and receiving transcripts via email.
|
||||
- A CLI and Python API for direct integration.
|
||||
- A watch-folder mode for automatic transcription, summarization, and email delivery.
|
||||
|
||||
No local speech models or heavy dependencies are required. ScrAIbe is designed as a thin client in front of your own AI services.
|
||||
|
||||
@@ -24,7 +25,8 @@ For more information: https://apstrom.ca
|
||||
- Key decisions and outcomes
|
||||
- Action items and responsibilities
|
||||
- Open issues and risks
|
||||
- Async web GUI:
|
||||
- Improved, configurable summary prompts (via environment or file).
|
||||
- Async web GUI (always enabled):
|
||||
- Upload audio via browser.
|
||||
- Jobs are queued and processed in the background (Celery + Redis).
|
||||
- Emails:
|
||||
@@ -32,13 +34,25 @@ For more information: https://apstrom.ca
|
||||
- Final transcript (MD + DOCX + JSON) when ready.
|
||||
- Summary as MD + DOCX (if requested).
|
||||
- Error notification if processing fails.
|
||||
- Watch-folder mode (optional):
|
||||
- Monitors a directory for audio files.
|
||||
- For each file:
|
||||
- Transcribes and summarizes.
|
||||
- Emails transcript + summary + JSON to a configured address.
|
||||
- Deletes the source file after successful processing (configurable).
|
||||
- Enabled via WATCH_ENABLED=true.
|
||||
- File formats:
|
||||
- Transcript: .md and .docx (line-numbered, no cover page)
|
||||
- Summary (if requested): .md and .docx (no line numbering, no cover page)
|
||||
- Transcript:
|
||||
- .md
|
||||
- .docx (line-numbered, 30 lines per page, optional cover page)
|
||||
- Summary (if requested):
|
||||
- .md
|
||||
- .docx (markdown-aware WYSIWYG styling, optional cover page)
|
||||
- Full structured output: .json
|
||||
- Customizable branding:
|
||||
- Web GUI title, logo, and accent color via environment variables.
|
||||
- Email logo, accent color, and subject lines via environment variables.
|
||||
- Optional cover pages for transcript and summary DOCX.
|
||||
- CLI and Python API:
|
||||
- Simple command-line interface.
|
||||
- Drop-in Scraibe class for integration into other tools.
|
||||
@@ -58,7 +72,8 @@ For more information: https://apstrom.ca
|
||||
- Chunked summarization
|
||||
- Output formatting (e.g., .md with transcript + summary)
|
||||
- Runs:
|
||||
- Web GUI (Gradio)
|
||||
- Web GUI (Gradio) – always enabled
|
||||
- Watch-folder mode – optional
|
||||
- Celery worker (async processing)
|
||||
- Redis (in-container by default)
|
||||
|
||||
@@ -209,6 +224,20 @@ Accent color (UI and emails):
|
||||
- Email headings, links, and email addresses
|
||||
- Default: #7C6DA0
|
||||
|
||||
Watch-folder mode:
|
||||
|
||||
- WATCH_ENABLED:
|
||||
- Enable watch-folder mode (default: false).
|
||||
- Values: true/false.
|
||||
- WATCH_DIR:
|
||||
- Directory to monitor for audio files (required if WATCH_ENABLED=true).
|
||||
- WATCH_EMAIL_TO:
|
||||
- Email address to send transcript and summary (required if WATCH_ENABLED=true).
|
||||
- WATCH_POLL_INTERVAL:
|
||||
- Seconds between scans (default: 10).
|
||||
- WATCH_DELETE_ON_SUCCESS:
|
||||
- Delete source file after successful processing (default: true).
|
||||
|
||||
Async processing (Celery + Redis):
|
||||
|
||||
- CELERY_BROKER_URL:
|
||||
@@ -253,16 +282,40 @@ Email subject lines (customizable):
|
||||
- Subject for error notification email.
|
||||
- Default: "ScrAIbe: Error with your transcription request"
|
||||
|
||||
Output files (async web GUI):
|
||||
Summary prompt customization:
|
||||
|
||||
- SUMMARY_PROMPT_CHUNK:
|
||||
- Override prompt used for each transcript chunk.
|
||||
- SUMMARY_PROMPT_COMBINED:
|
||||
- Override prompt used for the final combined summary.
|
||||
- SUMMARY_PROMPT_FILE:
|
||||
- Path to a file with prompts in sections:
|
||||
- [chunk]
|
||||
- [combined]
|
||||
|
||||
DOCX and cover pages:
|
||||
|
||||
- COVER_PAGE_ENABLED:
|
||||
- Add a cover page to transcript and summary DOCX files (default: false).
|
||||
- COVER_PAGE_ORGANIZATION:
|
||||
- Organization name shown on the cover page.
|
||||
- COVER_PAGE_TITLE_PREFIX:
|
||||
- Title prefix (e.g., "TRANSCRIPT" or "SUMMARY").
|
||||
- COVER_PAGE_LOGO_URL:
|
||||
- Logo URL to include on the cover page.
|
||||
- COVER_PAGE_LOGO_PATH:
|
||||
- Local logo path to include on the cover page.
|
||||
|
||||
Output files (async web GUI and watch-folder mode):
|
||||
|
||||
When a job completes, the user receives:
|
||||
|
||||
- Transcript:
|
||||
- .md file
|
||||
- .docx file (line-numbered, no cover page)
|
||||
- .docx file (line-numbered, 30 lines per page, optional cover page)
|
||||
- Summary (if requested):
|
||||
- .md file
|
||||
- .docx file (no line numbering, no cover page)
|
||||
- .docx file (markdown-aware styling, optional cover page)
|
||||
- JSON:
|
||||
- Structured transcript with diarization and metadata
|
||||
|
||||
|
||||
+17
-1
@@ -3,10 +3,26 @@ Entrypoint for running ScrAIbe as a module:
|
||||
|
||||
python -m scraibe
|
||||
|
||||
Always launches the Web GUI (Gradio), never the CLI.
|
||||
Always launches the Web GUI (Gradio).
|
||||
Optionally launches:
|
||||
- Watch-folder mode
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("scraibe.__main__")
|
||||
|
||||
from .webui import create_app
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Optionally start watch-folder mode (non-blocking)
|
||||
try:
|
||||
from .watcher import start_watcher
|
||||
start_watcher()
|
||||
logger.info("Watch-folder mode started.")
|
||||
except Exception as e:
|
||||
logger.warning("Failed to start watch-folder mode (WebUI will continue): %s", e)
|
||||
|
||||
# Always start WebUI (Gradio)
|
||||
create_app()
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Cover-page generator for transcript and summary DOCX files.
|
||||
|
||||
Layout (all centered):
|
||||
- 1 blank line
|
||||
- Title in uppercase: "TRANSCRIPT" or "SUMMARY"
|
||||
- Next line: "Transcript created on: [date]"
|
||||
- Next line: "Transcribed by <COVER_PAGE_ORGANIZATION>"
|
||||
- 2 blank lines
|
||||
- Disclaimer (underlined)
|
||||
- Disclaimer body
|
||||
- Page break (ensures content starts on page 2)
|
||||
|
||||
Environment variables:
|
||||
- COVER_PAGE_ENABLED: "true"/"false" (default: false)
|
||||
- COVER_PAGE_ORGANIZATION: e.g., "A.P.Strom"
|
||||
- SERVER_TIMEZONE: Timezone for document dates (e.g., "America/Toronto"). Defaults to "UTC".
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
from docx import Document
|
||||
from docx.shared import Pt
|
||||
from docx.enum.text import WD_ALIGN_PARAGRAPH
|
||||
from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
|
||||
|
||||
def _get_server_timezone():
|
||||
tz_name = (os.getenv("SERVER_TIMEZONE") or "UTC").strip()
|
||||
try:
|
||||
return ZoneInfo(tz_name)
|
||||
except (ZoneInfoNotFoundError, KeyError, ValueError):
|
||||
return ZoneInfo("UTC")
|
||||
|
||||
|
||||
def _add_centered_paragraph(doc: Document, text: str = "", font_size: Pt = Pt(12), bold: bool = False, underline: bool = False):
|
||||
"""Add a centered paragraph with given text and style."""
|
||||
p = doc.add_paragraph()
|
||||
p.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
if text:
|
||||
run = p.add_run(text)
|
||||
run.font.name = "Courier"
|
||||
run.font.size = font_size
|
||||
run.bold = bold
|
||||
run.underline = underline
|
||||
return p
|
||||
|
||||
|
||||
def _add_blank_lines(doc: Document, count: int):
|
||||
"""Add specified number of blank lines (empty centered paragraphs)."""
|
||||
for _ in range(count):
|
||||
_add_centered_paragraph(doc)
|
||||
|
||||
|
||||
def _add_page_break(doc: Document):
|
||||
"""Insert a page break after the last paragraph (no extra blank line)."""
|
||||
from docx.enum.text import WD_BREAK
|
||||
last_p = doc.paragraphs[-1]
|
||||
run = last_p.add_run()
|
||||
run.add_break(WD_BREAK.PAGE)
|
||||
|
||||
|
||||
def add_cover_page(
|
||||
doc: Document,
|
||||
title: str,
|
||||
date_str: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Insert a cover page at the current cursor position.
|
||||
|
||||
- title: e.g., "TRANSCRIPT" or "SUMMARY"
|
||||
- date_str: optional date string (e.g., "June 19, 2026");
|
||||
if not provided, uses current server time in SERVER_TIMEZONE.
|
||||
"""
|
||||
|
||||
# 1 blank line (removed two from top)
|
||||
_add_blank_lines(doc, 1)
|
||||
|
||||
# Title in uppercase
|
||||
_add_centered_paragraph(doc, text=title.upper().strip(), font_size=Pt(18), bold=True)
|
||||
|
||||
# Date of transcription (use provided or current server time)
|
||||
if not date_str:
|
||||
tz = _get_server_timezone()
|
||||
now = datetime.now(tz)
|
||||
date_str = now.strftime("%B %d, %Y")
|
||||
_add_centered_paragraph(doc, text=f"Transcript created on: {date_str}", font_size=Pt(14))
|
||||
|
||||
# Transcribed by + organization
|
||||
org = (os.getenv("COVER_PAGE_ORGANIZATION") or "").strip()
|
||||
if org:
|
||||
_add_centered_paragraph(doc, text=f"Transcribed by {org}", font_size=Pt(12))
|
||||
|
||||
# 2 blank lines
|
||||
_add_blank_lines(doc, 2)
|
||||
|
||||
# Disclaimer: only the word "Disclaimer" is underlined
|
||||
disclaimer_title = "Disclaimer"
|
||||
disclaimer_body = (
|
||||
"This transcription and any summary derived therefrom are created entirely by large language models. "
|
||||
"Large language models are imperfect and may create errors or inaccuracies. "
|
||||
"The following transcription and summary (if applicable) are designed to assist users. "
|
||||
"They are not a conclusive record or summary of what transpired in an audio recording. "
|
||||
"Use at your own risk."
|
||||
)
|
||||
|
||||
# Underlined "Disclaimer" line
|
||||
_add_centered_paragraph(doc, text=disclaimer_title, font_size=Pt(10), underline=True)
|
||||
|
||||
# Body text (not underlined)
|
||||
_add_centered_paragraph(doc, text=disclaimer_body, font_size=Pt(10), underline=False)
|
||||
|
||||
# Page break after cover page so main content starts on page 2
|
||||
_add_page_break(doc)
|
||||
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Utility module for applying styles and converting simple markdown
|
||||
into styled DOCX paragraphs/runs for summaries.
|
||||
"""
|
||||
|
||||
import re
|
||||
from docx import Document
|
||||
from docx.shared import Pt
|
||||
from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
|
||||
|
||||
def _ensure_style(doc, name, based_on="Normal", font_name="Courier", font_size=Pt(12)):
|
||||
"""
|
||||
Ensure a paragraph style exists in the document.
|
||||
"""
|
||||
styles = doc.styles
|
||||
if name not in [s.name for s in styles]:
|
||||
style = styles.add_style(name, 1) # 1 = WD_STYLE_TYPE.PARAGRAPH
|
||||
style.font.name = font_name
|
||||
style.font.size = font_size
|
||||
if based_on:
|
||||
style.base_style = styles[based_on]
|
||||
return styles[name]
|
||||
|
||||
|
||||
def apply_heading_style(doc, paragraph, level: int):
|
||||
"""
|
||||
Apply heading style to a paragraph based on level (1, 2, 3).
|
||||
"""
|
||||
if level == 1:
|
||||
style_name = "SummaryHeading1"
|
||||
size = Pt(16)
|
||||
elif level == 2:
|
||||
style_name = "SummaryHeading2"
|
||||
size = Pt(14)
|
||||
else:
|
||||
style_name = "SummaryHeading3"
|
||||
size = Pt(12)
|
||||
|
||||
style = _ensure_style(doc, style_name, font_size=size)
|
||||
paragraph.style = style
|
||||
paragraph.paragraph_format.space_before = Pt(4)
|
||||
paragraph.paragraph_format.space_after = Pt(2)
|
||||
|
||||
|
||||
def apply_bullet_style(doc, paragraph):
|
||||
"""
|
||||
Apply a simple bullet style to a paragraph.
|
||||
"""
|
||||
style_name = "SummaryBullet"
|
||||
style = _ensure_style(doc, style_name)
|
||||
paragraph.style = style
|
||||
pPr = paragraph._p.get_or_add_pPr()
|
||||
tabs = OxmlElement("w:tabs")
|
||||
tab = OxmlElement("w:tab")
|
||||
tab.set(qn("w:val"), "left")
|
||||
tab.set(qn("w:pos"), "360")
|
||||
tabs.append(tab)
|
||||
pPr.append(tabs)
|
||||
|
||||
|
||||
def parse_simple_md_to_paragraphs(doc, text: str):
|
||||
"""
|
||||
Convert simple markdown text into DOCX paragraphs with styles.
|
||||
|
||||
Supported:
|
||||
- # / ## / ### for headings
|
||||
- - / * for bullet lists
|
||||
- **bold** and *italic*
|
||||
|
||||
This is intentionally simple and robust for legal/business summaries.
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
current_paragraph = None
|
||||
in_list = False
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
current_paragraph = None
|
||||
in_list = False
|
||||
continue
|
||||
|
||||
# Headings
|
||||
heading_match = re.match(r"^(#{1,3})\s+(.*)", stripped)
|
||||
if heading_match:
|
||||
level = len(heading_match.group(1))
|
||||
content = heading_match.group(2).strip()
|
||||
p = doc.add_paragraph()
|
||||
apply_heading_style(doc, p, level)
|
||||
_add_run_with_inline_md(p, content)
|
||||
current_paragraph = p
|
||||
in_list = False
|
||||
continue
|
||||
|
||||
# Bullet list
|
||||
bullet_match = re.match(r"^[-*]\s+(.*)", stripped)
|
||||
if bullet_match:
|
||||
content = bullet_match.group(1).strip()
|
||||
if not in_list or current_paragraph is None:
|
||||
in_list = True
|
||||
current_paragraph = doc.add_paragraph()
|
||||
apply_bullet_style(doc, current_paragraph)
|
||||
else:
|
||||
current_paragraph = doc.add_paragraph()
|
||||
apply_bullet_style(doc, current_paragraph)
|
||||
_add_run_with_inline_md(current_paragraph, content)
|
||||
continue
|
||||
|
||||
# Normal paragraph
|
||||
if not in_list or current_paragraph is None:
|
||||
in_list = False
|
||||
current_paragraph = doc.add_paragraph()
|
||||
else:
|
||||
current_paragraph = doc.add_paragraph()
|
||||
|
||||
_add_run_with_inline_md(current_paragraph, stripped)
|
||||
|
||||
|
||||
def _add_run_with_inline_md(paragraph, text: str):
|
||||
"""
|
||||
Add runs to a paragraph, interpreting **bold** and *italic*.
|
||||
"""
|
||||
# Simple regex for bold and italic
|
||||
parts = re.split(r"(\*\*\*.*?\*\*\*|\*\*.*?\*\*|\*.*?\*)", text)
|
||||
for part in parts:
|
||||
if not part:
|
||||
continue
|
||||
|
||||
run = paragraph.add_run(part)
|
||||
run.font.name = "Courier"
|
||||
run.font.size = Pt(12)
|
||||
|
||||
# Bold
|
||||
bold_match = re.fullmatch(r"\*\*(.+?)\*\*", part)
|
||||
if bold_match:
|
||||
run.bold = True
|
||||
part = bold_match.group(1)
|
||||
|
||||
# Italic
|
||||
italic_match = re.fullmatch(r"\*(.+?)\*", part)
|
||||
if italic_match:
|
||||
run.italic = True
|
||||
part = italic_match.group(1)
|
||||
|
||||
run.text = part
|
||||
+74
-23
@@ -439,7 +439,7 @@ def _add_transcript_paragraph(doc, line_text, line_number):
|
||||
|
||||
# ------------ Public DOCX functions ------------
|
||||
|
||||
def create_transcript_docx(text: str, filename: str):
|
||||
def create_transcript_docx(text: str, filename: str, summary_text: str = ""):
|
||||
"""
|
||||
Create a transcript DOCX with:
|
||||
- 1" margins on all sides
|
||||
@@ -450,10 +450,14 @@ def create_transcript_docx(text: str, filename: str):
|
||||
- Blank spacing between number and text preserved
|
||||
- Page break after every 29 lines
|
||||
- Centered footer: "X of Y"
|
||||
- If summary_text is provided:
|
||||
- Page break after transcript
|
||||
- Centered "SUMMARY" title
|
||||
- Summary content (markdown-aware)
|
||||
"""
|
||||
from . import docx_styles
|
||||
|
||||
# Step 1: Prepare transcript into pages of 29 lines each
|
||||
# Each line <= 60 chars total, words preserved, no clipping
|
||||
# Structure: nested list of paragraphs (pages -> lines)
|
||||
prepared_pages = []
|
||||
current_page = []
|
||||
line_count = 0
|
||||
@@ -483,7 +487,7 @@ def create_transcript_docx(text: str, filename: str):
|
||||
|
||||
# Add segments to pages, enforcing 29 lines per page
|
||||
for seg in segments:
|
||||
if line_count == 30:
|
||||
if line_count == 29:
|
||||
prepared_pages.append(current_page)
|
||||
current_page = []
|
||||
line_count = 0
|
||||
@@ -505,25 +509,63 @@ def create_transcript_docx(text: str, filename: str):
|
||||
|
||||
_create_transcript_section_properties(doc.sections[0])
|
||||
|
||||
# Step 3: Write prepared pages into DOCX
|
||||
# Step 3: Optionally add cover page
|
||||
from . import docx_cover
|
||||
from datetime import datetime
|
||||
cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes")
|
||||
if cover_enabled:
|
||||
date_str = datetime.utcnow().strftime("%B %d, %Y")
|
||||
docx_cover.add_cover_page(
|
||||
doc,
|
||||
title="TRANSCRIPT",
|
||||
date_str=date_str,
|
||||
)
|
||||
|
||||
# Step 4: Write prepared pages into DOCX
|
||||
for page_idx, page_lines in enumerate(prepared_pages):
|
||||
# Insert page break between pages
|
||||
if page_idx > 0:
|
||||
# Write each line with its number (1-29)
|
||||
for line_num, line_text in enumerate(page_lines, start=1):
|
||||
_add_transcript_paragraph(doc, line_text, line_number=line_num)
|
||||
|
||||
# After each page except the last, add a page break via next paragraph
|
||||
if page_idx < len(prepared_pages) - 1:
|
||||
p_break = doc.add_paragraph()
|
||||
p_break.paragraph_format.page_break_before = True
|
||||
# Ensure it’s empty and doesn’t show as a visible line
|
||||
if p_break.runs:
|
||||
for r in p_break.runs:
|
||||
r.text = ""
|
||||
# Remove any formatting that would show as a visible line
|
||||
pPr = p_break._p.get_or_add_pPr()
|
||||
for child in list(pPr):
|
||||
tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
|
||||
if tag in ("tabs", "spacing", "ind"):
|
||||
pPr.remove(child)
|
||||
page_break = OxmlElement("w:pageBreak")
|
||||
page_break.set("{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val", "1")
|
||||
pPr.append(page_break)
|
||||
spacing = OxmlElement("w:spacing")
|
||||
_set_element_attr(spacing, "before", "0")
|
||||
_set_element_attr(spacing, "after", "0")
|
||||
_set_element_attr(spacing, "line", "240")
|
||||
_set_element_attr(spacing, "lineRule", "auto")
|
||||
pPr.append(spacing)
|
||||
|
||||
# Write each line with its number (1-29)
|
||||
for line_num, line_text in enumerate(page_lines, start=1):
|
||||
_add_transcript_paragraph(doc, line_text, line_number=line_num)
|
||||
# Step 5: If summary_text provided, append it with a hard page break
|
||||
if summary_text and summary_text.strip():
|
||||
# Hard page break after transcript
|
||||
p_break = doc.add_paragraph()
|
||||
p_break.paragraph_format.page_break_before = True
|
||||
|
||||
# Step 4: Add footer: "X of Y" centered
|
||||
# Centered "SUMMARY" title
|
||||
title_p = doc.add_paragraph()
|
||||
title_p.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
run = title_p.add_run("SUMMARY")
|
||||
run.font.name = "Courier"
|
||||
run.font.size = Pt(18)
|
||||
run.bold = True
|
||||
|
||||
# Add summary content using markdown-aware styling
|
||||
docx_styles.parse_simple_md_to_paragraphs(doc, summary_text.strip())
|
||||
|
||||
# Step 6: Add footer: "X of Y" centered
|
||||
section = doc.sections[0]
|
||||
footer = section.footer
|
||||
footer.is_linked_to_previous = False
|
||||
@@ -563,8 +605,10 @@ def create_summary_docx(text: str, filename: str):
|
||||
Create a summary DOCX with:
|
||||
- 1" margins on all sides
|
||||
- 12pt Courier font
|
||||
- No line numbering
|
||||
- Markdown-aware WYSIWYG styling (headings, bullets, bold/italic)
|
||||
"""
|
||||
from . import docx_styles
|
||||
|
||||
doc = Document()
|
||||
|
||||
# Base font
|
||||
@@ -584,13 +628,20 @@ def create_summary_docx(text: str, filename: str):
|
||||
for p in list(body.findall(f"{{{W_NS}}}p")):
|
||||
body.remove(p)
|
||||
|
||||
# Add summary content
|
||||
lines = text.strip().splitlines()
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
p = doc.add_paragraph(line)
|
||||
p.paragraph_format.space_after = Pt(4)
|
||||
# Optionally add cover page
|
||||
from . import docx_cover
|
||||
from datetime import datetime
|
||||
cover_enabled = os.getenv("COVER_PAGE_ENABLED", "false").strip().lower() in ("true", "1", "yes")
|
||||
if cover_enabled:
|
||||
date_str = datetime.utcnow().strftime("%B %d, %Y")
|
||||
docx_cover.add_cover_page(
|
||||
doc,
|
||||
title="SUMMARY",
|
||||
date_str=date_str,
|
||||
)
|
||||
|
||||
# Add summary content using markdown-aware styling
|
||||
if text.strip():
|
||||
docx_styles.parse_simple_md_to_paragraphs(doc, text.strip())
|
||||
|
||||
doc.save(filename)
|
||||
|
||||
+81
-26
@@ -148,19 +148,87 @@ class SummarizerClient:
|
||||
start = break_pos
|
||||
return chunks
|
||||
|
||||
def _load_summary_prompt(self, role: str) -> str:
|
||||
"""
|
||||
Load summary prompt for the given role: 'chunk' or 'combined'.
|
||||
|
||||
Priority:
|
||||
1) SUMMARY_PROMPT_{ROLE} (env)
|
||||
2) SUMMARY_PROMPT_FILE (env) with [chunk] / [combined] sections
|
||||
3) Built-in default prompt
|
||||
"""
|
||||
role_upper = role.upper()
|
||||
|
||||
# 1) Direct env var: SUMMARY_PROMPT_CHUNK / SUMMARY_PROMPT_COMBINED
|
||||
env_key = f"SUMMARY_PROMPT_{role_upper}"
|
||||
env_prompt = (os.getenv(env_key) or "").strip()
|
||||
if env_prompt:
|
||||
return env_prompt
|
||||
|
||||
# 2) File-based prompt with sections
|
||||
prompt_file = (os.getenv("SUMMARY_PROMPT_FILE") or "").strip()
|
||||
if prompt_file and os.path.exists(prompt_file):
|
||||
try:
|
||||
with open(prompt_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
# Simple section parser: [chunk], [combined]
|
||||
import re
|
||||
pattern = re.compile(
|
||||
r"\[" + role + r"\]\s*\n(.*?)(?=\n\[|$)",
|
||||
re.DOTALL,
|
||||
)
|
||||
m = pattern.search(content)
|
||||
if m:
|
||||
text = m.group(1).strip()
|
||||
if text:
|
||||
return text
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load SUMMARY_PROMPT_FILE for %s: %s", role, e)
|
||||
|
||||
# 3) Default prompts
|
||||
if role == "chunk":
|
||||
return (
|
||||
"You are an expert legal and business meeting summarizer. "
|
||||
"You will receive a segment of a longer transcript. "
|
||||
"Provide a detailed, structured summary of this segment, focusing on: "
|
||||
"- Topics discussed\n"
|
||||
"- Key points and arguments\n"
|
||||
"- Decisions and agreements\n"
|
||||
"- Action items and responsibilities\n"
|
||||
"- Any risks, conflicts, or open issues\n"
|
||||
"- Names, roles, or affiliations of speakers (when mentioned)\n\n"
|
||||
"Be concise but complete. Use bullet points where helpful. "
|
||||
"Do not add information that is not present in the transcript."
|
||||
)
|
||||
else:
|
||||
return (
|
||||
"You are an expert legal and business meeting summarizer. "
|
||||
"You will receive several intermediate summaries of a longer conversation. "
|
||||
"Produce a single, comprehensive summary using markdown. "
|
||||
"Structure your response with the following sections:\n\n"
|
||||
"1. PERSONS IN DISCUSSION\n"
|
||||
"- List all participants whose names or roles can be identified from the conversation.\n"
|
||||
"- If a name is not given, use their role/position (e.g., Judge, Client, Manager, Witness) or 'Speaker 1', etc.\n"
|
||||
"- Keep it concise.\n\n"
|
||||
"2. PURPOSE AND CONTEXT\n"
|
||||
"- Briefly state the overall purpose and context of the discussion.\n\n"
|
||||
"3. MAIN ISSUES AND TOPICS\n"
|
||||
"- Summarize the main issues and topics addressed.\n\n"
|
||||
"4. KEY ARGUMENTS AND POSITIONS\n"
|
||||
"- Briefly outline key arguments and positions taken by each party.\n\n"
|
||||
"5. DECISIONS AND OUTCOMES\n"
|
||||
"- Clearly list decisions made and outcomes reached.\n\n"
|
||||
"6. ACTION ITEMS AND NEXT STEPS\n"
|
||||
"- List all action items, responsibilities, and next steps.\n\n"
|
||||
"7. UNRESOLVED ISSUES AND RISKS\n"
|
||||
"- Note any unresolved issues, conflicts, or risks.\n\n"
|
||||
"The summary should be detailed enough that a reader who was not present "
|
||||
"can understand what happened and what is expected going forward. "
|
||||
"Use clear, concise language and bullet points where appropriate."
|
||||
)
|
||||
|
||||
def _summarize_chunk(self, chunk: str, index: int, total: int) -> str:
|
||||
system_prompt = (
|
||||
"You are an expert legal and business meeting summarizer. "
|
||||
"You will receive a segment of a longer transcript. "
|
||||
"Provide a detailed, structured summary of this segment, focusing on: "
|
||||
"- Topics discussed\n"
|
||||
"- Key points and arguments\n"
|
||||
"- Decisions and agreements\n"
|
||||
"- Action items and responsibilities\n"
|
||||
"- Any risks, conflicts, or open issues\n\n"
|
||||
"Be concise but complete. Use bullet points when helpful. "
|
||||
"Do not add information that is not present in the transcript."
|
||||
)
|
||||
system_prompt = self._load_summary_prompt("chunk")
|
||||
|
||||
user_prompt = (
|
||||
f"This is segment {index + 1} of {total} from a longer conversation.\n\n"
|
||||
@@ -170,20 +238,7 @@ class SummarizerClient:
|
||||
return self._chat_completion(system_prompt, user_prompt)
|
||||
|
||||
def _summarize_combined(self, combined_summaries: str) -> str:
|
||||
system_prompt = (
|
||||
"You are an expert legal and business meeting summarizer. "
|
||||
"You will receive several intermediate summaries of a longer conversation. "
|
||||
"Produce a single, comprehensive summary that makes it clear: "
|
||||
"- The overall purpose and context of the discussion\n"
|
||||
"- The main issues and topics addressed\n"
|
||||
"- Key arguments and positions (briefly)\n"
|
||||
"- Decisions and outcomes\n"
|
||||
"- Action items, responsibilities, and next steps\n"
|
||||
"- Any unresolved issues or risks\n\n"
|
||||
"The summary should be detailed enough that a reader who was not present "
|
||||
"can understand what happened and what is expected going forward. "
|
||||
"Use clear, concise language and bullet points where appropriate."
|
||||
)
|
||||
system_prompt = self._load_summary_prompt("combined")
|
||||
|
||||
user_prompt = (
|
||||
"Here are the intermediate summaries from different parts of the same conversation:\n\n"
|
||||
|
||||
+293
-142
@@ -6,6 +6,7 @@ import os
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from .celery_app import celery_app
|
||||
@@ -247,6 +248,112 @@ def send_error_email(to: str, error_message: str, task_id: str):
|
||||
logger.error("Failed to send error email to %s for job %s: %s", to, task_id, e)
|
||||
|
||||
|
||||
def _accent_color() -> str:
|
||||
"""
|
||||
Return the accent color used in emails/templates.
|
||||
"""
|
||||
return (os.getenv("EMAIL_ACCENT_COLOR") or "#7C6DA0").strip()
|
||||
|
||||
|
||||
def _identify_speakers_via_llm(scraibe, transcript_text: str) -> dict:
|
||||
"""
|
||||
Use the summarizer LLM to identify speakers in the transcript.
|
||||
|
||||
Returns: dict mapping normalized speaker labels to name/role, e.g.
|
||||
{"SPEAKER 1": "JUDGE MARTINEZ", "SPEAKER 2": "DEFENSE COUNSEL"}
|
||||
"""
|
||||
try:
|
||||
scraibe._ensure_summarizer()
|
||||
summarizer = scraibe._summarizer
|
||||
except Exception as e:
|
||||
logger.warning("Failed to initialize summarizer for speaker identification: %s", e)
|
||||
return {}
|
||||
|
||||
prompt = (
|
||||
"Below is a transcript with speaker labels like 'SPEAKER 1', 'SPEAKER 2', etc. "
|
||||
"Based on the context and how each speaker talks, identify each speaker as:\n"
|
||||
"- Their real name, if it is clearly mentioned or strongly implied, OR\n"
|
||||
"- A concise role/position (e.g., Judge, Doctor, Manager, Interviewer, Client, Witness), "
|
||||
"if their identity is not clear.\n"
|
||||
"Do not invent random personal names. "
|
||||
"Do not add extra commentary. Output ONLY a mapping in this exact format, one per line:\n"
|
||||
"SPEAKER 1: Name or Role\n"
|
||||
"SPEAKER 2: Name or Role\n"
|
||||
"SPEAKER 3: Name or Role\n"
|
||||
"\n"
|
||||
"Transcript:\n"
|
||||
+ transcript_text
|
||||
)
|
||||
|
||||
try:
|
||||
reply = summarizer._chat_completion(
|
||||
system_prompt="You are an expert legal and business meeting analyst.",
|
||||
user_prompt=prompt,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("LLM call failed during speaker identification: %s", e)
|
||||
return {}
|
||||
|
||||
speaker_map = {}
|
||||
for m in re.finditer(r"SPEAKER\s+(\d+)\s*:\s*(.+)", reply, re.IGNORECASE):
|
||||
spk = f"SPEAKER {m.group(1).strip()}"
|
||||
name = m.group(2).strip().rstrip(".").upper()
|
||||
if name:
|
||||
speaker_map[spk] = name
|
||||
|
||||
logger.info("Speaker identification mapping: %s", speaker_map)
|
||||
return speaker_map
|
||||
|
||||
|
||||
def _apply_speaker_map(transcript_text: str, segments: list, speaker_map: dict) -> tuple:
|
||||
"""
|
||||
Apply speaker_map to:
|
||||
- transcript_text lines like "[00:12] SPEAKER 1: ..."
|
||||
- segment speaker fields.
|
||||
|
||||
Returns (updated_transcript_text, updated_segments).
|
||||
"""
|
||||
if not speaker_map:
|
||||
return transcript_text, segments
|
||||
|
||||
# Normalize function: e.g. "SPEAKER 1" -> "SPEAKER 1"
|
||||
def normalize_label(label: str) -> str:
|
||||
return re.sub(r"\s+", " ", re.sub(r"[^A-Z0-9\s]", "", label.upper())).strip()
|
||||
|
||||
# Replace in transcript lines
|
||||
def replace_in_line(line: str) -> str:
|
||||
# Match: [MM:SS] or [HH:MM:SS] then speaker label then colon
|
||||
# Use a lazy match for the label up to the first colon
|
||||
def repl(m):
|
||||
prefix = m.group(1) # e.g. "[00:12] "
|
||||
label = m.group(2).strip()
|
||||
normalized = normalize_label(label)
|
||||
new_label = speaker_map.get(normalized, label)
|
||||
return f"{prefix}{new_label}: "
|
||||
return re.sub(
|
||||
r"(\[\d+:\d+(?::\d+)?\]\s*)(.+?):\s*",
|
||||
repl,
|
||||
line,
|
||||
count=1,
|
||||
)
|
||||
|
||||
updated_transcript = "\n".join(
|
||||
replace_in_line(line) for line in transcript_text.splitlines()
|
||||
)
|
||||
|
||||
# Update segments
|
||||
updated_segments = []
|
||||
for seg in segments:
|
||||
sp = (seg.get("speaker") or "").strip()
|
||||
sp_norm = normalize_label(sp)
|
||||
sp_new = speaker_map.get(sp_norm, sp)
|
||||
seg = dict(seg)
|
||||
seg["speaker"] = sp_new
|
||||
updated_segments.append(seg)
|
||||
|
||||
return updated_transcript, updated_segments
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="scraibe.tasks.process_transcription_task",
|
||||
bind=True,
|
||||
@@ -266,7 +373,7 @@ def process_transcription_task(
|
||||
identify_speakers: bool = False,
|
||||
):
|
||||
"""
|
||||
Async task: transcribe audio, optionally summarize, then email results.
|
||||
Async task: transcribe audio, optionally identify speakers, optionally summarize, then email results.
|
||||
Cleans up temporary files after completion.
|
||||
"""
|
||||
task_id = self.request.id
|
||||
@@ -294,121 +401,42 @@ def process_transcription_task(
|
||||
)
|
||||
raise
|
||||
|
||||
# 3) Transcription
|
||||
if task_type == "transcript_and_summarize":
|
||||
result = scraibe.transcript_and_summarize(
|
||||
audio_file=audio_path,
|
||||
language=language or None,
|
||||
num_speakers=int(num_speakers) if num_speakers else None,
|
||||
verbose=True,
|
||||
for_export=True,
|
||||
)
|
||||
transcript_text = result.get("transcript", "")
|
||||
summary_text = result.get("summary", "")
|
||||
segments = result.get("segments", [])
|
||||
raw_result = result.get("raw_result")
|
||||
else:
|
||||
result = scraibe.transcribe(
|
||||
audio_file=audio_path,
|
||||
language=language or None,
|
||||
num_speakers=int(num_speakers) if num_speakers else None,
|
||||
verbose=True,
|
||||
for_export=True,
|
||||
)
|
||||
transcript_text = result.get("transcript", "")
|
||||
summary_text = ""
|
||||
segments = result.get("segments", [])
|
||||
raw_result = result.get("raw_result")
|
||||
# 3) Transcription (always first, without summarization)
|
||||
result = scraibe.transcribe(
|
||||
audio_file=audio_path,
|
||||
language=language or None,
|
||||
num_speakers=int(num_speakers) if num_speakers else None,
|
||||
verbose=True,
|
||||
for_export=True,
|
||||
)
|
||||
transcript_text = result.get("transcript", "")
|
||||
segments = result.get("segments", [])
|
||||
raw_result = result.get("raw_result")
|
||||
summary_text = ""
|
||||
|
||||
# 3b) Optional speaker identification
|
||||
speaker_map = {} # e.g. {"SPEAKER 1": "John", "SPEAKER 2": "Maria"}
|
||||
# 4) Optional speaker identification (before summarization)
|
||||
if identify_speakers:
|
||||
try:
|
||||
# Use the same summarizer client as transcript_and_summarize
|
||||
scraibe._ensure_summarizer()
|
||||
summarizer = scraibe._summarizer
|
||||
|
||||
prompt = (
|
||||
"Below is a transcript with speaker labels like 'SPEAKER 1', 'SPEAKER 2', etc. "
|
||||
"Based on the context and how each speaker talks, identify each speaker as:\n"
|
||||
"- Their real name, if it is clearly mentioned or strongly implied, OR\n"
|
||||
"- A concise role/position (e.g., Judge, Doctor, Manager, Interviewer, Client, Witness), "
|
||||
"if their identity is not clear.\n"
|
||||
"Do not invent random personal names. "
|
||||
"Do not add extra commentary. Output ONLY a mapping in this exact format, one per line:\n"
|
||||
"SPEAKER 1: Name or Role\n"
|
||||
"SPEAKER 2: Name or Role\n"
|
||||
"SPEAKER 3: Name or Role\n"
|
||||
"\n"
|
||||
"Transcript:\n"
|
||||
+ transcript_text
|
||||
)
|
||||
|
||||
response = summarizer._chat_completion(
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.3,
|
||||
max_tokens=300,
|
||||
)
|
||||
reply = (response or {}).get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
|
||||
# Parse mapping
|
||||
import re
|
||||
for m in re.finditer(
|
||||
r"SPEAKER\s+(\d+)\s*:\s*(.+)",
|
||||
reply,
|
||||
re.IGNORECASE,
|
||||
):
|
||||
spk = f"SPEAKER {m.group(1).strip()}"
|
||||
name = m.group(2).strip().rstrip(".").upper()
|
||||
if name:
|
||||
speaker_map[spk] = name
|
||||
|
||||
logger.info("Speaker identification mapping: %s", speaker_map)
|
||||
|
||||
# Apply mapping to transcript text
|
||||
speaker_map = _identify_speakers_via_llm(scraibe, transcript_text)
|
||||
if speaker_map:
|
||||
def replace_speaker(m):
|
||||
label = m.group(0).strip()
|
||||
# normalize to "SPEAKER N"
|
||||
normalized = re.sub(
|
||||
r"\s+",
|
||||
" ",
|
||||
re.sub(r"[^A-Z0-9\s]", "", label.upper()),
|
||||
).strip()
|
||||
return speaker_map.get(normalized, label)
|
||||
|
||||
# Replace in lines like "[00:12] SPEAKER 1:" but preserve timestamp and colon
|
||||
def replace_in_line(line: str) -> str:
|
||||
# match after timestamp bracket and space: "SPEAKER N:"
|
||||
return re.sub(
|
||||
r"(\[\d+:\d+(?::\d+)?\]\s*)([A-Z\s]+?):\s*",
|
||||
lambda m: m.group(1) + (speaker_map.get(m.group(2).strip(), m.group(2)) + ": "),
|
||||
line,
|
||||
)
|
||||
|
||||
transcript_lines = transcript_text.splitlines()
|
||||
transcript_text = "\n".join(
|
||||
replace_in_line(line) for line in transcript_lines
|
||||
transcript_text, segments = _apply_speaker_map(
|
||||
transcript_text, segments, speaker_map
|
||||
)
|
||||
|
||||
# Also update segments for JSON export
|
||||
updated_segments = []
|
||||
for seg in segments:
|
||||
sp = (seg.get("speaker") or "").strip()
|
||||
sp_norm = re.sub(r"[^A-Z0-9\s]", "", sp.upper()).strip()
|
||||
sp_new = speaker_map.get(sp_norm, sp)
|
||||
seg = dict(seg)
|
||||
seg["speaker"] = sp_new
|
||||
updated_segments.append(seg)
|
||||
segments = updated_segments
|
||||
|
||||
except (SummarizerError, Exception) as e:
|
||||
logger.info("Applied speaker identification to transcript and segments.")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Speaker identification failed; falling back to Speaker IDs: %s", e
|
||||
"Speaker identification failed; continuing with original labels: %s", e
|
||||
)
|
||||
speaker_map = {}
|
||||
|
||||
# 4) Prepare files
|
||||
# 5) Summarization (if requested) using the (possibly identified) transcript
|
||||
if include_summary:
|
||||
try:
|
||||
summary_text = scraibe._summarizer.summarize_transcript(transcript_text)
|
||||
except Exception as e:
|
||||
logger.warning("Summarization failed; continuing without summary: %s", e)
|
||||
summary_text = ""
|
||||
|
||||
# 6) Prepare files
|
||||
|
||||
# Transcript .md
|
||||
md_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".md")
|
||||
@@ -417,12 +445,19 @@ def process_transcription_task(
|
||||
f.write(transcript_text)
|
||||
temp_files.append(md_transcript_path)
|
||||
|
||||
# Transcript .docx (standalone, no cover page)
|
||||
# Transcript .docx (with summary appended if present)
|
||||
docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx")
|
||||
create_transcript_docx(
|
||||
transcript_text,
|
||||
docx_transcript_path,
|
||||
)
|
||||
if summary_text:
|
||||
create_transcript_docx(
|
||||
transcript_text,
|
||||
docx_transcript_path,
|
||||
summary_text=summary_text,
|
||||
)
|
||||
else:
|
||||
create_transcript_docx(
|
||||
transcript_text,
|
||||
docx_transcript_path,
|
||||
)
|
||||
temp_files.append(docx_transcript_path)
|
||||
|
||||
# JSON as SOURCE
|
||||
@@ -445,40 +480,14 @@ def process_transcription_task(
|
||||
json.dump(json_data, f, indent=2, ensure_ascii=False)
|
||||
temp_files.append(json_path)
|
||||
|
||||
# Summary files (if present)
|
||||
md_summary_path = None
|
||||
docx_summary_path = None
|
||||
|
||||
if summary_text:
|
||||
# Summary .md
|
||||
md_summary_path = _safe_filename("SUMMARY", local, date_tag, ".md")
|
||||
with open(md_summary_path, "w", encoding="utf-8") as f:
|
||||
f.write("# Summary\n\n")
|
||||
f.write(summary_text)
|
||||
temp_files.append(md_summary_path)
|
||||
|
||||
# Summary .docx (standalone, no cover page)
|
||||
docx_summary_path = _safe_filename("SUMMARY", local, date_tag, ".docx")
|
||||
create_summary_docx(
|
||||
summary_text,
|
||||
docx_summary_path,
|
||||
)
|
||||
temp_files.append(docx_summary_path)
|
||||
|
||||
# 5) Build attachments list
|
||||
|
||||
# Always: JSON, transcript MD, transcript DOCX
|
||||
# 7) Build attachments list
|
||||
attachments = [
|
||||
md_transcript_path,
|
||||
docx_transcript_path,
|
||||
json_path,
|
||||
]
|
||||
|
||||
# If summary is present, add summary MD and DOCX
|
||||
if summary_text:
|
||||
attachments += [md_summary_path, docx_summary_path]
|
||||
|
||||
# 6) Send success email
|
||||
# 8) Send success email
|
||||
send_success_email(
|
||||
to=email_to,
|
||||
transcript_text=transcript_text,
|
||||
@@ -498,9 +507,151 @@ def process_transcription_task(
|
||||
)
|
||||
raise e
|
||||
finally:
|
||||
# 7) Cleanup
|
||||
# 9) Cleanup
|
||||
for path in temp_files:
|
||||
_remove_file(path)
|
||||
if audio_path:
|
||||
_remove_file(audio_path)
|
||||
logger.info("Cleanup completed for job %s.", task_id)
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="scraibe.tasks.process_watch_file_task",
|
||||
bind=True,
|
||||
max_retries=1,
|
||||
task_time_limit=14400,
|
||||
task_soft_time_limit=13500,
|
||||
)
|
||||
def process_watch_file_task(
|
||||
self,
|
||||
file_path: str,
|
||||
):
|
||||
"""
|
||||
Async task for watch-folder mode:
|
||||
- Transcribe + summarize
|
||||
- Email results
|
||||
- Optionally delete source file
|
||||
"""
|
||||
task_id = self.request.id
|
||||
|
||||
log_level = os.getenv("LOG_LEVEL", "INFO")
|
||||
setup_logging(level=log_level)
|
||||
|
||||
email_to = os.getenv("WATCH_EMAIL_TO") or os.getenv("EMAIL_DEFAULT_TO")
|
||||
if not email_to:
|
||||
logger.error("No email address configured for watch-folder mode.")
|
||||
raise RuntimeError("WATCH_EMAIL_TO or EMAIL_DEFAULT_TO not set.")
|
||||
|
||||
delete_on_success = os.getenv("WATCH_DELETE_ON_SUCCESS", "true").strip().lower() in ("true", "1", "yes")
|
||||
|
||||
temp_files = []
|
||||
local = "watch"
|
||||
date_tag = _date_tag()
|
||||
|
||||
try:
|
||||
scraibe = Scraibe(verbose=True)
|
||||
|
||||
result = scraibe.transcript_and_summarize(
|
||||
audio_file=file_path,
|
||||
language=None,
|
||||
num_speakers=None,
|
||||
verbose=True,
|
||||
for_export=True,
|
||||
)
|
||||
|
||||
transcript_text = result.get("transcript", "")
|
||||
summary_text = result.get("summary", "")
|
||||
segments = result.get("segments", [])
|
||||
raw_result = result.get("raw_result")
|
||||
|
||||
# Transcript .md
|
||||
md_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".md")
|
||||
with open(md_transcript_path, "w", encoding="utf-8") as f:
|
||||
f.write("# Transcript\n\n")
|
||||
f.write(transcript_text)
|
||||
temp_files.append(md_transcript_path)
|
||||
|
||||
# Transcript .docx
|
||||
docx_transcript_path = _safe_filename("TRANSCRIPT", local, date_tag, ".docx")
|
||||
create_transcript_docx(
|
||||
transcript_text,
|
||||
docx_transcript_path,
|
||||
)
|
||||
temp_files.append(docx_transcript_path)
|
||||
|
||||
# Summary .md
|
||||
md_summary_path = _safe_filename("SUMMARY", local, date_tag, ".md")
|
||||
with open(md_summary_path, "w", encoding="utf-8") as f:
|
||||
f.write("# Summary\n\n")
|
||||
f.write(summary_text)
|
||||
temp_files.append(md_summary_path)
|
||||
|
||||
# Summary .docx
|
||||
docx_summary_path = _safe_filename("SUMMARY", local, date_tag, ".docx")
|
||||
create_summary_docx(
|
||||
summary_text,
|
||||
docx_summary_path,
|
||||
)
|
||||
temp_files.append(docx_summary_path)
|
||||
|
||||
# JSON as SOURCE
|
||||
json_data = {
|
||||
"task": "watch_transcript_and_summarize",
|
||||
"transcript": transcript_text,
|
||||
"summary": summary_text,
|
||||
"segments": segments,
|
||||
"metadata": {
|
||||
"timestamp": datetime.utcnow().isoformat(),
|
||||
"job_id": task_id,
|
||||
"source_file": file_path,
|
||||
},
|
||||
}
|
||||
if raw_result is not None:
|
||||
json_data["raw_result"] = raw_result
|
||||
|
||||
json_path = _safe_filename("SOURCE", local, date_tag, ".json")
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(json_data, f, indent=2, ensure_ascii=False)
|
||||
temp_files.append(json_path)
|
||||
|
||||
# Attachments
|
||||
attachments = [
|
||||
md_transcript_path,
|
||||
docx_transcript_path,
|
||||
md_summary_path,
|
||||
docx_summary_path,
|
||||
json_path,
|
||||
]
|
||||
|
||||
# Send email
|
||||
send_success_email(
|
||||
to=email_to,
|
||||
transcript_text=transcript_text,
|
||||
summary_text=summary_text,
|
||||
attachments=attachments,
|
||||
task_id=task_id,
|
||||
)
|
||||
|
||||
logger.info("Watch-folder job %s completed for %s.", task_id, file_path)
|
||||
|
||||
# Delete source file if configured
|
||||
if delete_on_success and os.path.exists(file_path):
|
||||
try:
|
||||
os.remove(file_path)
|
||||
logger.info("Deleted source file: %s", file_path)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to delete source file %s: %s", file_path, e)
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Error processing watch file %s: %s", file_path, e, exc_info=True)
|
||||
send_error_email(
|
||||
to=email_to,
|
||||
error_message=str(e),
|
||||
task_id=task_id,
|
||||
)
|
||||
raise e
|
||||
finally:
|
||||
# Cleanup temp files
|
||||
for path in temp_files:
|
||||
_remove_file(path)
|
||||
logger.info("Watch-folder job %s cleanup completed.", task_id)
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
Watch-folder mode for ScrAIbe.
|
||||
|
||||
Monitors a folder for audio files. For each file:
|
||||
- Transcribes + summarizes
|
||||
- Emails results
|
||||
- Deletes source file
|
||||
|
||||
Configuration (env):
|
||||
- WATCH_ENABLED: "true"/"false" (default: false)
|
||||
- WATCH_DIR: directory to watch (required if enabled)
|
||||
- WATCH_EMAIL_TO: destination email (required if enabled)
|
||||
- WATCH_POLL_INTERVAL: seconds between scans (default: 10)
|
||||
- WATCH_DELETE_ON_SUCCESS: "true"/"false" (default: true)
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("scraibe.watcher")
|
||||
|
||||
AUDIO_EXTENSIONS = {
|
||||
".wav",
|
||||
".mp3",
|
||||
".flac",
|
||||
".m4a",
|
||||
".ogg",
|
||||
".webm",
|
||||
".mp4",
|
||||
}
|
||||
|
||||
|
||||
def _is_audio(path: Path) -> bool:
|
||||
return path.is_file() and path.suffix.lower() in AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
def _enqueue_file(file_path: Path):
|
||||
"""
|
||||
Enqueue a file for transcription + summarization via Celery.
|
||||
"""
|
||||
from .tasks import process_watch_file_task
|
||||
|
||||
try:
|
||||
process_watch_file_task.delay(str(file_path))
|
||||
except Exception as e:
|
||||
logger.error("Failed to enqueue watch file %s: %s", file_path, e)
|
||||
|
||||
|
||||
def _scan_directory(watch_dir: Path):
|
||||
"""
|
||||
Scan directory and enqueue all audio files.
|
||||
"""
|
||||
if not watch_dir.is_dir():
|
||||
logger.warning("WATCH_DIR does not exist or is not a directory: %s", watch_dir)
|
||||
return
|
||||
|
||||
for p in watch_dir.iterdir():
|
||||
if _is_audio(p):
|
||||
logger.info("Found audio file in WATCH_DIR: %s", p)
|
||||
_enqueue_file(p)
|
||||
|
||||
|
||||
def start_watcher():
|
||||
"""
|
||||
Start watch-folder loop in a background thread.
|
||||
"""
|
||||
enabled = os.getenv("WATCH_ENABLED", "false").strip().lower() in ("true", "1", "yes")
|
||||
if not enabled:
|
||||
return
|
||||
|
||||
watch_dir = os.getenv("WATCH_DIR")
|
||||
if not watch_dir:
|
||||
logger.warning("WATCH_ENABLED is true but WATCH_DIR is not set. Watcher disabled.")
|
||||
return
|
||||
|
||||
email_to = os.getenv("WATCH_EMAIL_TO")
|
||||
if not email_to:
|
||||
logger.warning("WATCH_ENABLED is true but WATCH_EMAIL_TO is not set. Watcher disabled.")
|
||||
return
|
||||
|
||||
interval = float(os.getenv("WATCH_POLL_INTERVAL", "10"))
|
||||
|
||||
watch_path = Path(watch_dir).expanduser().resolve()
|
||||
watch_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info("Starting watch-folder: dir=%s, email=%s, interval=%s", watch_dir, email_to, interval)
|
||||
|
||||
def _loop():
|
||||
while True:
|
||||
try:
|
||||
_scan_directory(watch_path)
|
||||
except Exception as e:
|
||||
logger.error("Error scanning WATCH_DIR: %s", e)
|
||||
time.sleep(interval)
|
||||
|
||||
t = threading.Thread(target=_loop, daemon=True)
|
||||
t.start()
|
||||
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
Diagnostic script for ScrAIbe email (SMTP) connectivity.
|
||||
|
||||
Run inside the container to verify:
|
||||
- Required env vars are set
|
||||
- SMTP connection
|
||||
- TLS (if configured)
|
||||
- Authentication
|
||||
|
||||
Output is written to stdout/stderr so it appears in docker logs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import smtplib
|
||||
import logging
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="DIAG [%(levelname)s] %(message)s",
|
||||
stream=sys.stdout,
|
||||
force=True,
|
||||
)
|
||||
log = logging.getLogger("email_diagnostics")
|
||||
|
||||
|
||||
def main():
|
||||
# 1) Check required env vars
|
||||
required = [
|
||||
"EMAIL_SMTP_HOST",
|
||||
"EMAIL_SMTP_PORT",
|
||||
"EMAIL_SMTP_USER",
|
||||
"EMAIL_SMTP_PASSWORD",
|
||||
"EMAIL_FROM_ADDRESS",
|
||||
]
|
||||
missing = [v for v in required if not os.getenv(v)]
|
||||
if missing:
|
||||
log.error("Missing required env vars: %s", ", ".join(missing))
|
||||
sys.exit(1)
|
||||
|
||||
smtp_host = os.getenv("EMAIL_SMTP_HOST")
|
||||
smtp_port = int(os.getenv("EMAIL_SMTP_PORT"))
|
||||
smtp_user = os.getenv("EMAIL_SMTP_USER")
|
||||
smtp_password = os.getenv("EMAIL_SMTP_PASSWORD")
|
||||
from_address = os.getenv("EMAIL_FROM_ADDRESS")
|
||||
use_tls_str = (os.getenv("EMAIL_SMTP_USE_TLS") or "true").strip().lower()
|
||||
use_tls = use_tls_str not in ("false", "0", "no")
|
||||
|
||||
log.info("SMTP config:")
|
||||
log.info(" host: %s", smtp_host)
|
||||
log.info(" port: %s", smtp_port)
|
||||
log.info(" user: %s", smtp_user)
|
||||
log.info(" from: %s", from_address)
|
||||
log.info(" use_tls: %s", use_tls)
|
||||
log.info(" password: %s", ("set" if smtp_password else "NOT SET"))
|
||||
|
||||
# 2) Connect to SMTP server
|
||||
log.info("Attempting SMTP connection...")
|
||||
try:
|
||||
if use_tls:
|
||||
server = smtplib.SMTP(smtp_host, smtp_port, timeout=15)
|
||||
log.info("Connected (SMTP) to %s:%s", smtp_host, smtp_port)
|
||||
else:
|
||||
server = smtplib.SMTP(smtp_host, smtp_port, timeout=15)
|
||||
log.info("Connected (SMTP, no TLS) to %s:%s", smtp_host, smtp_port)
|
||||
except Exception as e:
|
||||
log.error("SMTP connection failed: %s", e)
|
||||
sys.exit(1)
|
||||
|
||||
# 3) EHLO
|
||||
log.info("Sending EHLO...")
|
||||
try:
|
||||
ehlo_resp = server.ehlo()
|
||||
log.info("EHLO response code: %s", ehlo_resp[0])
|
||||
except Exception as e:
|
||||
log.error("EHLO failed: %s", e)
|
||||
server.quit()
|
||||
sys.exit(1)
|
||||
|
||||
# 4) STARTTLS if configured
|
||||
if use_tls:
|
||||
log.info("Attempting STARTTLS...")
|
||||
try:
|
||||
server.starttls()
|
||||
server.ehlo()
|
||||
log.info("STARTTLS succeeded.")
|
||||
except Exception as e:
|
||||
log.error("STARTTLS failed: %s", e)
|
||||
server.quit()
|
||||
sys.exit(1)
|
||||
else:
|
||||
log.info("TLS not requested; continuing without STARTTLS.")
|
||||
|
||||
# 5) AUTH LOGIN
|
||||
log.info("Attempting AUTH LOGIN with user: %s", smtp_user)
|
||||
try:
|
||||
server.login(smtp_user, smtp_password)
|
||||
log.info("AUTH LOGIN succeeded.")
|
||||
except smtplib.SMTPAuthenticationError as e:
|
||||
log.error("AUTH LOGIN failed (bad credentials?): %s", e)
|
||||
server.quit()
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
log.error("AUTH LOGIN failed: %s", e)
|
||||
server.quit()
|
||||
sys.exit(1)
|
||||
|
||||
# 6) Optional: quick MAIL FROM / RCPT TO / QUIT test (no message sent)
|
||||
log.info("Testing MAIL FROM / RCPT TO / RSET...")
|
||||
try:
|
||||
server.mail(from_address)
|
||||
# Use from_address as recipient for test
|
||||
server.rcpt(from_address)
|
||||
server.reset()
|
||||
log.info("MAIL FROM / RCPT TO / RSET succeeded.")
|
||||
except Exception as e:
|
||||
log.warning("MAIL FROM / RCPT TO / RSET failed (non-critical): %s", e)
|
||||
|
||||
# 7) Quit
|
||||
server.quit()
|
||||
log.info("All email diagnostics passed. SMTP is reachable and authenticated.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user