Switch transcription backend from Whisper to Mistral AI API

README.md

@@ -1,6 +1,6 @@
 # Audio Summary with Local LLM
 
-This tool is designed to provide a quick and concise summary of audio and video files. It supports summarizing content either from a local file or directly from YouTube. The tool uses Whisper for transcription and a local version of Llama3 (via Ollama) for generating summaries.
+This tool is designed to provide a quick and concise summary of audio and video files. It supports summarizing content either from a local file or directly from YouTube. The tool uses Mistral AI API for transcription and a local version of Llama3 (via Ollama) for generating summaries.
 
 > [!TIP]  
 > It is possible to change the model you wish to use.
@@ -10,7 +10,7 @@ This tool is designed to provide a quick and concise summary of audio and video
 
 - **YouTube Integration**: Download and summarize content directly from YouTube.
 - **Local File Support**: Summarize audio/video files available on your local disk.
-- **Transcription**: Converts audio content to text using Whisper.
+- **Transcription**: Converts audio content to text using Mistral AI API.
 - **Summarization**: Generates a concise summary using Llama3 (Ollama).
 - **Transcript Only Option**: Option to only transcribe the audio content without generating a summary.
 - **Device Optimization**: Automatically uses the best available hardware (MPS for Mac, CUDA for NVIDIA GPUs, or CPU).
@@ -23,6 +23,8 @@ Before you start using this tool, you need to install the following dependencies
 - [Ollama](https://ollama.com) for LLM model management
 - `ffmpeg` (required for audio processing)
 - [uv](https://docs.astral.sh/uv/getting-started/installation/) for package management
+- Mistral AI API key (free tier available)
+- Mistral AI API key (set as `MISTRAL_API_KEY` environment variable)
 
 ## Installation
 
@@ -39,6 +41,17 @@ uv sync
 source .venv/bin/activate  # On Windows: .venv\Scripts\activate
 ```
 
+### Mistral API Setup
+
+1. Sign up for a Mistral AI account and get your API key from the [Mistral AI platform](https://mistral.ai/)
+2. Set your API key as an environment variable:
+
+```bash
+export MISTRAL_API_KEY="your-api-key-here"
+```
+
+Or add it to your `.bashrc` or `.zshrc` file for persistent use.
+
 ### LLM Requirement
 
 [Download and install](https://ollama.com) Ollama to carry out LLM Management. More details about LLM models supported can be found on the Ollama [GitHub](https://github.com/ollama/ollama).
@@ -196,24 +209,21 @@ OLLAMA_MODEL = "llama3"
 WHISPER_MODEL = "openai/whisper-large-v2"
 ```
 
-#### Changing the Whisper Model
+#### Changing the Mistral Transcription Model
 
-To use a different Whisper model for transcription:
+To use a different Mistral model for transcription:
 
-1. Update the `WHISPER_MODEL` variable with one of these options:
+1. Update the `MISTRAL_MODEL` variable with one of the available Mistral audio models:
-   - `"openai/whisper-tiny"` (fastest, least accurate)
+   - `"voxtral-mini-latest"` (default, good balance)
-   - `"openai/whisper-base"` (faster, less accurate)
+   - `"voxtral-small-latest"` (smaller, faster)
-   - `"openai/whisper-small"` (balanced)
-   - `"openai/whisper-medium"` (slower, more accurate)
-   - `"openai/whisper-large-v2"` (slowest, most accurate)
 
 2. Example:
 
    ```python
-   WHISPER_MODEL = "openai/whisper-medium"  # A good balance between speed and accuracy
+   MISTRAL_MODEL = "voxtral-small-latest"  # Faster transcription
    ```
 
-For CPU-only systems, using a smaller model like `whisper-base` is recommended for better performance.
+The Mistral API handles all hardware acceleration automatically, so no device selection is needed.
 
 #### Changing the LLM Model
 

pyproject.toml

@@ -14,12 +14,11 @@ authors = [
 ]
 dependencies = [
     "ffmpeg>=1.4",
+    "mistralai>=0.1.0",
     "ollama>=0.4.7",
-    "openai-whisper>=20240930",
     "torch>=2.6.0",
     "torchaudio>=2.6.0",
     "torchvision>=0.21.0",
-    "transformers>=4.50.2",
     "yt-dlp>=2025.3.27",
 ]
 [project.scripts]

src/summary.py

@@ -1,88 +1,85 @@
-import ollama
 import argparse
+import os
 from pathlib import Path
-from transformers import pipeline
+
+import ollama
 import yt_dlp
-import torch
+from mistralai.client import Mistral
+from mistralai.client.models import File
 
 OLLAMA_MODEL = "llama3"
-WHISPER_MODEL = "openai/whisper-large-v2"
+MISTRAL_MODEL = "voxtral-mini-latest"  # Mistral audio transcription model
-WHISPER_LANGUAGE = "en"  # Set to desired language or None for auto-detection
+MISTRAL_LANGUAGE = "en"  # Set to desired language or None for auto-detection
+
 
 # Function to download a video from YouTube using yt-dlp
 def download_from_youtube(url: str, path: str):
     ydl_opts = {
-        'format': 'bestaudio/best',
+        "format": "bestaudio/best",
-        'outtmpl': str(Path(path) / 'to_transcribe.%(ext)s'),
+        "outtmpl": str(Path(path) / "to_transcribe.%(ext)s"),
-        'postprocessors': [{
+        "postprocessors": [
-            'key': 'FFmpegExtractAudio',
+            {
-            'preferredcodec': 'mp3',
+                "key": "FFmpegExtractAudio",
-            'preferredquality': '192',
+                "preferredcodec": "mp3",
-        }],
+                "preferredquality": "192",
+            }
+        ],
     }
     with yt_dlp.YoutubeDL(ydl_opts) as ydl:
         ydl.download([url])
 
-# Function to get the best available device
-def get_device():
-    if torch.backends.mps.is_available():
-        return "mps"
-    elif torch.cuda.is_available():
-        return "cuda"
-    else:
-        return "cpu"
 
-# Function to transcribe an audio file using the transformers pipeline
+# Function to get Mistral API key from environment variables
+def get_mistral_api_key():
+    api_key = os.getenv("MISTRAL_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "MISTRAL_API_KEY environment variable not set. Please set your Mistral API key."
+        )
+    return api_key
+
+
+# Function to transcribe an audio file using Mistral API
 def transcribe_file(file_path: str, output_file: str, language: str = None) -> str:
-    # Get the best available device
+    # Get Mistral API key
-    device = get_device()
+    api_key = get_mistral_api_key()
-    print(f"Using device: {device} for transcription")
 
-    # Load the pipeline model for automatic speech recognition
+    # Set up language parameter
-    transcriber = pipeline(
+    transcribe_language = language if language and language.lower() != "auto" else None
-        "automatic-speech-recognition", 
+    if transcribe_language:
-        model=WHISPER_MODEL, 
+        print(f"Transcribing in language: {transcribe_language}")
-        device=device,
-        chunk_length_s=30,  # Process in 30-second chunks
-        return_timestamps=True  # Enable timestamp generation for longer audio
-    )
-    
-    # Transcribe the audio file
-    # For CPU, we might want to use a smaller model or chunk the audio if memory is an issue
-    if device == "cpu":
-        print("Warning: Using CPU for transcription. This may be slow.")
-    
-    # Set up generation keyword arguments including language
-    generate_kwargs = {}
-    if language and language.lower() != "auto":
-        generate_kwargs["language"] = language
-        print(f"Transcribing in language: {language}")
     else:
         print("Using automatic language detection")
 
-    # Transcribe the audio file
+    # Initialize Mistral client
-    print("Starting transcription (this may take a while for longer files)...")
+    print("Starting transcription with Mistral API...")
-    transcribe = transcriber(file_path, generate_kwargs=generate_kwargs)
+    # Open the audio file and send to Mistral API
+    with open(file_path, "rb") as audio_file:
+        # Create File object as expected by Mistral API
+        file_obj = File(content=audio_file, file_name=Path(file_path).name)
 
-    # Extract the full text from the chunked transcription
+        # Use async transcription for better performance
-    if isinstance(transcribe, dict) and "text" in transcribe:
+        import asyncio
-        # Simple case - just one chunk
+
-        full_text = transcribe["text"]
+        async def transcribe_async():
-    elif isinstance(transcribe, dict) and "chunks" in transcribe:
+            with Mistral(api_key=api_key) as mistral:
-        # Multiple chunks with timestamps
+                response = await mistral.audio.transcriptions.complete_async(
-        full_text = " ".join([chunk["text"] for chunk in transcribe["chunks"]])
+                    model=MISTRAL_MODEL, file=file_obj, language=transcribe_language
-    else:
+                )
-        # Fallback for other return formats
+                return response.text
-        full_text = transcribe["text"] if "text" in transcribe else str(transcribe)
+
+        # Run the async transcription
+        full_text = asyncio.run(transcribe_async())
 
     # Save the transcribed text to the specified temporary file
-    with open(output_file, 'w') as tmp_file:
+    with open(output_file, "w") as tmp_file:
         tmp_file.write(full_text)
         print(f"Transcription saved to file: {output_file}")
 
     # Return the transcribed text
     return full_text
 
+
 # Function to summarize a text using the Ollama model
 def summarize_text(text: str, output_path: str) -> str:
     # Define the system prompt for the Ollama model
@@ -112,20 +109,33 @@ def summarize_text(text: str, output_path: str) -> str:
     # Print the generated summary
     return response["message"]["content"]
 
+
 def main():
     # Parse command line arguments
-    parser = argparse.ArgumentParser(description="Download, transcribe, and summarize audio or video files.")
+    parser = argparse.ArgumentParser(
+        description="Download, transcribe, and summarize audio or video files."
+    )
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--from-youtube", type=str, help="YouTube URL to download.")
     group.add_argument("--from-local", type=str, help="Path to the local audio file.")
-    parser.add_argument("--output", type=str, default="./summary.md", help="Output markdown file path.")
+    parser.add_argument(
-    parser.add_argument("--transcript-only", action='store_true', help="Only transcribe the file, do not summarize.")
+        "--output", type=str, default="./summary.md", help="Output markdown file path."
-    parser.add_argument("--language", type=str, help="Language code for transcription (e.g., 'en', 'fr', 'es', or 'auto' for detection)")
+    )
+    parser.add_argument(
+        "--transcript-only",
+        action="store_true",
+        help="Only transcribe the file, do not summarize.",
+    )
+    parser.add_argument(
+        "--language",
+        type=str,
+        help="Language code for transcription (e.g., 'en', 'fr', 'es', or 'auto' for detection)",
+    )
 
     args = parser.parse_args()
 
     # Determine language setting
-    language = args.language if args.language else WHISPER_LANGUAGE
+    language = args.language if args.language else MISTRAL_LANGUAGE
     if language and language.lower() == "auto":
         language = None  # None triggers automatic language detection
 
@@ -147,7 +157,9 @@ def main():
 
     print(f"Transcribing file: {file_path}")
     # Transcribe the audio file
-    transcript = transcribe_file(str(file_path), data_directory / "transcript.txt", language)
+    transcript = transcribe_file(
+        str(file_path), data_directory / "transcript.txt", language
+    )
 
     if args.transcript_only:
         print("Transcription complete. Skipping summary generation.")
@@ -163,5 +175,6 @@ def main():
         md_file.write(summary)
         print(f"Summary written to {args.output}")
 
+
 if __name__ == "__main__":
     main()