from openai import OpenAI
import base64

client = OpenAI()

def analyze_image(image_path: str, question: str) -> str:
    """Analyze an image with GPT-4o."""
    
    with open(image_path, "rb") as f:
        image_data = base64.standard_b64encode(f.read()).decode("utf-8")
    
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": question},
                {"type": "image_url", "image_url": {
                    "url": f"data:image/png;base64,{image_data}",
                    "detail": "high"  # "low", "high", or "auto"
                }}
            ]
        }],
        max_tokens=1000
    )
    return response.choices[0].message.content


def analyze_image_url(url: str, question: str) -> str:
    """Analyze an image from URL."""
    
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": question},
                {"type": "image_url", "image_url": {"url": url}}
            ]
        }]
    )
    return response.choices[0].message.content

Token Cost for Images (GPT-4o):

Detail Level    Tokens Used         Best For
─────────────────────────────────────────────────
low             85 tokens           Quick classification, simple questions
high            85-1105 tokens      OCR, detailed analysis, small text
auto            Model decides       General use

High detail: image split into 512x512 tiles
Each tile = 170 tokens + 85 base tokens
Max: 2048x2048 before downscaling

LLaVA Training Pipeline:

Stage 1: Feature Alignment (pretrain)
  - Freeze: Vision encoder + LLM
  - Train: Projection layer only
  - Data: 558K image-caption pairs
  - Goal: Align visual features to language embedding space

Stage 2: Visual Instruction Tuning
  - Freeze: Vision encoder
  - Train: Projection layer + LLM (full or LoRA)
  - Data: 665K multimodal instruction-following examples
  - Goal: Follow complex visual instructions

Forward Process (Training):
  x0 → x1 → x2 → ... → xT (pure Gaussian noise)
  Each step adds a small amount of noise

Reverse Process (Generation):
  xT → xT-1 → ... → x1 → x0 (clean image)
  Model learns to predict and remove noise at each step

The model learns: given noisy image xt and timestep t,
predict the noise that was added (or predict x0 directly)

Stable Diffusion Architecture:

Text Prompt ──→ CLIP Text Encoder ──→ Cross-Attention
                                           │
Random Noise ──→ U-Net (in latent space) ──┘
                      │
                      ↓ (iterative denoising)
                 Clean Latent
                      │
                      ↓
                 VAE Decoder ──→ Final Image (512x512 or higher)

Key insight: 512x512 image = 64x64 latent (64x less compute)

from openai import OpenAI

client = OpenAI()

def generate_image(prompt: str, size: str = "1024x1024") -> str:
    """Generate an image with DALL-E 3."""
    
    response = client.images.generate(
        model="dall-e-3",
        prompt=prompt,
        size=size,  # "1024x1024", "1792x1024", "1024x1792"
        quality="hd",  # "standard" or "hd"
        n=1
    )
    return response.data[0].url


def edit_image(image_path: str, mask_path: str, prompt: str) -> str:
    """Edit an image with inpainting (DALL-E 2)."""
    
    response = client.images.edit(
        model="dall-e-2",
        image=open(image_path, "rb"),
        mask=open(mask_path, "rb"),  # Transparent area = edit region
        prompt=prompt,
        size="1024x1024"
    )
    return response.data[0].url

from openai import OpenAI

client = OpenAI()

def transcribe_audio(audio_path: str, language: str = None) -> str:
    """Transcribe audio using Whisper."""
    
    with open(audio_path, "rb") as audio_file:
        transcript = client.audio.transcriptions.create(
            model="whisper-1",
            file=audio_file,
            language=language,  # ISO 639-1 code, or None for auto-detect
            response_format="verbose_json",
            timestamp_granularities=["segment"]
        )
    return transcript


def translate_audio(audio_path: str) -> str:
    """Translate any language audio to English text."""
    
    with open(audio_path, "rb") as audio_file:
        translation = client.audio.translations.create(
            model="whisper-1",
            file=audio_file
        )
    return translation.text

def text_to_speech(text: str, voice: str = "alloy") -> bytes:
    """Generate speech from text using OpenAI TTS."""
    
    response = client.audio.speech.create(
        model="tts-1-hd",  # "tts-1" (fast) or "tts-1-hd" (quality)
        voice=voice,  # alloy, echo, fable, onyx, nova, shimmer
        input=text,
        speed=1.0  # 0.25 to 4.0
    )
    return response.content  # Raw audio bytes (mp3)

# GPT-4o with audio input (native audio understanding)
def analyze_audio(audio_path: str, question: str) -> str:
    """Analyze audio content with GPT-4o audio."""
    
    import base64
    with open(audio_path, "rb") as f:
        audio_data = base64.standard_b64encode(f.read()).decode()
    
    response = client.chat.completions.create(
        model="gpt-4o-audio-preview",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": question},
                {"type": "input_audio", "input_audio": {
                    "data": audio_data,
                    "format": "mp3"
                }}
            ]
        }]
    )
    return response.choices[0].message.content

Video Diffusion Pipeline:

Text Prompt ──→ Text Encoder
                    │
                    ↓
Random Noise ──→ Spatial-Temporal U-Net/DiT ──→ Clean Latent Frames
(T frames)       │                                    │
                 │ Temporal attention                  ↓
                 │ (consistency across frames)    VAE Decoder
                 │                                    │
                 └── Spatial attention                ↓
                     (quality per frame)         Video Output

Key challenges:
- Temporal consistency (objects don't flicker/morph)
- Physics simulation (gravity, collisions, fluid)
- Long-range coherence (beginning matches end)
- Camera motion (smooth, intentional movement)

CLIP Contrastive Learning:

Given a batch of N (image, text) pairs:
- Encode all images → N image embeddings
- Encode all texts → N text embeddings
- Compute N×N similarity matrix
- Maximize similarity for matching pairs (diagonal)
- Minimize similarity for non-matching pairs (off-diagonal)

Result: images and text that describe the same thing
        land near each other in embedding space

from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import torch

model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")

def get_image_embedding(image_path: str) -> torch.Tensor:
    """Get CLIP embedding for an image."""
    image = Image.open(image_path)
    inputs = processor(images=image, return_tensors="pt")
    embedding = model.get_image_features(**inputs)
    return embedding / embedding.norm(dim=-1, keepdim=True)

def get_text_embedding(text: str) -> torch.Tensor:
    """Get CLIP embedding for text."""
    inputs = processor(text=text, return_tensors="pt", padding=True)
    embedding = model.get_text_features(**inputs)
    return embedding / embedding.norm(dim=-1, keepdim=True)

def image_text_similarity(image_path: str, texts: list[str]) -> list[float]:
    """Score how well each text describes the image."""
    image = Image.open(image_path)
    inputs = processor(text=texts, images=image, return_tensors="pt", padding=True)
    outputs = model(**inputs)
    # Cosine similarity scaled to [0, 1]
    similarities = outputs.logits_per_image.softmax(dim=1)
    return similarities[0].tolist()

Early Fusion:
  Modalities combined BEFORE main processing
  ┌─────────┐   ┌─────────┐
  │  Image  │   │  Text   │
  └────┬────┘   └────┬────┘
       │              │
       └──────┬───────┘  ← Combine early (concatenate, project)
              │
       ┌──────┴──────┐
       │  Shared      │
       │  Transformer │
       └──────┬──────┘
              │
         Output

Late Fusion:
  Modalities processed SEPARATELY, combined at the end
  ┌─────────┐   ┌─────────┐
  │  Image  │   │  Text   │
  └────┬────┘   └────┬────┘
       │              │
  ┌────┴────┐   ┌────┴────┐
  │ Vision  │   │Language │
  │ Model   │   │ Model   │
  └────┬────┘   └────┬────┘
       │              │
       └──────┬───────┘  ← Combine late (attention, MLP)
              │
         Output

def process_document(image_path: str) -> dict:
    """Extract structured data from a document image."""
    
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": """Analyze this document and extract:
1. Document type (invoice, receipt, form, letter, etc.)
2. Key fields as structured JSON
3. Any tables present (as arrays)
4. Handwritten notes if any

Return as JSON."""},
                {"type": "image_url", "image_url": {
                    "url": f"data:image/png;base64,{encode_image(image_path)}",
                    "detail": "high"
                }}
            ]
        }],
        response_format={"type": "json_object"}
    )
    return json.loads(response.choices[0].message.content)

class MultimodalAssistant:
    """Assistant that handles text, images, and audio."""
    
    def __init__(self):
        self.client = OpenAI()
        self.conversation = []
    
    def process_input(self, text: str = None, image_path: str = None, 
                      audio_path: str = None) -> str:
        """Process any combination of modalities."""
        
        content = []
        
        if text:
            content.append({"type": "text", "text": text})
        
        if image_path:
            with open(image_path, "rb") as f:
                img_data = base64.standard_b64encode(f.read()).decode()
            content.append({"type": "image_url", "image_url": {
                "url": f"data:image/png;base64,{img_data}"
            }})
        
        if audio_path:
            # Transcribe audio first, then include as text
            transcript = self.client.audio.transcriptions.create(
                model="whisper-1",
                file=open(audio_path, "rb")
            )
            content.append({"type": "text", 
                          "text": f"[Audio transcript]: {transcript.text}"})
        
        self.conversation.append({"role": "user", "content": content})
        
        response = self.client.chat.completions.create(
            model="gpt-4o",
            messages=self.conversation,
            max_tokens=2000
        )
        
        assistant_msg = response.choices[0].message.content
        self.conversation.append({"role": "assistant", "content": assistant_msg})
        return assistant_msg