GPT-4o: Omnimodal AI Deneyimi

1Text Input ──┐
2              ├──→ Unified Transformer ──→ Text Output
3Image Input ──┤    (Shared Attention)      Audio Output
4              ├──→     Layers              Image Output
5Audio Input ──┘

1# Cross-modal attention pseudocode
2class CrossModalAttention(nn.Module):
3    def __init__(self, dim=1024, num_heads=16):
4        super().__init__()
5        self.text_attn = nn.MultiheadAttention(dim, num_heads)
6        self.vision_attn = nn.MultiheadAttention(dim, num_heads)
7        self.audio_attn = nn.MultiheadAttention(dim, num_heads)
8        self.cross_attn = nn.MultiheadAttention(dim, num_heads)
9 
10    def forward(self, text_emb, vision_emb, audio_emb):
11        # Her modalite kendi self-attention'i
12        text_out = self.text_attn(text_emb, text_emb, text_emb)[0]
13        vis_out = self.vision_attn(vision_emb, vision_emb, vision_emb)[0]
14        audio_out = self.audio_attn(audio_emb, audio_emb, audio_emb)[0]
15 
16        # Modaliteler arasi cross-attention
17        # Metin, gorselden bilgi aliyor
18        fused = torch.cat([text_out, vis_out, audio_out], dim=1)
19        cross_out = self.cross_attn(fused, fused, fused)[0]
20 
21        return cross_out

1Kullanici sesi → Ses encoder → Unified model → Ses decoder → Yanit sesi
2 
3Latency breakdown:
4- Ses encoding: ~50ms
5- Model inference: ~200ms
6- Ses decoding: ~70ms
7- TOPLAM: ~320ms (insan algilama esigi ~400ms)

1from openai import OpenAI
2import base64
3 
4client = OpenAI()
5 
6def analyze_image(image_path: str, question: str) -> str:
7    """Gorseli analiz et ve soruya yanit ver"""
8    with open(image_path, "rb") as f:
9        image_data = base64.b64encode(f.read()).decode()
10 
11    response = client.chat.completions.create(
12        model="gpt-4o",
13        messages=[
14            {
15                "role": "user",
16                "content": [
17                    {"type": "text", "text": question},
18                    {
19                        "type": "image_url",
20                        "image_url": {
21                            "url": f"data:image/jpeg;base64,{image_data}",
22                            "detail": "high"  # low, auto, high
23                        }
24                    }
25                ]
26            }
27        ],
28        max_tokens=1000
29    )
30 
31    return response.choices[0].message.content
32 
33# Kullanim ornekleri
34# 1. UI tasarim incelemesi
35result = analyze_image(
36    "app_screenshot.png",
37    "Bu mobil uygulama ekranindaki UI/UX sorunlarini listele"
38)
39 
40# 2. Makbuz okuma
41receipt = analyze_image(
42    "receipt.jpg",
43    "Bu makbuzdaki toplam tutari ve kalemleri JSON formatinda ver"
44)
45 
46# 3. Whiteboard'dan kod cikarma
47code = analyze_image(
48    "whiteboard.jpg",
49    "Bu whiteboard'daki algoritmayı Python koduna cevir"
50)

1# Real-time ceviri pipeline ornegi
2import asyncio
3from openai import AsyncOpenAI
4 
5client = AsyncOpenAI()
6 
7async def live_translate(source_lang: str, target_lang: str):
8    """Canli sesli ceviri"""
9 
10    system_prompt = f"""Sen profesyonel bir canli tercumansin.
11    {source_lang} dilinde konusulanlari {target_lang} diline cevir.
12    Kurallarin:
13    - Anlam butunlugunu koru
14    - Kulturel bagLami adapte et
15    - Dogal konusma tonunda cevir
16    - Teknik terimleri hedef dilde acikla
17    """
18 
19    response = await client.chat.completions.create(
20        model="gpt-4o",
21        messages=[
22            {"role": "system", "content": system_prompt},
23            {"role": "user", "content": [
24                {"type": "input_audio", "input_audio": {
25                    "data": audio_base64,
26                    "format": "wav"
27                }}
28            ]}
29        ],
30        modalities=["text", "audio"],
31        audio={"voice": "alloy", "format": "wav"}
32    )
33 
34    return response.choices[0].message.audio

1from openai import OpenAI
2 
3client = OpenAI()
4 
5# Cok-modaliteli sohbet
6response = client.chat.completions.create(
7    model="gpt-4o",
8    messages=[
9        {
10            "role": "system",
11            "content": "Sen yardimsever bir AI asistansin. Turkce yanit ver."
12        },
13        {
14            "role": "user",
15            "content": [
16                {"type": "text", "text": "Bu gorseldeki kodu analiz et ve iyilestirme onerileri ver"},
17                {"type": "image_url", "image_url": {"url": "https://example.com/code.png"}}
18            ]
19        }
20    ],
21    max_tokens=2000,
22    temperature=0.7
23)
24 
25print(response.choices[0].message.content)

1from pydantic import BaseModel
2from typing import List
3 
4class UIAnalysis(BaseModel):
5    overall_score: int
6    issues: List[str]
7    suggestions: List[str]
8    accessibility_notes: List[str]
9 
10response = client.beta.chat.completions.parse(
11    model="gpt-4o",
12    messages=[
13        {"role": "system", "content": "UI/UX analisti olarak gorsel incele"},
14        {"role": "user", "content": [
15            {"type": "text", "text": "Bu ekran goruntusunu analiz et"},
16            {"type": "image_url", "image_url": {"url": screenshot_url}}
17        ]}
18    ],
19    response_format=UIAnalysis
20)
21 
22analysis = response.choices[0].message.parsed
23print(f"Skor: {analysis.overall_score}/100")
24for issue in analysis.issues:
25    print(f"  - {issue}")

1// Realtime API WebSocket baglantisi
2const ws = new WebSocket(
3  'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview',
4  {
5    headers: {
6      'Authorization': 'Bearer ' + process.env.OPENAI_API_KEY,
7      'OpenAI-Beta': 'realtime=v1'
8    }
9  }
10);
11 
12ws.on('open', () => {
13  // Session konfigurasyonu
14  ws.send(JSON.stringify({
15    type: 'session.update',
16    session: {
17      modalities: ['text', 'audio'],
18      instructions: 'Sen Turkce konusan yardimsever bir asistansin.',
19      voice: 'alloy',
20      input_audio_format: 'pcm16',
21      output_audio_format: 'pcm16',
22      input_audio_transcription: { model: 'whisper-1' },
23      turn_detection: { type: 'server_vad', threshold: 0.5 },
24      temperature: 0.8
25    }
26  }));
27});
28 
29// Ses verisi gonder
30ws.on('message', (data) => {
31  const event = JSON.parse(data);
32 
33  switch (event.type) {
34    case 'response.audio.delta':
35      // Ses parcasini oynat
36      playAudioChunk(Buffer.from(event.delta, 'base64'));
37      break;
38 
39    case 'response.audio_transcript.delta':
40      // Transkript guncelle
41      updateTranscript(event.delta);
42      break;
43 
44    case 'input_audio_buffer.speech_started':
45      // Kullanici konusmaya basladi
46      stopCurrentPlayback();
47      break;
48 
49    case 'error':
50      console.error('Realtime API error:', event.error);
51      break;
52  }
53});
54 
55// Mikrofon verisini gonder
56function sendAudioChunk(pcmData) {
57  ws.send(JSON.stringify({
58    type: 'input_audio_buffer.append',
59    audio: pcmData.toString('base64')
60  }));
61}

1import Foundation
2import AVFoundation
3 
4class GPT4oService {
5    private let apiKey: String
6    private let session = URLSession.shared
7 
8    init(apiKey: String) {
9        self.apiKey = apiKey
10    }
11 
12    func analyzeImage(_ image: UIImage, question: String) async throws -> String {
13        guard let imageData = image.jpegData(compressionQuality: 0.8) else {
14            throw GPT4oError.invalidImage
15        }
16 
17        let base64Image = imageData.base64EncodedString()
18 
19        let requestBody: [String: Any] = [
20            "model": "gpt-4o",
21            "messages": [
22                [
23                    "role": "user",
24                    "content": [
25                        ["type": "text", "text": question],
26                        ["type": "image_url", "image_url": [
27                            "url": "data:image/jpeg;base64,\(base64Image)",
28                            "detail": "auto"
29                        ]]
30                    ]
31                ]
32            ],
33            "max_tokens": 1000
34        ]
35 
36        var request = URLRequest(url: URL(string: "https://api.openai.com/v1/chat/completions")!)
37        request.httpMethod = "POST"
38        request.setValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
39        request.setValue("application/json", forHTTPHeaderField: "Content-Type")
40        request.httpBody = try JSONSerialization.data(withJSONObject: requestBody)
41 
42        let (data, _) = try await session.data(for: request)
43        let response = try JSONDecoder().decode(ChatResponse.self, from: data)
44 
45        return response.choices.first?.message.content ?? ""
46    }
47}

1import 'dart:convert';
2import 'package:http/http.dart' as http;
3 
4class GPT4oClient {
5  final String apiKey;
6 
7  GPT4oClient({required this.apiKey});
8 
9  Future chat({
10    required String message,
11    String? imageBase64,
12    String model = 'gpt-4o',
13  }) async {
14    final content = >[
15      {'type': 'text', 'text': message},
16    ];
17 
18    if (imageBase64 != null) {
19      content.add({
20        'type': 'image_url',
21        'image_url': {
22          'url': 'data:image/jpeg;base64,$imageBase64',
23          'detail': 'auto',
24        },
25      });
26    }
27 
28    final response = await http.post(
29      Uri.parse('https://api.openai.com/v1/chat/completions'),
30      headers: {
31        'Authorization': 'Bearer $apiKey',
32        'Content-Type': 'application/json',
33      },
34      body: jsonEncode({
35        'model': model,
36        'messages': [
37          {'role': 'user', 'content': content},
38        ],
39        'max_tokens': 1000,
40      }),
41    );
42 
43    if (response.statusCode != 200) {
44      throw Exception('API error: ${response.statusCode}');
45    }
46 
47    final data = jsonDecode(response.body);
48    return data['choices'][0]['message']['content'];
49  }
50}

1Metin: ~4 karakter = 1 token(Turkce icin ~3 karakter)
2Gorsel(low detail): ~85 token
3Gorsel(high detail): 85 + 170 * tile sayisi
4Ses: ~1 saniye = ~25 token(Realtime API)