BYOK kann jetzt Bilder verstehen. Vision läuft bei allen vier Providern über denselben Chat-Endpoint + Schlüssel — nur die Nachricht wird multimodal. - ByokContentPart (.text/.imageData) + multimodale ByokMessage; Text- Init bleibt back-compat (alle Text-Pfade unverändert). - ByokCapability + supportedCapabilities/visionModels/defaultVisionModel. - Adapter serialisieren Bild-Parts pro Schema (OpenAI image_url-DataURI, Anthropic source.base64, Gemini inline_data) — testbar extrahiert. - ManaLLM.describeImage(...) Vision-Verb; greift nur bei BYOK+vision- fähig, kein lokaler/Server-Fallback (nil ⇒ App nutzt Server/metered). - 9 neue netzwerk-freie Tests, swift test 23/23 grün. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
267 lines
10 KiB
Swift
267 lines
10 KiB
Swift
import Foundation
|
|
|
|
// Die vier eingebauten BYOK-Adapter. Jeder macht den direkten
|
|
// Gerät→Anbieter-Streaming-Call mit dem Key des Nutzers.
|
|
//
|
|
// Symmetrisch zu `@mana/byok-providers` (TS). Streaming via
|
|
// `URLSession.bytes` + zeilenweises SSE-Parsing; `[DONE]` und
|
|
// fehlerhafte Zeilen werden übersprungen.
|
|
|
|
// MARK: - Shared SSE plumbing
|
|
|
|
/// Sendet `request`, iteriert die `data:`-Zeilen der SSE-Antwort und
|
|
/// akkumuliert die per `extractDelta` gezogenen Text-Stücke. Liefert
|
|
/// den vollständigen Text. Wirft `ByokError.httpError` bei Status ≥ 400.
|
|
private func streamSSE(
|
|
_ request: URLRequest,
|
|
providerName _: String,
|
|
onToken: (@Sendable (String) -> Void)?,
|
|
extractDelta: (Any) -> String?
|
|
) async throws -> String {
|
|
let (bytes, response) = try await URLSession.shared.bytes(for: request)
|
|
|
|
if let http = response as? HTTPURLResponse, http.statusCode >= 400 {
|
|
// Body bestmöglich einsammeln für die Fehlermeldung.
|
|
var body = ""
|
|
for try await line in bytes.lines {
|
|
body += line
|
|
if body.count > 500 { break }
|
|
}
|
|
throw ByokError.httpError(status: http.statusCode, body: String(body.prefix(300)))
|
|
}
|
|
|
|
var content = ""
|
|
for try await line in bytes.lines {
|
|
let trimmed = line.trimmingCharacters(in: .whitespaces)
|
|
guard trimmed.hasPrefix("data:") else { continue }
|
|
let payload = trimmed.dropFirst("data:".count).trimmingCharacters(in: .whitespaces)
|
|
if payload.isEmpty || payload == "[DONE]" { continue }
|
|
guard let data = payload.data(using: .utf8),
|
|
let json = try? JSONSerialization.jsonObject(with: data)
|
|
else { continue }
|
|
if let token = extractDelta(json), !token.isEmpty {
|
|
content += token
|
|
onToken?(token)
|
|
}
|
|
}
|
|
return content
|
|
}
|
|
|
|
/// Baut eine JSON-POST-Request.
|
|
private func jsonRequest(url: URL, headers: [String: String], body: [String: Any]) throws -> URLRequest {
|
|
var request = URLRequest(url: url)
|
|
request.httpMethod = "POST"
|
|
request.httpBody = try JSONSerialization.data(withJSONObject: body)
|
|
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
|
|
for (k, v) in headers {
|
|
request.setValue(v, forHTTPHeaderField: k)
|
|
}
|
|
return request
|
|
}
|
|
|
|
// MARK: - Multimodaler Content (Phase A: Text + Bild)
|
|
|
|
//
|
|
// Pro Provider-Schema serialisiert. Bewusst `internal` (nicht private),
|
|
// damit die Encoding-Form netzwerk-frei getestet werden kann.
|
|
|
|
/// OpenAI / Mistral (OpenAI-kompatibel). Reiner Text → `String`
|
|
/// (back-compat); multimodal → Array aus `{type:text}` / `{type:image_url}`
|
|
/// mit Data-URI.
|
|
func byokOpenAICompatContent(_ parts: [ByokContentPart]) -> Any {
|
|
if parts.allSatisfy(\.isText) {
|
|
return parts.map(\.textOrEmpty).joined(separator: "\n")
|
|
}
|
|
return parts.map { part -> [String: Any] in
|
|
switch part {
|
|
case let .text(text):
|
|
["type": "text", "text": text]
|
|
case let .imageData(data, mime):
|
|
["type": "image_url", "image_url": ["url": "data:\(mime);base64,\(data.base64EncodedString())"]]
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Anthropic Messages-API. Reiner Text → `String`; multimodal → Array
|
|
/// aus `{type:text}` / `{type:image, source:{type:base64,…}}`.
|
|
func byokAnthropicContent(_ parts: [ByokContentPart]) -> Any {
|
|
if parts.allSatisfy(\.isText) {
|
|
return parts.map(\.textOrEmpty).joined(separator: "\n")
|
|
}
|
|
return parts.map { part -> [String: Any] in
|
|
switch part {
|
|
case let .text(text):
|
|
["type": "text", "text": text]
|
|
case let .imageData(data, mime):
|
|
["type": "image", "source": ["type": "base64", "media_type": mime, "data": data.base64EncodedString()]]
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Gemini `parts`-Array: `{text:…}` / `{inline_data:{mime_type,data}}`.
|
|
func byokGeminiParts(_ parts: [ByokContentPart]) -> [[String: Any]] {
|
|
parts.map { part -> [String: Any] in
|
|
switch part {
|
|
case let .text(text):
|
|
["text": text]
|
|
case let .imageData(data, mime):
|
|
["inline_data": ["mime_type": mime, "data": data.base64EncodedString()]]
|
|
}
|
|
}
|
|
}
|
|
|
|
private extension ByokContentPart {
|
|
var isText: Bool {
|
|
if case .text = self { true } else { false }
|
|
}
|
|
|
|
var textOrEmpty: String {
|
|
if case let .text(text) = self { text } else { "" }
|
|
}
|
|
}
|
|
|
|
// MARK: - OpenAI-compatible (OpenAI, Mistral, …)
|
|
|
|
/// Basis für OpenAI-kompatible ChatCompletions-APIs.
|
|
struct OpenAICompatByokProvider: ByokProvider {
|
|
let id: ByokProviderID
|
|
let baseURL: String
|
|
|
|
func call(
|
|
apiKey: String,
|
|
model: String,
|
|
messages: [ByokMessage],
|
|
temperature: Double,
|
|
maxTokens: Int,
|
|
onToken: (@Sendable (String) -> Void)?
|
|
) async throws -> String {
|
|
let url = URL(string: "\(baseURL)/chat/completions")!
|
|
let body: [String: Any] = [
|
|
"model": model,
|
|
"messages": messages.map { ["role": $0.role.rawValue, "content": byokOpenAICompatContent($0.parts)] },
|
|
"temperature": temperature,
|
|
"max_tokens": maxTokens,
|
|
"stream": true,
|
|
"stream_options": ["include_usage": true],
|
|
]
|
|
let request = try jsonRequest(
|
|
url: url,
|
|
headers: ["Authorization": "Bearer \(apiKey)"],
|
|
body: body
|
|
)
|
|
return try await streamSSE(request, providerName: id.displayName, onToken: onToken) { json in
|
|
guard let obj = json as? [String: Any],
|
|
let choices = obj["choices"] as? [[String: Any]],
|
|
let delta = choices.first?["delta"] as? [String: Any]
|
|
else { return nil }
|
|
return delta["content"] as? String
|
|
}
|
|
}
|
|
}
|
|
|
|
struct OpenAIByokProvider: ByokProvider {
|
|
let id: ByokProviderID = .openai
|
|
func call(apiKey: String, model: String, messages: [ByokMessage], temperature: Double, maxTokens: Int, onToken: (@Sendable (String) -> Void)?) async throws -> String {
|
|
try await OpenAICompatByokProvider(id: .openai, baseURL: "https://api.openai.com/v1")
|
|
.call(apiKey: apiKey, model: model, messages: messages, temperature: temperature, maxTokens: maxTokens, onToken: onToken)
|
|
}
|
|
}
|
|
|
|
struct MistralByokProvider: ByokProvider {
|
|
let id: ByokProviderID = .mistral
|
|
func call(apiKey: String, model: String, messages: [ByokMessage], temperature: Double, maxTokens: Int, onToken: (@Sendable (String) -> Void)?) async throws -> String {
|
|
try await OpenAICompatByokProvider(id: .mistral, baseURL: "https://api.mistral.ai/v1")
|
|
.call(apiKey: apiKey, model: model, messages: messages, temperature: temperature, maxTokens: maxTokens, onToken: onToken)
|
|
}
|
|
}
|
|
|
|
// MARK: - Anthropic
|
|
|
|
/// Anthropic Messages-API. System-Prompt separat; SSE-Schema mit
|
|
/// `content_block_delta` / `text_delta`.
|
|
struct AnthropicByokProvider: ByokProvider {
|
|
let id: ByokProviderID = .anthropic
|
|
|
|
func call(
|
|
apiKey: String,
|
|
model: String,
|
|
messages: [ByokMessage],
|
|
temperature: Double,
|
|
maxTokens: Int,
|
|
onToken: (@Sendable (String) -> Void)?
|
|
) async throws -> String {
|
|
let systemText = messages.filter { $0.role == .system }.map(\.textContent).joined(separator: "\n\n")
|
|
let chat = messages.filter { $0.role != .system }
|
|
var body: [String: Any] = [
|
|
"model": model,
|
|
"messages": chat.map { ["role": $0.role.rawValue, "content": byokAnthropicContent($0.parts)] },
|
|
"temperature": temperature,
|
|
"max_tokens": maxTokens,
|
|
"stream": true,
|
|
]
|
|
if !systemText.isEmpty { body["system"] = systemText }
|
|
|
|
let request = try jsonRequest(
|
|
url: URL(string: "https://api.anthropic.com/v1/messages")!,
|
|
headers: [
|
|
"x-api-key": apiKey,
|
|
"anthropic-version": "2023-06-01",
|
|
],
|
|
body: body
|
|
)
|
|
return try await streamSSE(request, providerName: id.displayName, onToken: onToken) { json in
|
|
guard let obj = json as? [String: Any],
|
|
obj["type"] as? String == "content_block_delta",
|
|
let delta = obj["delta"] as? [String: Any],
|
|
delta["type"] as? String == "text_delta"
|
|
else { return nil }
|
|
return delta["text"] as? String
|
|
}
|
|
}
|
|
}
|
|
|
|
// MARK: - Gemini
|
|
|
|
/// Gemini REST. Key im Query-String; eigenes Message-Schema
|
|
/// (`role`/`parts`, Rollen `user`/`model`); SSE via `:streamGenerateContent`.
|
|
struct GeminiByokProvider: ByokProvider {
|
|
let id: ByokProviderID = .gemini
|
|
|
|
func call(
|
|
apiKey: String,
|
|
model: String,
|
|
messages: [ByokMessage],
|
|
temperature: Double,
|
|
maxTokens: Int,
|
|
onToken: (@Sendable (String) -> Void)?
|
|
) async throws -> String {
|
|
let systemText = messages.filter { $0.role == .system }.map(\.textContent).joined(separator: "\n\n")
|
|
let contents = messages.filter { $0.role != .system }.map { msg -> [String: Any] in
|
|
[
|
|
"role": msg.role == .assistant ? "model" : "user",
|
|
"parts": byokGeminiParts(msg.parts),
|
|
]
|
|
}
|
|
var body: [String: Any] = [
|
|
"contents": contents,
|
|
"generationConfig": [
|
|
"temperature": temperature,
|
|
"maxOutputTokens": maxTokens,
|
|
],
|
|
]
|
|
if !systemText.isEmpty {
|
|
body["systemInstruction"] = ["parts": [["text": systemText]]]
|
|
}
|
|
|
|
let urlString = "https://generativelanguage.googleapis.com/v1beta/models/\(model):streamGenerateContent?alt=sse&key=\(apiKey)"
|
|
let request = try jsonRequest(url: URL(string: urlString)!, headers: [:], body: body)
|
|
return try await streamSSE(request, providerName: id.displayName, onToken: onToken) { json in
|
|
guard let obj = json as? [String: Any],
|
|
let candidates = obj["candidates"] as? [[String: Any]],
|
|
let content = candidates.first?["content"] as? [String: Any],
|
|
let parts = content["parts"] as? [[String: Any]]
|
|
else { return nil }
|
|
return parts.first?["text"] as? String
|
|
}
|
|
}
|
|
}
|