diff --git a/Package.resolved b/Package.resolved index d54996e..fe207e5 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,13 +1,22 @@ { - "originHash" : "73128af91f020c013de06bf6af5d06131ff05e38285118f5ff904ee06a3a6e24", + "originHash" : "1d1344dab64c4f153b2a1af227098e02f62d2c1f627c95dcad4304f1c16a97a3", "pins" : [ { - "identity" : "grpc-swift", + "identity" : "grpc-swift-2", "kind" : "remoteSourceControl", - "location" : "https://github.com/grpc/grpc-swift.git", + "location" : "https://github.com/grpc/grpc-swift-2.git", "state" : { - "revision" : "adc18c3e1c55027d0ce43893897ac448e3f27ebe", - "version" : "2.2.3" + "revision" : "531924b28fde0cf7585123c781c6f55cc35ef7fc", + "version" : "2.2.1" + } + }, + { + "identity" : "grpc-swift-extras", + "kind" : "remoteSourceControl", + "location" : "https://github.com/grpc/grpc-swift-extras.git", + "state" : { + "revision" : "7ab4a690ac09696689a9c4b99320af7ef809bb3d", + "version" : "2.1.1" } }, { @@ -15,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/grpc/grpc-swift-nio-transport.git", "state" : { - "revision" : "ca2303eb7f3df556beafbba33a143ffa30d5b786", - "version" : "1.2.3" + "revision" : "dcfa8dc858bba5ded7a3760cede8c5fc03558a42", + "version" : "2.4.0" } }, { @@ -24,8 +33,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/grpc/grpc-swift-protobuf.git", "state" : { - "revision" : "53e89e3a5d417307f70a721c7b83e564fefb1e1c", - "version" : "1.3.1" + "revision" : "a1aa982cb2a276c72b478433eb75a4ec6508a277", + "version" : "2.1.2" } }, { @@ -100,6 +109,15 @@ "version" : "4.2.0" } }, + { + "identity" : "swift-distributed-tracing", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-distributed-tracing.git", + "state" : { + "revision" : "baa932c1336f7894145cbaafcd34ce2dd0b77c97", + "version" : "1.3.1" + } + }, { "identity" : "swift-http-structured-headers", "kind" : "remoteSourceControl", @@ -190,6 +208,15 @@ "version" : "1.33.3" } }, + { + "identity" : "swift-service-context", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-service-context.git", + "state" : { + "revision" : "1983448fefc717a2bc2ebde5490fe99873c5b8a6", + "version" : "1.2.1" + } + }, { "identity" : "swift-service-lifecycle", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index cb50bce..a837d94 100644 --- a/Package.swift +++ b/Package.swift @@ -11,9 +11,10 @@ let package = Package( .executable(name: "AppleIntelligenceApp", targets: ["AppleIntelligenceApp"]), ], dependencies: [ - .package(url: "https://github.com/grpc/grpc-swift.git", from: "2.0.0"), - .package(url: "https://github.com/grpc/grpc-swift-nio-transport.git", from: "1.0.0"), - .package(url: "https://github.com/grpc/grpc-swift-protobuf.git", from: "1.0.0"), + .package(url: "https://github.com/grpc/grpc-swift-2.git", from: "2.0.0"), + .package(url: "https://github.com/grpc/grpc-swift-nio-transport.git", from: "2.0.0"), + .package(url: "https://github.com/grpc/grpc-swift-protobuf.git", from: "2.0.0"), + .package(url: "https://github.com/grpc/grpc-swift-extras.git", from: "2.0.0"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.28.0"), .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.3.0"), ], @@ -22,11 +23,15 @@ let package = Package( .target( name: "AppleIntelligenceCore", dependencies: [ - .product(name: "GRPCCore", package: "grpc-swift"), + .product(name: "GRPCCore", package: "grpc-swift-2"), .product(name: "GRPCNIOTransportHTTP2", package: "grpc-swift-nio-transport"), .product(name: "GRPCProtobuf", package: "grpc-swift-protobuf"), + .product(name: "GRPCReflectionService", package: "grpc-swift-extras"), .product(name: "SwiftProtobuf", package: "swift-protobuf"), ], + resources: [ + .copy("Resources/apple_intelligence.pb") + ], swiftSettings: [ .unsafeFlags(["-Xfrontend", "-suppress-warnings"]) ] diff --git a/Proto/apple_intelligence.proto b/Proto/apple_intelligence.proto new file mode 100644 index 0000000..c73947a --- /dev/null +++ b/Proto/apple_intelligence.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package appleintelligence; + +// Image data for vision requests +message ImageData { + bytes data = 1; + string filename = 2; + string mime_type = 3; +} + +// Vision analysis results +message ImageAnalysis { + string text_content = 1; + repeated string labels = 2; + string description = 3; +} + +// Completion request +message CompletionRequest { + string prompt = 1; + optional float temperature = 2; + optional int32 max_tokens = 3; + repeated ImageData images = 4; + bool include_analysis = 5; +} + +// Completion response (non-streaming) +message CompletionResponse { + string id = 1; + string text = 2; + string finish_reason = 3; + repeated ImageAnalysis image_analyses = 4; +} + +// Streaming completion chunk +message CompletionChunk { + string id = 1; + string delta = 2; + bool is_final = 3; + string finish_reason = 4; + repeated ImageAnalysis image_analyses = 5; +} + +// Health check request +message HealthRequest {} + +// Health check response +message HealthResponse { + bool healthy = 1; + string model_status = 2; +} + +// Apple Intelligence Service +service AppleIntelligenceService { + // Single completion request + rpc Complete(CompletionRequest) returns (CompletionResponse); + + // Streaming completion request + rpc StreamComplete(CompletionRequest) returns (stream CompletionChunk); + + // Health check + rpc Health(HealthRequest) returns (HealthResponse); +} diff --git a/Sources/AppleIntelligenceApp/App.swift b/Sources/AppleIntelligenceApp/App.swift index 5a47b93..51d36b9 100644 --- a/Sources/AppleIntelligenceApp/App.swift +++ b/Sources/AppleIntelligenceApp/App.swift @@ -34,7 +34,7 @@ struct AppleIntelligenceApp: App { .defaultSize(width: 500, height: 600) Window("Settings", id: "settings") { - SettingsView(settings: settings) + SettingsView(settings: settings, serverManager: serverManager) } .windowResizability(.contentSize) } diff --git a/Sources/AppleIntelligenceApp/Models/AppSettings.swift b/Sources/AppleIntelligenceApp/Models/AppSettings.swift index 46d17a5..a1e7ec0 100644 --- a/Sources/AppleIntelligenceApp/Models/AppSettings.swift +++ b/Sources/AppleIntelligenceApp/Models/AppSettings.swift @@ -19,6 +19,10 @@ final class AppSettings { didSet { UserDefaults.standard.set(autoStartServer, forKey: "auto_start_server") } } + var enableReflection: Bool { + didSet { UserDefaults.standard.set(enableReflection, forKey: "enable_reflection") } + } + var launchAtLogin: Bool { didSet { do { @@ -39,6 +43,12 @@ final class AppSettings { self.port = savedPort == 0 ? 50051 : savedPort self.apiKey = UserDefaults.standard.string(forKey: "api_key") ?? "" self.autoStartServer = UserDefaults.standard.bool(forKey: "auto_start_server") + // Default to true if not set + if UserDefaults.standard.object(forKey: "enable_reflection") == nil { + self.enableReflection = true + } else { + self.enableReflection = UserDefaults.standard.bool(forKey: "enable_reflection") + } self.launchAtLogin = SMAppService.mainApp.status == .enabled } @@ -47,6 +57,7 @@ final class AppSettings { port = 50051 apiKey = "" autoStartServer = false + enableReflection = true launchAtLogin = false } } diff --git a/Sources/AppleIntelligenceApp/Models/ChatMessage.swift b/Sources/AppleIntelligenceApp/Models/ChatMessage.swift index fb93efd..a5530fd 100644 --- a/Sources/AppleIntelligenceApp/Models/ChatMessage.swift +++ b/Sources/AppleIntelligenceApp/Models/ChatMessage.swift @@ -1,4 +1,62 @@ import Foundation +import AppKit + +/// Represents an attached image in a chat message +struct ImageAttachment: Identifiable, Equatable { + let id: UUID + let data: Data + let filename: String? + let thumbnail: NSImage? + let mimeType: String + + init(data: Data, filename: String? = nil) { + self.id = UUID() + self.data = data + self.filename = filename + self.thumbnail = Self.generateThumbnail(from: data) + self.mimeType = Self.detectMimeType(from: data) + } + + private static func generateThumbnail(from data: Data) -> NSImage? { + guard let image = NSImage(data: data) else { return nil } + + let maxSize: CGFloat = 100 + let ratio = min(maxSize / image.size.width, maxSize / image.size.height, 1.0) + let newSize = NSSize( + width: image.size.width * ratio, + height: image.size.height * ratio + ) + + let thumbnail = NSImage(size: newSize) + thumbnail.lockFocus() + image.draw( + in: NSRect(origin: .zero, size: newSize), + from: NSRect(origin: .zero, size: image.size), + operation: .copy, + fraction: 1.0 + ) + thumbnail.unlockFocus() + return thumbnail + } + + private static func detectMimeType(from data: Data) -> String { + guard data.count >= 4 else { return "application/octet-stream" } + let bytes = [UInt8](data.prefix(4)) + + if bytes[0] == 0x89 && bytes[1] == 0x50 && bytes[2] == 0x4E && bytes[3] == 0x47 { + return "image/png" + } else if bytes[0] == 0xFF && bytes[1] == 0xD8 { + return "image/jpeg" + } else if bytes[0] == 0x47 && bytes[1] == 0x49 && bytes[2] == 0x46 { + return "image/gif" + } + return "image/png" // Default to PNG + } + + static func == (lhs: ImageAttachment, rhs: ImageAttachment) -> Bool { + lhs.id == rhs.id + } +} struct ChatMessage: Identifiable, Equatable { let id: UUID @@ -6,17 +64,19 @@ struct ChatMessage: Identifiable, Equatable { var content: String let timestamp: Date var isStreaming: Bool + var images: [ImageAttachment] enum Role: Equatable { case user case assistant } - init(role: Role, content: String, isStreaming: Bool = false) { + init(role: Role, content: String, isStreaming: Bool = false, images: [ImageAttachment] = []) { self.id = UUID() self.role = role self.content = content self.timestamp = Date() self.isStreaming = isStreaming + self.images = images } } diff --git a/Sources/AppleIntelligenceApp/ServerManager.swift b/Sources/AppleIntelligenceApp/ServerManager.swift index 7fb414d..35bdd7f 100644 --- a/Sources/AppleIntelligenceApp/ServerManager.swift +++ b/Sources/AppleIntelligenceApp/ServerManager.swift @@ -2,6 +2,7 @@ import Foundation import AppleIntelligenceCore import GRPCCore import GRPCNIOTransportHTTP2 +import GRPCReflectionService @MainActor @Observable @@ -51,6 +52,7 @@ final class ServerManager { let host = settings.host let port = settings.port let apiKey = settings.apiKey.isEmpty ? nil : settings.apiKey + let enableReflection = settings.enableReflection serverTask = Task { do { @@ -82,7 +84,16 @@ final class ServerManager { config: .defaults ) - let server = GRPCServer(transport: transport, services: [provider]) + // Build services list with optional reflection + var services: [any RegistrableRPCService] = [provider] + if enableReflection { + if let descriptorURL = AppleIntelligenceResources.descriptorSetURL { + let reflectionService = try ReflectionService(descriptorSetFileURLs: [descriptorURL]) + services.append(reflectionService) + } + } + + let server = GRPCServer(transport: transport, services: services) await MainActor.run { self.state = .running(host: host, port: port) @@ -113,6 +124,19 @@ final class ServerManager { state = .stopped } + func restart() { + guard state.isRunning else { return } + + // Stop the current server + stop() + state = .starting + + // Start again after a short delay to allow port release + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in + self?.start() + } + } + func toggle() { if state.isRunning { stop() diff --git a/Sources/AppleIntelligenceApp/ViewModels/ChatViewModel.swift b/Sources/AppleIntelligenceApp/ViewModels/ChatViewModel.swift index 0a92ff7..cf14370 100644 --- a/Sources/AppleIntelligenceApp/ViewModels/ChatViewModel.swift +++ b/Sources/AppleIntelligenceApp/ViewModels/ChatViewModel.swift @@ -1,4 +1,6 @@ import Foundation +import AppKit +import UniformTypeIdentifiers import AppleIntelligenceCore @MainActor @@ -9,11 +11,69 @@ final class ChatViewModel { var isLoading: Bool = false var errorMessage: String? + // Image attachment state + var pendingImages: [ImageAttachment] = [] + private var service: AppleIntelligenceService? private var currentTask: Task? + // Maximum images per message + private let maxImagesPerMessage = 5 + + // Supported image types + static let supportedImageTypes: [UTType] = [.png, .jpeg, .gif, .webP, .heic] + + // Recent images from Downloads and Desktop + var recentImages: [URL] = [] + func initialize() async { service = await AppleIntelligenceService() + loadRecentImages() + } + + // MARK: - Recent Images + + func loadRecentImages() { + let fileManager = FileManager.default + let homeDir = fileManager.homeDirectoryForCurrentUser + + let folders = [ + homeDir.appendingPathComponent("Downloads"), + homeDir.appendingPathComponent("Desktop") + ] + + let imageExtensions = ["png", "jpg", "jpeg", "gif", "webp", "heic", "heif"] + + var allImages: [(url: URL, date: Date)] = [] + + for folder in folders { + guard let contents = try? fileManager.contentsOfDirectory( + at: folder, + includingPropertiesForKeys: [.contentModificationDateKey, .isRegularFileKey], + options: [.skipsHiddenFiles] + ) else { continue } + + for url in contents { + let ext = url.pathExtension.lowercased() + guard imageExtensions.contains(ext) else { continue } + + if let attributes = try? url.resourceValues(forKeys: [.contentModificationDateKey, .isRegularFileKey]), + attributes.isRegularFile == true, + let modDate = attributes.contentModificationDate { + allImages.append((url: url, date: modDate)) + } + } + } + + // Sort by date descending and take last 10 + recentImages = allImages + .sorted { $0.date > $1.date } + .prefix(10) + .map { $0.url } + } + + func addRecentImage(_ url: URL) { + addImage(from: url) } var isServiceAvailable: Bool { @@ -22,19 +82,77 @@ final class ChatViewModel { } } + var canSend: Bool { + !inputText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !pendingImages.isEmpty + } + + // MARK: - Image Handling + + func addImage(from url: URL) { + guard pendingImages.count < maxImagesPerMessage else { + errorMessage = "Maximum \(maxImagesPerMessage) images per message" + return + } + + do { + let data = try Data(contentsOf: url) + let attachment = ImageAttachment(data: data, filename: url.lastPathComponent) + pendingImages.append(attachment) + errorMessage = nil + } catch { + errorMessage = "Failed to load image: \(error.localizedDescription)" + } + } + + func addImageFromPasteboard() { + guard let image = NSPasteboard.general.readObjects( + forClasses: [NSImage.self], + options: nil + )?.first as? NSImage else { + return + } + + guard pendingImages.count < maxImagesPerMessage else { + errorMessage = "Maximum \(maxImagesPerMessage) images per message" + return + } + + if let tiffData = image.tiffRepresentation, + let bitmap = NSBitmapImageRep(data: tiffData), + let pngData = bitmap.representation(using: .png, properties: [:]) { + let attachment = ImageAttachment(data: pngData, filename: "pasted_image.png") + pendingImages.append(attachment) + errorMessage = nil + } + } + + func removePendingImage(_ attachment: ImageAttachment) { + pendingImages.removeAll { $0.id == attachment.id } + } + + func clearPendingImages() { + pendingImages.removeAll() + } + + // MARK: - Messaging + func sendMessage() { let text = inputText.trimmingCharacters(in: .whitespacesAndNewlines) - guard !text.isEmpty else { return } + guard !text.isEmpty || !pendingImages.isEmpty else { return } guard !isLoading else { return } - // Add user message - let userMessage = ChatMessage(role: .user, content: text) + // Capture images before clearing + let imagesToSend = pendingImages + + // Add user message with images + let userMessage = ChatMessage(role: .user, content: text, images: imagesToSend) messages.append(userMessage) inputText = "" + pendingImages = [] errorMessage = nil // Add placeholder for assistant response - var assistantMessage = ChatMessage(role: .assistant, content: "", isStreaming: true) + let assistantMessage = ChatMessage(role: .assistant, content: "", isStreaming: true) messages.append(assistantMessage) isLoading = true @@ -45,14 +163,20 @@ final class ChatViewModel { throw AppleIntelligenceError.modelNotAvailable } + // Convert attachments to service format + let images = imagesToSend.map { attachment in + (data: attachment.data, filename: attachment.filename) + } + let stream = await service.streamComplete( prompt: text, temperature: nil, - maxTokens: nil + maxTokens: nil, + images: images ) var fullResponse = "" - for try await partialResponse in stream { + for try await (partialResponse, _) in stream { fullResponse = partialResponse // Update the last message (assistant's response) if let index = messages.lastIndex(where: { $0.role == .assistant }) { diff --git a/Sources/AppleIntelligenceApp/Views/ChatView.swift b/Sources/AppleIntelligenceApp/Views/ChatView.swift index 29c67c4..c678407 100644 --- a/Sources/AppleIntelligenceApp/Views/ChatView.swift +++ b/Sources/AppleIntelligenceApp/Views/ChatView.swift @@ -1,97 +1,103 @@ import SwiftUI +import UniformTypeIdentifiers struct ChatView: View { @Bindable var viewModel: ChatViewModel @FocusState private var isInputFocused: Bool + @State private var isShowingFilePicker = false + @State private var isDragOver = false + @State private var previewImageURL: URL? var body: some View { - VStack(spacing: 0) { - // Messages list - ScrollViewReader { proxy in - ScrollView { - LazyVStack(spacing: 12) { - ForEach(viewModel.messages) { message in - MessageBubble(message: message) - .id(message.id) - } - } - .padding() - } - .onChange(of: viewModel.messages.count) { _, _ in - if let lastMessage = viewModel.messages.last { - withAnimation { - proxy.scrollTo(lastMessage.id, anchor: .bottom) - } - } - } - .onChange(of: viewModel.messages.last?.content) { _, _ in - if let lastMessage = viewModel.messages.last { - withAnimation { - proxy.scrollTo(lastMessage.id, anchor: .bottom) - } - } - } + HStack(spacing: 0) { + // Recent images sidebar + if !viewModel.recentImages.isEmpty { + recentImagesSidebar + Divider() } - // Error message - if let error = viewModel.errorMessage { - HStack { - Image(systemName: "exclamationmark.triangle.fill") - .foregroundStyle(.yellow) - Text(error) + // Main chat area + VStack(spacing: 0) { + // Messages list + ScrollViewReader { proxy in + ScrollView { + LazyVStack(spacing: 12) { + ForEach(viewModel.messages) { message in + MessageBubble(message: message) + .id(message.id) + } + } + .padding() + } + .onChange(of: viewModel.messages.count) { _, _ in + if let lastMessage = viewModel.messages.last { + withAnimation { + proxy.scrollTo(lastMessage.id, anchor: .bottom) + } + } + } + .onChange(of: viewModel.messages.last?.content) { _, _ in + if let lastMessage = viewModel.messages.last { + withAnimation { + proxy.scrollTo(lastMessage.id, anchor: .bottom) + } + } + } + } + + // Error message + if let error = viewModel.errorMessage { + HStack { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + Text(error) + .font(.caption) + .foregroundStyle(.secondary) + Spacer() + Button("Dismiss") { + viewModel.errorMessage = nil + } + .buttonStyle(.plain) .font(.caption) - .foregroundStyle(.secondary) - Spacer() - Button("Dismiss") { - viewModel.errorMessage = nil } - .buttonStyle(.plain) - .font(.caption) + .padding(.horizontal) + .padding(.vertical, 8) + .background(.red.opacity(0.1)) } - .padding(.horizontal) - .padding(.vertical, 8) - .background(.red.opacity(0.1)) + + Divider() + + // Pending images preview + if !viewModel.pendingImages.isEmpty { + pendingImagesView + } + + // Input area + inputArea } - - Divider() - - // Input area - HStack(spacing: 12) { - TextField("Message...", text: $viewModel.inputText, axis: .vertical) - .textFieldStyle(.plain) - .lineLimit(1...5) - .focused($isInputFocused) - .onSubmit { - if !viewModel.inputText.isEmpty { - viewModel.sendMessage() - } - } - - if viewModel.isLoading { - Button { - viewModel.stopGeneration() - } label: { - Image(systemName: "stop.circle.fill") - .font(.title2) - .foregroundStyle(.red) - } - .buttonStyle(.plain) - } else { - Button { - viewModel.sendMessage() - } label: { - Image(systemName: "arrow.up.circle.fill") - .font(.title2) - .foregroundStyle(viewModel.inputText.isEmpty ? .gray : .accentColor) - } - .buttonStyle(.plain) - .disabled(viewModel.inputText.isEmpty) + .onDrop(of: [.fileURL, .image], isTargeted: $isDragOver) { providers in + handleDrop(providers: providers) + return true + } + .overlay { + if isDragOver { + RoundedRectangle(cornerRadius: 8) + .stroke(Color.accentColor, lineWidth: 3) + .background(Color.accentColor.opacity(0.1)) + .padding(4) } } - .padding() } - .frame(minWidth: 400, minHeight: 500) + .frame(minWidth: 500, minHeight: 500) .toolbar { + ToolbarItem(placement: .primaryAction) { + Button { + viewModel.loadRecentImages() + } label: { + Image(systemName: "arrow.clockwise") + } + .help("Refresh recent images") + } ToolbarItem(placement: .primaryAction) { Button { viewModel.clearChat() @@ -106,12 +112,10 @@ struct ChatView: View { await viewModel.initialize() } .onAppear { - // Force the app to become active and accept keyboard input NSApp.setActivationPolicy(.regular) NSApp.activate(ignoringOtherApps: true) DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { - // Make sure the window is key if let window = NSApp.windows.first(where: { $0.title == "Chat" }) { window.makeKeyAndOrderFront(nil) } @@ -119,16 +123,270 @@ struct ChatView: View { } } .onDisappear { - // Return to accessory mode when chat is closed if NSApp.windows.filter({ $0.isVisible && $0.title != "" }).isEmpty { NSApp.setActivationPolicy(.accessory) } } + .fileImporter( + isPresented: $isShowingFilePicker, + allowedContentTypes: ChatViewModel.supportedImageTypes, + allowsMultipleSelection: true + ) { result in + switch result { + case .success(let urls): + for url in urls { + if url.startAccessingSecurityScopedResource() { + viewModel.addImage(from: url) + url.stopAccessingSecurityScopedResource() + } + } + case .failure(let error): + viewModel.errorMessage = error.localizedDescription + } + } + .sheet(item: $previewImageURL) { url in + ImagePreviewSheet(url: url) { + viewModel.addRecentImage(url) + previewImageURL = nil + } onCancel: { + previewImageURL = nil + } + } + } + + // MARK: - Drag & Drop Handler + + private func handleDrop(providers: [NSItemProvider]) { + for provider in providers { + // Try to load as file URL first + if provider.hasItemConformingToTypeIdentifier(UTType.fileURL.identifier) { + provider.loadItem(forTypeIdentifier: UTType.fileURL.identifier, options: nil) { item, error in + guard error == nil else { return } + + if let data = item as? Data, + let url = URL(dataRepresentation: data, relativeTo: nil) { + DispatchQueue.main.async { + viewModel.addImage(from: url) + } + } else if let url = item as? URL { + DispatchQueue.main.async { + viewModel.addImage(from: url) + } + } + } + } + // Try to load as image data + else if provider.hasItemConformingToTypeIdentifier(UTType.image.identifier) { + provider.loadDataRepresentation(forTypeIdentifier: UTType.image.identifier) { data, error in + guard let data = data, error == nil else { return } + DispatchQueue.main.async { + let attachment = ImageAttachment(data: data, filename: "dropped_image.png") + if viewModel.pendingImages.count < 5 { + viewModel.pendingImages.append(attachment) + } + } + } + } + } + } + + // MARK: - Recent Images Sidebar + + private var recentImagesSidebar: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Recent") + .font(.headline) + .foregroundStyle(.secondary) + .padding(.horizontal, 8) + .padding(.top, 8) + + ScrollView { + LazyVStack(spacing: 8) { + ForEach(viewModel.recentImages, id: \.self) { url in + RecentImageThumbnail(url: url) { + previewImageURL = url + } + } + } + .padding(.horizontal, 8) + .padding(.bottom, 8) + } + } + .frame(width: 100) + .background(Color(nsColor: .controlBackgroundColor).opacity(0.5)) + } + + // MARK: - Pending Images Preview + + private var pendingImagesView: some View { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(viewModel.pendingImages) { attachment in + pendingImageThumbnail(attachment) + } + } + .padding(.horizontal) + .padding(.vertical, 8) + } + .background(Color(nsColor: .controlBackgroundColor)) + } + + private func pendingImageThumbnail(_ attachment: ImageAttachment) -> some View { + ZStack(alignment: .topTrailing) { + if let thumbnail = attachment.thumbnail { + Image(nsImage: thumbnail) + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: 60, height: 60) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } else { + RoundedRectangle(cornerRadius: 8) + .fill(Color.gray.opacity(0.3)) + .frame(width: 60, height: 60) + .overlay { + Image(systemName: "photo") + .foregroundStyle(.secondary) + } + } + + Button { + viewModel.removePendingImage(attachment) + } label: { + Image(systemName: "xmark.circle.fill") + .font(.system(size: 16)) + .foregroundStyle(.white) + .background(Circle().fill(.black.opacity(0.6)).frame(width: 18, height: 18)) + } + .buttonStyle(.plain) + .offset(x: 6, y: -6) + } + } + + // MARK: - Input Area + + private var inputArea: some View { + HStack(spacing: 8) { + Button { + isShowingFilePicker = true + } label: { + Image(systemName: "photo.badge.plus") + .font(.title3) + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + .help("Add image") + + Button { + viewModel.addImageFromPasteboard() + } label: { + Image(systemName: "doc.on.clipboard") + .font(.title3) + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + .help("Paste image from clipboard") + + TextField("Message...", text: $viewModel.inputText, axis: .vertical) + .textFieldStyle(.plain) + .lineLimit(1...5) + .focused($isInputFocused) + .onSubmit { + if viewModel.canSend { + viewModel.sendMessage() + } + } + + if viewModel.isLoading { + Button { + viewModel.stopGeneration() + } label: { + Image(systemName: "stop.circle.fill") + .font(.title2) + .foregroundStyle(.red) + } + .buttonStyle(.plain) + } else { + Button { + viewModel.sendMessage() + } label: { + Image(systemName: "arrow.up.circle.fill") + .font(.title2) + .foregroundStyle(viewModel.canSend ? Color.accentColor : Color.gray) + } + .buttonStyle(.plain) + .disabled(!viewModel.canSend) + } + } + .padding() } } +// MARK: - Recent Image Thumbnail + +struct RecentImageThumbnail: View { + let url: URL + let onTap: () -> Void + + @State private var thumbnail: NSImage? + + var body: some View { + Button(action: onTap) { + ZStack { + if let thumbnail = thumbnail { + Image(nsImage: thumbnail) + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: 80, height: 80) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } else { + RoundedRectangle(cornerRadius: 8) + .fill(Color.gray.opacity(0.3)) + .frame(width: 80, height: 80) + .overlay { + ProgressView() + .scaleEffect(0.6) + } + } + } + } + .buttonStyle(.plain) + .help(url.lastPathComponent) + .task { + await loadThumbnail() + } + } + + private func loadThumbnail() async { + guard let image = NSImage(contentsOf: url) else { return } + + let maxSize: CGFloat = 80 + let ratio = min(maxSize / image.size.width, maxSize / image.size.height, 1.0) + let newSize = NSSize( + width: image.size.width * ratio, + height: image.size.height * ratio + ) + + let thumb = NSImage(size: newSize) + thumb.lockFocus() + image.draw( + in: NSRect(origin: .zero, size: newSize), + from: NSRect(origin: .zero, size: image.size), + operation: .copy, + fraction: 1.0 + ) + thumb.unlockFocus() + + await MainActor.run { + thumbnail = thumb + } + } +} + +// MARK: - Message Bubble + struct MessageBubble: View { let message: ChatMessage + @State private var showCopied = false var body: some View { HStack { @@ -137,13 +395,19 @@ struct MessageBubble: View { } VStack(alignment: message.role == .user ? .trailing : .leading, spacing: 4) { - Text(message.content) - .textSelection(.enabled) - .padding(.horizontal, 12) - .padding(.vertical, 8) - .background(bubbleColor) - .foregroundStyle(message.role == .user ? .white : .primary) - .clipShape(RoundedRectangle(cornerRadius: 16)) + if !message.images.isEmpty { + imageGrid + } + + if !message.content.isEmpty { + Text(message.content) + .textSelection(.enabled) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(bubbleColor) + .foregroundStyle(message.role == .user ? .white : .primary) + .clipShape(RoundedRectangle(cornerRadius: 16)) + } if message.isStreaming { HStack(spacing: 4) { @@ -154,6 +418,30 @@ struct MessageBubble: View { .foregroundStyle(.secondary) } } + + // Copy button for assistant messages + if message.role == .assistant && !message.content.isEmpty && !message.isStreaming { + HStack { + Spacer() + Button { + NSPasteboard.general.clearContents() + NSPasteboard.general.setString(message.content, forType: .string) + showCopied = true + DispatchQueue.main.asyncAfter(deadline: .now() + 1.5) { + showCopied = false + } + } label: { + HStack(spacing: 4) { + Image(systemName: showCopied ? "checkmark" : "doc.on.doc") + Text(showCopied ? "Copied" : "Copy") + } + .font(.caption) + .foregroundStyle(.secondary) + } + .buttonStyle(.plain) + } + .padding(.top, 2) + } } if message.role == .assistant { @@ -162,6 +450,32 @@ struct MessageBubble: View { } } + @ViewBuilder + private var imageGrid: some View { + let columns = min(message.images.count, 3) + LazyVGrid( + columns: Array(repeating: GridItem(.flexible(), spacing: 4), count: columns), + spacing: 4 + ) { + ForEach(message.images) { attachment in + if let thumbnail = attachment.thumbnail { + Image(nsImage: thumbnail) + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: 80, height: 80) + .clipShape(RoundedRectangle(cornerRadius: 8)) + } + } + } + .padding(4) + .background( + message.role == .user + ? Color.accentColor.opacity(0.8) + : Color(nsColor: .controlBackgroundColor) + ) + .clipShape(RoundedRectangle(cornerRadius: 12)) + } + private var bubbleColor: Color { switch message.role { case .user: @@ -171,3 +485,65 @@ struct MessageBubble: View { } } } + +// MARK: - Image Preview Sheet + +struct ImagePreviewSheet: View { + let url: URL + let onConfirm: () -> Void + let onCancel: () -> Void + + @State private var image: NSImage? + + var body: some View { + VStack(spacing: 16) { + Text("Add Image") + .font(.headline) + + if let image = image { + Image(nsImage: image) + .resizable() + .aspectRatio(contentMode: .fit) + .frame(maxWidth: 500, maxHeight: 400) + .clipShape(RoundedRectangle(cornerRadius: 8)) + .shadow(radius: 4) + } else { + RoundedRectangle(cornerRadius: 8) + .fill(Color.gray.opacity(0.2)) + .frame(width: 300, height: 200) + .overlay { + ProgressView() + } + } + + Text(url.lastPathComponent) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + + HStack(spacing: 16) { + Button("Cancel") { + onCancel() + } + .keyboardShortcut(.cancelAction) + + Button("Add to Message") { + onConfirm() + } + .keyboardShortcut(.defaultAction) + .buttonStyle(.borderedProminent) + } + } + .padding(24) + .frame(minWidth: 400, minHeight: 300) + .task { + image = NSImage(contentsOf: url) + } + } +} + +// MARK: - URL Identifiable Extension + +extension URL: @retroactive Identifiable { + public var id: String { absoluteString } +} diff --git a/Sources/AppleIntelligenceApp/Views/SettingsView.swift b/Sources/AppleIntelligenceApp/Views/SettingsView.swift index b2e1390..afcbe1a 100644 --- a/Sources/AppleIntelligenceApp/Views/SettingsView.swift +++ b/Sources/AppleIntelligenceApp/Views/SettingsView.swift @@ -2,6 +2,7 @@ import SwiftUI struct SettingsView: View { @Bindable var settings: AppSettings + var serverManager: ServerManager? @Environment(\.dismiss) private var dismiss var body: some View { @@ -10,7 +11,7 @@ struct SettingsView: View { TextField("Host", text: $settings.host) .textFieldStyle(.roundedBorder) - TextField("Port", value: $settings.port, format: .number) + TextField("Port", value: $settings.port, format: .number.grouping(.never)) .textFieldStyle(.roundedBorder) SecureField("API Key (optional)", text: $settings.apiKey) @@ -22,6 +23,13 @@ struct SettingsView: View { Toggle("Auto-start server on launch", isOn: $settings.autoStartServer) } + Section("API") { + Toggle("Enable gRPC reflection", isOn: $settings.enableReflection) + .onChange(of: settings.enableReflection) { _, _ in + serverManager?.restart() + } + } + Section { HStack { Button("Reset to Defaults") { @@ -38,7 +46,7 @@ struct SettingsView: View { } } .formStyle(.grouped) - .frame(width: 400, height: 310) + .frame(width: 400, height: 380) .fixedSize() .onAppear { NSApp.setActivationPolicy(.regular) diff --git a/Sources/AppleIntelligenceCore/Generated/AppleIntelligence.pb.swift b/Sources/AppleIntelligenceCore/Generated/AppleIntelligence.pb.swift deleted file mode 100644 index 9956962..0000000 --- a/Sources/AppleIntelligenceCore/Generated/AppleIntelligence.pb.swift +++ /dev/null @@ -1,238 +0,0 @@ -// DO NOT EDIT. -// swift-format-ignore-file -// swiftlint:disable all -// -// Generated protocol buffer code for apple_intelligence.proto - -import Foundation -import SwiftProtobuf - -// MARK: - Messages - -struct Appleintelligence_CompletionRequest: Sendable, SwiftProtobuf.Message { - static let protoMessageName: String = "appleintelligence.CompletionRequest" - - var prompt: String = "" - var temperature: Float = 0 - var maxTokens: Int32 = 0 - - var hasTemperature: Bool = false - var hasMaxTokens: Bool = false - - var unknownFields = SwiftProtobuf.UnknownStorage() - - init() {} - - init(prompt: String, temperature: Float? = nil, maxTokens: Int32? = nil) { - self.prompt = prompt - if let temp = temperature { - self.temperature = temp - self.hasTemperature = true - } - if let tokens = maxTokens { - self.maxTokens = tokens - self.hasMaxTokens = true - } - } - - mutating func decodeMessage(decoder: inout D) throws { - while let fieldNumber = try decoder.nextFieldNumber() { - switch fieldNumber { - case 1: try decoder.decodeSingularStringField(value: &prompt) - case 2: - try decoder.decodeSingularFloatField(value: &temperature) - hasTemperature = true - case 3: - try decoder.decodeSingularInt32Field(value: &maxTokens) - hasMaxTokens = true - default: break - } - } - } - - func traverse(visitor: inout V) throws { - if !prompt.isEmpty { - try visitor.visitSingularStringField(value: prompt, fieldNumber: 1) - } - if hasTemperature { - try visitor.visitSingularFloatField(value: temperature, fieldNumber: 2) - } - if hasMaxTokens { - try visitor.visitSingularInt32Field(value: maxTokens, fieldNumber: 3) - } - try unknownFields.traverse(visitor: &visitor) - } - - static func ==(lhs: Self, rhs: Self) -> Bool { - lhs.prompt == rhs.prompt && lhs.temperature == rhs.temperature && lhs.maxTokens == rhs.maxTokens && lhs.unknownFields == rhs.unknownFields - } - - func isEqualTo(message: any SwiftProtobuf.Message) -> Bool { - guard let other = message as? Self else { return false } - return self == other - } -} - -struct Appleintelligence_CompletionResponse: Sendable, SwiftProtobuf.Message { - static let protoMessageName: String = "appleintelligence.CompletionResponse" - - var id: String = "" - var text: String = "" - var finishReason: String = "" - - var unknownFields = SwiftProtobuf.UnknownStorage() - - init() {} - - mutating func decodeMessage(decoder: inout D) throws { - while let fieldNumber = try decoder.nextFieldNumber() { - switch fieldNumber { - case 1: try decoder.decodeSingularStringField(value: &id) - case 2: try decoder.decodeSingularStringField(value: &text) - case 3: try decoder.decodeSingularStringField(value: &finishReason) - default: break - } - } - } - - func traverse(visitor: inout V) throws { - if !id.isEmpty { - try visitor.visitSingularStringField(value: id, fieldNumber: 1) - } - if !text.isEmpty { - try visitor.visitSingularStringField(value: text, fieldNumber: 2) - } - if !finishReason.isEmpty { - try visitor.visitSingularStringField(value: finishReason, fieldNumber: 3) - } - try unknownFields.traverse(visitor: &visitor) - } - - static func ==(lhs: Self, rhs: Self) -> Bool { - lhs.id == rhs.id && lhs.text == rhs.text && lhs.finishReason == rhs.finishReason && lhs.unknownFields == rhs.unknownFields - } - - func isEqualTo(message: any SwiftProtobuf.Message) -> Bool { - guard let other = message as? Self else { return false } - return self == other - } -} - -struct Appleintelligence_CompletionChunk: Sendable, SwiftProtobuf.Message { - static let protoMessageName: String = "appleintelligence.CompletionChunk" - - var id: String = "" - var delta: String = "" - var isFinal: Bool = false - var finishReason: String = "" - - var hasFinishReason: Bool { - !finishReason.isEmpty - } - - var unknownFields = SwiftProtobuf.UnknownStorage() - - init() {} - - mutating func decodeMessage(decoder: inout D) throws { - while let fieldNumber = try decoder.nextFieldNumber() { - switch fieldNumber { - case 1: try decoder.decodeSingularStringField(value: &id) - case 2: try decoder.decodeSingularStringField(value: &delta) - case 3: try decoder.decodeSingularBoolField(value: &isFinal) - case 4: try decoder.decodeSingularStringField(value: &finishReason) - default: break - } - } - } - - func traverse(visitor: inout V) throws { - if !id.isEmpty { - try visitor.visitSingularStringField(value: id, fieldNumber: 1) - } - if !delta.isEmpty { - try visitor.visitSingularStringField(value: delta, fieldNumber: 2) - } - if isFinal { - try visitor.visitSingularBoolField(value: isFinal, fieldNumber: 3) - } - if !finishReason.isEmpty { - try visitor.visitSingularStringField(value: finishReason, fieldNumber: 4) - } - try unknownFields.traverse(visitor: &visitor) - } - - static func ==(lhs: Self, rhs: Self) -> Bool { - lhs.id == rhs.id && lhs.delta == rhs.delta && lhs.isFinal == rhs.isFinal && lhs.finishReason == rhs.finishReason && lhs.unknownFields == rhs.unknownFields - } - - func isEqualTo(message: any SwiftProtobuf.Message) -> Bool { - guard let other = message as? Self else { return false } - return self == other - } -} - -struct Appleintelligence_HealthRequest: Sendable, SwiftProtobuf.Message { - static let protoMessageName: String = "appleintelligence.HealthRequest" - - var unknownFields = SwiftProtobuf.UnknownStorage() - - init() {} - - mutating func decodeMessage(decoder: inout D) throws { - while let _ = try decoder.nextFieldNumber() {} - } - - func traverse(visitor: inout V) throws { - try unknownFields.traverse(visitor: &visitor) - } - - static func ==(lhs: Self, rhs: Self) -> Bool { - lhs.unknownFields == rhs.unknownFields - } - - func isEqualTo(message: any SwiftProtobuf.Message) -> Bool { - guard let other = message as? Self else { return false } - return self == other - } -} - -struct Appleintelligence_HealthResponse: Sendable, SwiftProtobuf.Message { - static let protoMessageName: String = "appleintelligence.HealthResponse" - - var healthy: Bool = false - var modelStatus: String = "" - - var unknownFields = SwiftProtobuf.UnknownStorage() - - init() {} - - mutating func decodeMessage(decoder: inout D) throws { - while let fieldNumber = try decoder.nextFieldNumber() { - switch fieldNumber { - case 1: try decoder.decodeSingularBoolField(value: &healthy) - case 2: try decoder.decodeSingularStringField(value: &modelStatus) - default: break - } - } - } - - func traverse(visitor: inout V) throws { - if healthy { - try visitor.visitSingularBoolField(value: healthy, fieldNumber: 1) - } - if !modelStatus.isEmpty { - try visitor.visitSingularStringField(value: modelStatus, fieldNumber: 2) - } - try unknownFields.traverse(visitor: &visitor) - } - - static func ==(lhs: Self, rhs: Self) -> Bool { - lhs.healthy == rhs.healthy && lhs.modelStatus == rhs.modelStatus && lhs.unknownFields == rhs.unknownFields - } - - func isEqualTo(message: any SwiftProtobuf.Message) -> Bool { - guard let other = message as? Self else { return false } - return self == other - } -} diff --git a/Sources/AppleIntelligenceCore/Generated/apple_intelligence.grpc.swift b/Sources/AppleIntelligenceCore/Generated/apple_intelligence.grpc.swift new file mode 100644 index 0000000..b3a4668 --- /dev/null +++ b/Sources/AppleIntelligenceCore/Generated/apple_intelligence.grpc.swift @@ -0,0 +1,799 @@ +// DO NOT EDIT. +// swift-format-ignore-file +// swiftlint:disable all +// +// Generated by the gRPC Swift generator plugin for the protocol buffer compiler. +// Source: apple_intelligence.proto +// +// For information on using the generated types, please see the documentation: +// https://github.com/grpc/grpc-swift + +import GRPCCore +import GRPCProtobuf + +// MARK: - appleintelligence.AppleIntelligenceService + +/// Namespace containing generated types for the "appleintelligence.AppleIntelligenceService" service. +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +public enum Appleintelligence_AppleIntelligenceService: Sendable { + /// Service descriptor for the "appleintelligence.AppleIntelligenceService" service. + public static let descriptor = GRPCCore.ServiceDescriptor(fullyQualifiedService: "appleintelligence.AppleIntelligenceService") + /// Namespace for method metadata. + public enum Method: Sendable { + /// Namespace for "Complete" metadata. + public enum Complete: Sendable { + /// Request type for "Complete". + public typealias Input = Appleintelligence_CompletionRequest + /// Response type for "Complete". + public typealias Output = Appleintelligence_CompletionResponse + /// Descriptor for "Complete". + public static let descriptor = GRPCCore.MethodDescriptor( + service: GRPCCore.ServiceDescriptor(fullyQualifiedService: "appleintelligence.AppleIntelligenceService"), + method: "Complete" + ) + } + /// Namespace for "StreamComplete" metadata. + public enum StreamComplete: Sendable { + /// Request type for "StreamComplete". + public typealias Input = Appleintelligence_CompletionRequest + /// Response type for "StreamComplete". + public typealias Output = Appleintelligence_CompletionChunk + /// Descriptor for "StreamComplete". + public static let descriptor = GRPCCore.MethodDescriptor( + service: GRPCCore.ServiceDescriptor(fullyQualifiedService: "appleintelligence.AppleIntelligenceService"), + method: "StreamComplete" + ) + } + /// Namespace for "Health" metadata. + public enum Health: Sendable { + /// Request type for "Health". + public typealias Input = Appleintelligence_HealthRequest + /// Response type for "Health". + public typealias Output = Appleintelligence_HealthResponse + /// Descriptor for "Health". + public static let descriptor = GRPCCore.MethodDescriptor( + service: GRPCCore.ServiceDescriptor(fullyQualifiedService: "appleintelligence.AppleIntelligenceService"), + method: "Health" + ) + } + /// Descriptors for all methods in the "appleintelligence.AppleIntelligenceService" service. + public static let descriptors: [GRPCCore.MethodDescriptor] = [ + Complete.descriptor, + StreamComplete.descriptor, + Health.descriptor + ] + } +} + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension GRPCCore.ServiceDescriptor { + /// Service descriptor for the "appleintelligence.AppleIntelligenceService" service. + public static let appleintelligence_AppleIntelligenceService = GRPCCore.ServiceDescriptor(fullyQualifiedService: "appleintelligence.AppleIntelligenceService") +} + +// MARK: appleintelligence.AppleIntelligenceService (server) + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService { + /// Streaming variant of the service protocol for the "appleintelligence.AppleIntelligenceService" service. + /// + /// This protocol is the lowest-level of the service protocols generated for this service + /// giving you the most flexibility over the implementation of your service. This comes at + /// the cost of more verbose and less strict APIs. Each RPC requires you to implement it in + /// terms of a request stream and response stream. Where only a single request or response + /// message is expected, you are responsible for enforcing this invariant is maintained. + /// + /// Where possible, prefer using the stricter, less-verbose ``ServiceProtocol`` + /// or ``SimpleServiceProtocol`` instead. + /// + /// > Source IDL Documentation: + /// > + /// > Apple Intelligence Service + public protocol StreamingServiceProtocol: GRPCCore.RegistrableRPCService { + /// Handle the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - request: A streaming request of `Appleintelligence_CompletionRequest` messages. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A streaming response of `Appleintelligence_CompletionResponse` messages. + func complete( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse + + /// Handle the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - request: A streaming request of `Appleintelligence_CompletionRequest` messages. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A streaming response of `Appleintelligence_CompletionChunk` messages. + func streamComplete( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse + + /// Handle the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - request: A streaming request of `Appleintelligence_HealthRequest` messages. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A streaming response of `Appleintelligence_HealthResponse` messages. + func health( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse + } + + /// Service protocol for the "appleintelligence.AppleIntelligenceService" service. + /// + /// This protocol is higher level than ``StreamingServiceProtocol`` but lower level than + /// the ``SimpleServiceProtocol``, it provides access to request and response metadata and + /// trailing response metadata. If you don't need these then consider using + /// the ``SimpleServiceProtocol``. If you need fine grained control over your RPCs then + /// use ``StreamingServiceProtocol``. + /// + /// > Source IDL Documentation: + /// > + /// > Apple Intelligence Service + public protocol ServiceProtocol: Appleintelligence_AppleIntelligenceService.StreamingServiceProtocol { + /// Handle the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A response containing a single `Appleintelligence_CompletionResponse` message. + func complete( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse + + /// Handle the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A streaming response of `Appleintelligence_CompletionChunk` messages. + func streamComplete( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse + + /// Handle the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_HealthRequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A response containing a single `Appleintelligence_HealthResponse` message. + func health( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse + } + + /// Simple service protocol for the "appleintelligence.AppleIntelligenceService" service. + /// + /// This is the highest level protocol for the service. The API is the easiest to use but + /// doesn't provide access to request or response metadata. If you need access to these + /// then use ``ServiceProtocol`` instead. + /// + /// > Source IDL Documentation: + /// > + /// > Apple Intelligence Service + public protocol SimpleServiceProtocol: Appleintelligence_AppleIntelligenceService.ServiceProtocol { + /// Handle the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - request: A `Appleintelligence_CompletionRequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A `Appleintelligence_CompletionResponse` to respond with. + func complete( + request: Appleintelligence_CompletionRequest, + context: GRPCCore.ServerContext + ) async throws -> Appleintelligence_CompletionResponse + + /// Handle the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - request: A `Appleintelligence_CompletionRequest` message. + /// - response: A response stream of `Appleintelligence_CompletionChunk` messages. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + func streamComplete( + request: Appleintelligence_CompletionRequest, + response: GRPCCore.RPCWriter, + context: GRPCCore.ServerContext + ) async throws + + /// Handle the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - request: A `Appleintelligence_HealthRequest` message. + /// - context: Context providing information about the RPC. + /// - Throws: Any error which occurred during the processing of the request. Thrown errors + /// of type `RPCError` are mapped to appropriate statuses. All other errors are converted + /// to an internal error. + /// - Returns: A `Appleintelligence_HealthResponse` to respond with. + func health( + request: Appleintelligence_HealthRequest, + context: GRPCCore.ServerContext + ) async throws -> Appleintelligence_HealthResponse + } +} + +// Default implementation of 'registerMethods(with:)'. +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService.StreamingServiceProtocol { + public func registerMethods(with router: inout GRPCCore.RPCRouter) where Transport: GRPCCore.ServerTransport { + router.registerHandler( + forMethod: Appleintelligence_AppleIntelligenceService.Method.Complete.descriptor, + deserializer: GRPCProtobuf.ProtobufDeserializer(), + serializer: GRPCProtobuf.ProtobufSerializer(), + handler: { request, context in + try await self.complete( + request: request, + context: context + ) + } + ) + router.registerHandler( + forMethod: Appleintelligence_AppleIntelligenceService.Method.StreamComplete.descriptor, + deserializer: GRPCProtobuf.ProtobufDeserializer(), + serializer: GRPCProtobuf.ProtobufSerializer(), + handler: { request, context in + try await self.streamComplete( + request: request, + context: context + ) + } + ) + router.registerHandler( + forMethod: Appleintelligence_AppleIntelligenceService.Method.Health.descriptor, + deserializer: GRPCProtobuf.ProtobufDeserializer(), + serializer: GRPCProtobuf.ProtobufSerializer(), + handler: { request, context in + try await self.health( + request: request, + context: context + ) + } + ) + } +} + +// Default implementation of streaming methods from 'StreamingServiceProtocol'. +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService.ServiceProtocol { + public func complete( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + let response = try await self.complete( + request: GRPCCore.ServerRequest(stream: request), + context: context + ) + return GRPCCore.StreamingServerResponse(single: response) + } + + public func streamComplete( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + let response = try await self.streamComplete( + request: GRPCCore.ServerRequest(stream: request), + context: context + ) + return response + } + + public func health( + request: GRPCCore.StreamingServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + let response = try await self.health( + request: GRPCCore.ServerRequest(stream: request), + context: context + ) + return GRPCCore.StreamingServerResponse(single: response) + } +} + +// Default implementation of methods from 'ServiceProtocol'. +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService.SimpleServiceProtocol { + public func complete( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse { + return GRPCCore.ServerResponse( + message: try await self.complete( + request: request.message, + context: context + ), + metadata: [:] + ) + } + + public func streamComplete( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + return GRPCCore.StreamingServerResponse( + metadata: [:], + producer: { writer in + try await self.streamComplete( + request: request.message, + response: writer, + context: context + ) + return [:] + } + ) + } + + public func health( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse { + return GRPCCore.ServerResponse( + message: try await self.health( + request: request.message, + context: context + ), + metadata: [:] + ) + } +} + +// MARK: appleintelligence.AppleIntelligenceService (client) + +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService { + /// Generated client protocol for the "appleintelligence.AppleIntelligenceService" service. + /// + /// You don't need to implement this protocol directly, use the generated + /// implementation, ``Client``. + /// + /// > Source IDL Documentation: + /// > + /// > Apple Intelligence Service + public protocol ClientProtocol: Sendable { + /// Call the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - serializer: A serializer for `Appleintelligence_CompletionRequest` messages. + /// - deserializer: A deserializer for `Appleintelligence_CompletionResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + func complete( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable + + /// Call the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - serializer: A serializer for `Appleintelligence_CompletionRequest` messages. + /// - deserializer: A deserializer for `Appleintelligence_CompletionChunk` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + func streamComplete( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions, + onResponse handleResponse: @Sendable @escaping (GRPCCore.StreamingClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable + + /// Call the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_HealthRequest` message. + /// - serializer: A serializer for `Appleintelligence_HealthRequest` messages. + /// - deserializer: A deserializer for `Appleintelligence_HealthResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + func health( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable + } + + /// Generated client for the "appleintelligence.AppleIntelligenceService" service. + /// + /// The ``Client`` provides an implementation of ``ClientProtocol`` which wraps + /// a `GRPCCore.GRPCCClient`. The underlying `GRPCClient` provides the long-lived + /// means of communication with the remote peer. + /// + /// > Source IDL Documentation: + /// > + /// > Apple Intelligence Service + public struct Client: ClientProtocol where Transport: GRPCCore.ClientTransport { + private let client: GRPCCore.GRPCClient + + /// Creates a new client wrapping the provided `GRPCCore.GRPCClient`. + /// + /// - Parameters: + /// - client: A `GRPCCore.GRPCClient` providing a communication channel to the service. + public init(wrapping client: GRPCCore.GRPCClient) { + self.client = client + } + + /// Call the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - serializer: A serializer for `Appleintelligence_CompletionRequest` messages. + /// - deserializer: A deserializer for `Appleintelligence_CompletionResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func complete( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.client.unary( + request: request, + descriptor: Appleintelligence_AppleIntelligenceService.Method.Complete.descriptor, + serializer: serializer, + deserializer: deserializer, + options: options, + onResponse: handleResponse + ) + } + + /// Call the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - serializer: A serializer for `Appleintelligence_CompletionRequest` messages. + /// - deserializer: A deserializer for `Appleintelligence_CompletionChunk` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func streamComplete( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.StreamingClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable { + try await self.client.serverStreaming( + request: request, + descriptor: Appleintelligence_AppleIntelligenceService.Method.StreamComplete.descriptor, + serializer: serializer, + deserializer: deserializer, + options: options, + onResponse: handleResponse + ) + } + + /// Call the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_HealthRequest` message. + /// - serializer: A serializer for `Appleintelligence_HealthRequest` messages. + /// - deserializer: A deserializer for `Appleintelligence_HealthResponse` messages. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func health( + request: GRPCCore.ClientRequest, + serializer: some GRPCCore.MessageSerializer, + deserializer: some GRPCCore.MessageDeserializer, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.client.unary( + request: request, + descriptor: Appleintelligence_AppleIntelligenceService.Method.Health.descriptor, + serializer: serializer, + deserializer: deserializer, + options: options, + onResponse: handleResponse + ) + } + } +} + +// Helpers providing default arguments to 'ClientProtocol' methods. +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService.ClientProtocol { + /// Call the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func complete( + request: GRPCCore.ClientRequest, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.complete( + request: request, + serializer: GRPCProtobuf.ProtobufSerializer(), + deserializer: GRPCProtobuf.ProtobufDeserializer(), + options: options, + onResponse: handleResponse + ) + } + + /// Call the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_CompletionRequest` message. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func streamComplete( + request: GRPCCore.ClientRequest, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.StreamingClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable { + try await self.streamComplete( + request: request, + serializer: GRPCProtobuf.ProtobufSerializer(), + deserializer: GRPCProtobuf.ProtobufDeserializer(), + options: options, + onResponse: handleResponse + ) + } + + /// Call the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - request: A request containing a single `Appleintelligence_HealthRequest` message. + /// - options: Options to apply to this RPC. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func health( + request: GRPCCore.ClientRequest, + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + try await self.health( + request: request, + serializer: GRPCProtobuf.ProtobufSerializer(), + deserializer: GRPCProtobuf.ProtobufDeserializer(), + options: options, + onResponse: handleResponse + ) + } +} + +// Helpers providing sugared APIs for 'ClientProtocol' methods. +@available(macOS 15.0, iOS 18.0, watchOS 11.0, tvOS 18.0, visionOS 2.0, *) +extension Appleintelligence_AppleIntelligenceService.ClientProtocol { + /// Call the "Complete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Single completion request + /// + /// - Parameters: + /// - message: request message to send. + /// - metadata: Additional metadata to send, defaults to empty. + /// - options: Options to apply to this RPC, defaults to `.defaults`. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func complete( + _ message: Appleintelligence_CompletionRequest, + metadata: GRPCCore.Metadata = [:], + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + let request = GRPCCore.ClientRequest( + message: message, + metadata: metadata + ) + return try await self.complete( + request: request, + options: options, + onResponse: handleResponse + ) + } + + /// Call the "StreamComplete" method. + /// + /// > Source IDL Documentation: + /// > + /// > Streaming completion request + /// + /// - Parameters: + /// - message: request message to send. + /// - metadata: Additional metadata to send, defaults to empty. + /// - options: Options to apply to this RPC, defaults to `.defaults`. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func streamComplete( + _ message: Appleintelligence_CompletionRequest, + metadata: GRPCCore.Metadata = [:], + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.StreamingClientResponse) async throws -> Result + ) async throws -> Result where Result: Sendable { + let request = GRPCCore.ClientRequest( + message: message, + metadata: metadata + ) + return try await self.streamComplete( + request: request, + options: options, + onResponse: handleResponse + ) + } + + /// Call the "Health" method. + /// + /// > Source IDL Documentation: + /// > + /// > Health check + /// + /// - Parameters: + /// - message: request message to send. + /// - metadata: Additional metadata to send, defaults to empty. + /// - options: Options to apply to this RPC, defaults to `.defaults`. + /// - handleResponse: A closure which handles the response, the result of which is + /// returned to the caller. Returning from the closure will cancel the RPC if it + /// hasn't already finished. + /// - Returns: The result of `handleResponse`. + public func health( + _ message: Appleintelligence_HealthRequest, + metadata: GRPCCore.Metadata = [:], + options: GRPCCore.CallOptions = .defaults, + onResponse handleResponse: @Sendable @escaping (GRPCCore.ClientResponse) async throws -> Result = { response in + try response.message + } + ) async throws -> Result where Result: Sendable { + let request = GRPCCore.ClientRequest( + message: message, + metadata: metadata + ) + return try await self.health( + request: request, + options: options, + onResponse: handleResponse + ) + } +} \ No newline at end of file diff --git a/Sources/AppleIntelligenceCore/Generated/apple_intelligence.pb.swift b/Sources/AppleIntelligenceCore/Generated/apple_intelligence.pb.swift new file mode 100644 index 0000000..f3d512a --- /dev/null +++ b/Sources/AppleIntelligenceCore/Generated/apple_intelligence.pb.swift @@ -0,0 +1,447 @@ +// DO NOT EDIT. +// swift-format-ignore-file +// swiftlint:disable all +// +// Generated by the Swift generator plugin for the protocol buffer compiler. +// Source: apple_intelligence.proto +// +// For information on using the generated types, please see the documentation: +// https://github.com/apple/swift-protobuf/ + +import Foundation +import SwiftProtobuf + +// If the compiler emits an error on this type, it is because this file +// was generated by a version of the `protoc` Swift plug-in that is +// incompatible with the version of SwiftProtobuf to which you are linking. +// Please ensure that you are building against the same version of the API +// that was used to generate this file. +fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { + struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} + typealias Version = _2 +} + +/// Image data for vision requests +public struct Appleintelligence_ImageData: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var data: Data = Data() + + public var filename: String = String() + + public var mimeType: String = String() + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +/// Vision analysis results +public struct Appleintelligence_ImageAnalysis: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var textContent: String = String() + + public var labels: [String] = [] + + public var description_p: String = String() + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +/// Completion request +public struct Appleintelligence_CompletionRequest: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var prompt: String = String() + + public var temperature: Float { + get {return _temperature ?? 0} + set {_temperature = newValue} + } + /// Returns true if `temperature` has been explicitly set. + public var hasTemperature: Bool {return self._temperature != nil} + /// Clears the value of `temperature`. Subsequent reads from it will return its default value. + public mutating func clearTemperature() {self._temperature = nil} + + public var maxTokens: Int32 { + get {return _maxTokens ?? 0} + set {_maxTokens = newValue} + } + /// Returns true if `maxTokens` has been explicitly set. + public var hasMaxTokens: Bool {return self._maxTokens != nil} + /// Clears the value of `maxTokens`. Subsequent reads from it will return its default value. + public mutating func clearMaxTokens() {self._maxTokens = nil} + + public var images: [Appleintelligence_ImageData] = [] + + public var includeAnalysis: Bool = false + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} + + fileprivate var _temperature: Float? = nil + fileprivate var _maxTokens: Int32? = nil +} + +/// Completion response (non-streaming) +public struct Appleintelligence_CompletionResponse: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var id: String = String() + + public var text: String = String() + + public var finishReason: String = String() + + public var imageAnalyses: [Appleintelligence_ImageAnalysis] = [] + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +/// Streaming completion chunk +public struct Appleintelligence_CompletionChunk: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var id: String = String() + + public var delta: String = String() + + public var isFinal: Bool = false + + public var finishReason: String = String() + + public var imageAnalyses: [Appleintelligence_ImageAnalysis] = [] + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +/// Health check request +public struct Appleintelligence_HealthRequest: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +/// Health check response +public struct Appleintelligence_HealthResponse: Sendable { + // SwiftProtobuf.Message conformance is added in an extension below. See the + // `Message` and `Message+*Additions` files in the SwiftProtobuf library for + // methods supported on all messages. + + public var healthy: Bool = false + + public var modelStatus: String = String() + + public var unknownFields = SwiftProtobuf.UnknownStorage() + + public init() {} +} + +// MARK: - Code below here is support for the SwiftProtobuf runtime. + +fileprivate let _protobuf_package = "appleintelligence" + +extension Appleintelligence_ImageData: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".ImageData" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}data\0\u{1}filename\0\u{3}mime_type\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularBytesField(value: &self.data) }() + case 2: try { try decoder.decodeSingularStringField(value: &self.filename) }() + case 3: try { try decoder.decodeSingularStringField(value: &self.mimeType) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + if !self.data.isEmpty { + try visitor.visitSingularBytesField(value: self.data, fieldNumber: 1) + } + if !self.filename.isEmpty { + try visitor.visitSingularStringField(value: self.filename, fieldNumber: 2) + } + if !self.mimeType.isEmpty { + try visitor.visitSingularStringField(value: self.mimeType, fieldNumber: 3) + } + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_ImageData, rhs: Appleintelligence_ImageData) -> Bool { + if lhs.data != rhs.data {return false} + if lhs.filename != rhs.filename {return false} + if lhs.mimeType != rhs.mimeType {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +extension Appleintelligence_ImageAnalysis: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".ImageAnalysis" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{3}text_content\0\u{1}labels\0\u{1}description\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularStringField(value: &self.textContent) }() + case 2: try { try decoder.decodeRepeatedStringField(value: &self.labels) }() + case 3: try { try decoder.decodeSingularStringField(value: &self.description_p) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + if !self.textContent.isEmpty { + try visitor.visitSingularStringField(value: self.textContent, fieldNumber: 1) + } + if !self.labels.isEmpty { + try visitor.visitRepeatedStringField(value: self.labels, fieldNumber: 2) + } + if !self.description_p.isEmpty { + try visitor.visitSingularStringField(value: self.description_p, fieldNumber: 3) + } + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_ImageAnalysis, rhs: Appleintelligence_ImageAnalysis) -> Bool { + if lhs.textContent != rhs.textContent {return false} + if lhs.labels != rhs.labels {return false} + if lhs.description_p != rhs.description_p {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +extension Appleintelligence_CompletionRequest: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".CompletionRequest" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}prompt\0\u{1}temperature\0\u{3}max_tokens\0\u{1}images\0\u{3}include_analysis\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularStringField(value: &self.prompt) }() + case 2: try { try decoder.decodeSingularFloatField(value: &self._temperature) }() + case 3: try { try decoder.decodeSingularInt32Field(value: &self._maxTokens) }() + case 4: try { try decoder.decodeRepeatedMessageField(value: &self.images) }() + case 5: try { try decoder.decodeSingularBoolField(value: &self.includeAnalysis) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every if/case branch local when no optimizations + // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and + // https://github.com/apple/swift-protobuf/issues/1182 + if !self.prompt.isEmpty { + try visitor.visitSingularStringField(value: self.prompt, fieldNumber: 1) + } + try { if let v = self._temperature { + try visitor.visitSingularFloatField(value: v, fieldNumber: 2) + } }() + try { if let v = self._maxTokens { + try visitor.visitSingularInt32Field(value: v, fieldNumber: 3) + } }() + if !self.images.isEmpty { + try visitor.visitRepeatedMessageField(value: self.images, fieldNumber: 4) + } + if self.includeAnalysis != false { + try visitor.visitSingularBoolField(value: self.includeAnalysis, fieldNumber: 5) + } + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_CompletionRequest, rhs: Appleintelligence_CompletionRequest) -> Bool { + if lhs.prompt != rhs.prompt {return false} + if lhs._temperature != rhs._temperature {return false} + if lhs._maxTokens != rhs._maxTokens {return false} + if lhs.images != rhs.images {return false} + if lhs.includeAnalysis != rhs.includeAnalysis {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +extension Appleintelligence_CompletionResponse: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".CompletionResponse" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}id\0\u{1}text\0\u{3}finish_reason\0\u{3}image_analyses\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularStringField(value: &self.id) }() + case 2: try { try decoder.decodeSingularStringField(value: &self.text) }() + case 3: try { try decoder.decodeSingularStringField(value: &self.finishReason) }() + case 4: try { try decoder.decodeRepeatedMessageField(value: &self.imageAnalyses) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + if !self.id.isEmpty { + try visitor.visitSingularStringField(value: self.id, fieldNumber: 1) + } + if !self.text.isEmpty { + try visitor.visitSingularStringField(value: self.text, fieldNumber: 2) + } + if !self.finishReason.isEmpty { + try visitor.visitSingularStringField(value: self.finishReason, fieldNumber: 3) + } + if !self.imageAnalyses.isEmpty { + try visitor.visitRepeatedMessageField(value: self.imageAnalyses, fieldNumber: 4) + } + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_CompletionResponse, rhs: Appleintelligence_CompletionResponse) -> Bool { + if lhs.id != rhs.id {return false} + if lhs.text != rhs.text {return false} + if lhs.finishReason != rhs.finishReason {return false} + if lhs.imageAnalyses != rhs.imageAnalyses {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +extension Appleintelligence_CompletionChunk: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".CompletionChunk" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}id\0\u{1}delta\0\u{3}is_final\0\u{3}finish_reason\0\u{3}image_analyses\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularStringField(value: &self.id) }() + case 2: try { try decoder.decodeSingularStringField(value: &self.delta) }() + case 3: try { try decoder.decodeSingularBoolField(value: &self.isFinal) }() + case 4: try { try decoder.decodeSingularStringField(value: &self.finishReason) }() + case 5: try { try decoder.decodeRepeatedMessageField(value: &self.imageAnalyses) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + if !self.id.isEmpty { + try visitor.visitSingularStringField(value: self.id, fieldNumber: 1) + } + if !self.delta.isEmpty { + try visitor.visitSingularStringField(value: self.delta, fieldNumber: 2) + } + if self.isFinal != false { + try visitor.visitSingularBoolField(value: self.isFinal, fieldNumber: 3) + } + if !self.finishReason.isEmpty { + try visitor.visitSingularStringField(value: self.finishReason, fieldNumber: 4) + } + if !self.imageAnalyses.isEmpty { + try visitor.visitRepeatedMessageField(value: self.imageAnalyses, fieldNumber: 5) + } + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_CompletionChunk, rhs: Appleintelligence_CompletionChunk) -> Bool { + if lhs.id != rhs.id {return false} + if lhs.delta != rhs.delta {return false} + if lhs.isFinal != rhs.isFinal {return false} + if lhs.finishReason != rhs.finishReason {return false} + if lhs.imageAnalyses != rhs.imageAnalyses {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +extension Appleintelligence_HealthRequest: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".HealthRequest" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap() + + public mutating func decodeMessage(decoder: inout D) throws { + // Load everything into unknown fields + while try decoder.nextFieldNumber() != nil {} + } + + public func traverse(visitor: inout V) throws { + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_HealthRequest, rhs: Appleintelligence_HealthRequest) -> Bool { + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} + +extension Appleintelligence_HealthResponse: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { + public static let protoMessageName: String = _protobuf_package + ".HealthResponse" + public static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}healthy\0\u{3}model_status\0") + + public mutating func decodeMessage(decoder: inout D) throws { + while let fieldNumber = try decoder.nextFieldNumber() { + // The use of inline closures is to circumvent an issue where the compiler + // allocates stack space for every case branch when no optimizations are + // enabled. https://github.com/apple/swift-protobuf/issues/1034 + switch fieldNumber { + case 1: try { try decoder.decodeSingularBoolField(value: &self.healthy) }() + case 2: try { try decoder.decodeSingularStringField(value: &self.modelStatus) }() + default: break + } + } + } + + public func traverse(visitor: inout V) throws { + if self.healthy != false { + try visitor.visitSingularBoolField(value: self.healthy, fieldNumber: 1) + } + if !self.modelStatus.isEmpty { + try visitor.visitSingularStringField(value: self.modelStatus, fieldNumber: 2) + } + try unknownFields.traverse(visitor: &visitor) + } + + public static func ==(lhs: Appleintelligence_HealthResponse, rhs: Appleintelligence_HealthResponse) -> Bool { + if lhs.healthy != rhs.healthy {return false} + if lhs.modelStatus != rhs.modelStatus {return false} + if lhs.unknownFields != rhs.unknownFields {return false} + return true + } +} diff --git a/Sources/AppleIntelligenceCore/Providers/AppleIntelligenceProvider.swift b/Sources/AppleIntelligenceCore/Providers/AppleIntelligenceProvider.swift index f0b888d..742f5d0 100644 --- a/Sources/AppleIntelligenceCore/Providers/AppleIntelligenceProvider.swift +++ b/Sources/AppleIntelligenceCore/Providers/AppleIntelligenceProvider.swift @@ -4,28 +4,7 @@ import GRPCProtobuf import GRPCNIOTransportHTTP2 /// gRPC service provider for Apple Intelligence -public struct AppleIntelligenceProvider: RegistrableRPCService { - /// Service descriptor - public static let serviceDescriptor = ServiceDescriptor( - fullyQualifiedService: "appleintelligence.AppleIntelligence" - ) - - /// Method descriptors - enum Methods { - static let complete = MethodDescriptor( - service: AppleIntelligenceProvider.serviceDescriptor, - method: "Complete" - ) - static let streamComplete = MethodDescriptor( - service: AppleIntelligenceProvider.serviceDescriptor, - method: "StreamComplete" - ) - static let health = MethodDescriptor( - service: AppleIntelligenceProvider.serviceDescriptor, - method: "Health" - ) - } - +public struct AppleIntelligenceProvider: Appleintelligence_AppleIntelligenceService.ServiceProtocol { /// The underlying AI service private let service: AppleIntelligenceService @@ -37,123 +16,131 @@ public struct AppleIntelligenceProvider: RegistrableRPCService { self.apiKey = apiKey } - public func registerMethods(with router: inout RPCRouter) { - // Register Complete method (unary) - router.registerHandler( - forMethod: Methods.complete, - deserializer: ProtobufDeserializer(), - serializer: ProtobufSerializer() - ) { request, context in - try self.validateApiKey(metadata: request.metadata) + // MARK: - ServiceProtocol Implementation - // Collect the single message from the request stream - var requestMessage: Appleintelligence_CompletionRequest? - for try await message in request.messages { - requestMessage = message - break + public func complete( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse { + try validateApiKey(metadata: request.metadata) + + let message = request.message + + // Convert protobuf images to service format + let images = message.images.map { img in + (data: img.data, filename: img.filename.isEmpty ? nil : img.filename) + } + + let (text, analyses) = try await service.complete( + prompt: message.prompt, + temperature: message.hasTemperature ? message.temperature : nil, + maxTokens: message.hasMaxTokens ? Int(message.maxTokens) : nil, + images: images + ) + + var response = Appleintelligence_CompletionResponse() + response.id = UUID().uuidString + response.text = text + response.finishReason = "stop" + + // Include analysis results if requested + if message.includeAnalysis { + response.imageAnalyses = analyses.map { analysis in + var protoAnalysis = Appleintelligence_ImageAnalysis() + protoAnalysis.textContent = analysis.textContent + protoAnalysis.labels = analysis.labels + protoAnalysis.description_p = analysis.description + return protoAnalysis } + } - guard let message = requestMessage else { - throw RPCError(code: .invalidArgument, message: "No request message received") - } + return ServerResponse(message: response) + } - let text = try await self.service.complete( + public func streamComplete( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.StreamingServerResponse { + try validateApiKey(metadata: request.metadata) + + let message = request.message + let completionId = UUID().uuidString + + // Convert protobuf images to service format + let images = message.images.map { img in + (data: img.data, filename: img.filename.isEmpty ? nil : img.filename) + } + + return StreamingServerResponse { writer in + let stream = await self.service.streamComplete( prompt: message.prompt, temperature: message.hasTemperature ? message.temperature : nil, - maxTokens: message.hasMaxTokens ? Int(message.maxTokens) : nil + maxTokens: message.hasMaxTokens ? Int(message.maxTokens) : nil, + images: images ) - var response = Appleintelligence_CompletionResponse() - response.id = UUID().uuidString - response.text = text - response.finishReason = "stop" - - return StreamingServerResponse(single: ServerResponse(message: response)) - } - - // Register StreamComplete method (server streaming) - router.registerHandler( - forMethod: Methods.streamComplete, - deserializer: ProtobufDeserializer(), - serializer: ProtobufSerializer() - ) { request, context in - try self.validateApiKey(metadata: request.metadata) - - // Collect the single message from the request stream - var requestMessage: Appleintelligence_CompletionRequest? - for try await message in request.messages { - requestMessage = message - break - } - - guard let message = requestMessage else { - throw RPCError(code: .invalidArgument, message: "No request message received") - } - - let completionId = UUID().uuidString - let prompt = message.prompt - let temperature = message.hasTemperature ? message.temperature : nil - let maxTokens = message.hasMaxTokens ? Int(message.maxTokens) : nil - - return StreamingServerResponse { writer in - let stream = await self.service.streamComplete( - prompt: prompt, - temperature: temperature, - maxTokens: maxTokens - ) - - var lastContent = "" - for try await partialResponse in stream { - // Calculate the delta (new text since last response) - let delta: String - if partialResponse.hasPrefix(lastContent) { - delta = String(partialResponse.dropFirst(lastContent.count)) - } else { - delta = partialResponse - } - lastContent = partialResponse - - if !delta.isEmpty { - var chunk = Appleintelligence_CompletionChunk() - chunk.id = completionId - chunk.delta = delta - chunk.isFinal = false - try await writer.write(chunk) - } + var lastContent = "" + var isFirstChunk = true + for try await (partialResponse, analyses) in stream { + // Calculate the delta (new text since last response) + let delta: String + if partialResponse.hasPrefix(lastContent) { + delta = String(partialResponse.dropFirst(lastContent.count)) + } else { + delta = partialResponse } + lastContent = partialResponse - // Send final chunk - var finalChunk = Appleintelligence_CompletionChunk() - finalChunk.id = completionId - finalChunk.delta = "" - finalChunk.isFinal = true - finalChunk.finishReason = "stop" - try await writer.write(finalChunk) + if !delta.isEmpty || isFirstChunk { + var chunk = Appleintelligence_CompletionChunk() + chunk.id = completionId + chunk.delta = delta + chunk.isFinal = false - return [:] + // Include analyses in first chunk if requested + if isFirstChunk && message.includeAnalysis, let analyses = analyses { + chunk.imageAnalyses = analyses.map { analysis in + var protoAnalysis = Appleintelligence_ImageAnalysis() + protoAnalysis.textContent = analysis.textContent + protoAnalysis.labels = analysis.labels + protoAnalysis.description_p = analysis.description + return protoAnalysis + } + } + + try await writer.write(chunk) + isFirstChunk = false + } } - } - // Register Health method (unary) - router.registerHandler( - forMethod: Methods.health, - deserializer: ProtobufDeserializer(), - serializer: ProtobufSerializer() - ) { request, context in - // Consume request messages (empty for health check) - for try await _ in request.messages {} + // Send final chunk + var finalChunk = Appleintelligence_CompletionChunk() + finalChunk.id = completionId + finalChunk.delta = "" + finalChunk.isFinal = true + finalChunk.finishReason = "stop" + try await writer.write(finalChunk) - let isHealthy = await self.service.isAvailable - let modelStatus = await self.service.getModelStatus() - - var response = Appleintelligence_HealthResponse() - response.healthy = isHealthy - response.modelStatus = modelStatus - - return StreamingServerResponse(single: ServerResponse(message: response)) + return [:] } } + public func health( + request: GRPCCore.ServerRequest, + context: GRPCCore.ServerContext + ) async throws -> GRPCCore.ServerResponse { + let isHealthy = await service.isAvailable + let modelStatus = await service.getModelStatus() + + var response = Appleintelligence_HealthResponse() + response.healthy = isHealthy + response.modelStatus = modelStatus + + return ServerResponse(message: response) + } + + // MARK: - Private Helpers + /// Validate API key if configured private func validateApiKey(metadata: Metadata) throws { guard let expectedKey = apiKey else { diff --git a/Sources/AppleIntelligenceCore/Resources.swift b/Sources/AppleIntelligenceCore/Resources.swift new file mode 100644 index 0000000..5da4eb3 --- /dev/null +++ b/Sources/AppleIntelligenceCore/Resources.swift @@ -0,0 +1,9 @@ +import Foundation + +/// Helper for accessing bundled resources +public enum AppleIntelligenceResources { + /// URL to the protobuf descriptor set file for reflection + public static var descriptorSetURL: URL? { + Bundle.module.url(forResource: "apple_intelligence", withExtension: "pb") + } +} diff --git a/Sources/AppleIntelligenceCore/Resources/apple_intelligence.pb b/Sources/AppleIntelligenceCore/Resources/apple_intelligence.pb new file mode 100644 index 0000000..eacd668 Binary files /dev/null and b/Sources/AppleIntelligenceCore/Resources/apple_intelligence.pb differ diff --git a/Sources/AppleIntelligenceCore/Services/AppleIntelligenceService.swift b/Sources/AppleIntelligenceCore/Services/AppleIntelligenceService.swift index 95c2043..d925eea 100644 --- a/Sources/AppleIntelligenceCore/Services/AppleIntelligenceService.swift +++ b/Sources/AppleIntelligenceCore/Services/AppleIntelligenceService.swift @@ -6,6 +6,7 @@ public enum AppleIntelligenceError: Error, CustomStringConvertible, Sendable { case modelNotAvailable case generationFailed(String) case sessionCreationFailed + case imageAnalysisFailed(String) public var description: String { switch self { @@ -15,6 +16,8 @@ public enum AppleIntelligenceError: Error, CustomStringConvertible, Sendable { return "Generation failed: \(reason)" case .sessionCreationFailed: return "Failed to create language model session" + case .imageAnalysisFailed(let reason): + return "Image analysis failed: \(reason)" } } } @@ -24,6 +27,9 @@ public actor AppleIntelligenceService { /// The language model session private var session: LanguageModelSession? + /// Vision analysis service for image processing + private let visionService = VisionAnalysisService() + /// Whether the model is available public private(set) var isAvailable: Bool = false @@ -60,21 +66,42 @@ public actor AppleIntelligenceService { } /// Generate a completion for the given prompt (non-streaming) - public func complete(prompt: String, temperature: Float?, maxTokens: Int?) async throws -> String { + public func complete( + prompt: String, + temperature: Float?, + maxTokens: Int?, + images: [(data: Data, filename: String?)] = [] + ) async throws -> (text: String, analyses: [VisionAnalysisResult]) { guard isAvailable, let session = session else { throw AppleIntelligenceError.modelNotAvailable } - let response = try await session.respond(to: prompt) - return response.content + // Analyze images if provided + var analyses: [VisionAnalysisResult] = [] + var enhancedPrompt = prompt + + if !images.isEmpty { + do { + analyses = try await visionService.analyzeMultiple(images: images) + let analysesWithFilenames = zip(analyses, images).map { (result: $0.0, filename: $0.1.filename) } + let context = await visionService.formatAnalysesAsPromptContext(analyses: analysesWithFilenames) + enhancedPrompt = context + "\n\n" + prompt + } catch { + throw AppleIntelligenceError.imageAnalysisFailed(error.localizedDescription) + } + } + + let response = try await session.respond(to: enhancedPrompt) + return (text: response.content, analyses: analyses) } /// Generate a streaming completion for the given prompt public func streamComplete( prompt: String, temperature: Float?, - maxTokens: Int? - ) -> AsyncThrowingStream { + maxTokens: Int?, + images: [(data: Data, filename: String?)] = [] + ) -> AsyncThrowingStream<(text: String, analyses: [VisionAnalysisResult]?), Error> { AsyncThrowingStream { continuation in Task { guard self.isAvailable, let session = self.session else { @@ -82,10 +109,33 @@ public actor AppleIntelligenceService { return } + // Analyze images first if provided + var analyses: [VisionAnalysisResult] = [] + var enhancedPrompt = prompt + + if !images.isEmpty { + do { + analyses = try await self.visionService.analyzeMultiple(images: images) + let analysesWithFilenames = zip(analyses, images).map { (result: $0.0, filename: $0.1.filename) } + let context = await self.visionService.formatAnalysesAsPromptContext(analyses: analysesWithFilenames) + enhancedPrompt = context + "\n\n" + prompt + } catch { + continuation.finish(throwing: AppleIntelligenceError.imageAnalysisFailed(error.localizedDescription)) + return + } + } + do { - let stream = session.streamResponse(to: prompt) + let stream = session.streamResponse(to: enhancedPrompt) + var isFirst = true for try await partialResponse in stream { - continuation.yield(partialResponse.content) + // Include analyses only in first chunk + if isFirst { + continuation.yield((text: partialResponse.content, analyses: analyses)) + isFirst = false + } else { + continuation.yield((text: partialResponse.content, analyses: nil)) + } } continuation.finish() } catch { diff --git a/Sources/AppleIntelligenceCore/Services/VisionAnalysisService.swift b/Sources/AppleIntelligenceCore/Services/VisionAnalysisService.swift new file mode 100644 index 0000000..b984678 --- /dev/null +++ b/Sources/AppleIntelligenceCore/Services/VisionAnalysisService.swift @@ -0,0 +1,243 @@ +import Foundation +import Vision +import CoreImage + +#if canImport(AppKit) +import AppKit +#endif + +/// Result of Vision framework analysis on an image +public struct VisionAnalysisResult: Sendable { + public let textContent: String + public let labels: [String] + public let description: String + + public init(textContent: String = "", labels: [String] = [], description: String = "") { + self.textContent = textContent + self.labels = labels + self.description = description + } + + /// Format analysis for LLM context + public func formatAsContext(imageIndex: Int, filename: String?) -> String { + var parts: [String] = [] + + let imageName = filename ?? "Image \(imageIndex + 1)" + + if !textContent.isEmpty { + parts.append("Text: \"\(textContent)\"") + } + + if !labels.isEmpty { + parts.append("Objects: \(labels.joined(separator: ", "))") + } + + if parts.isEmpty { + return "\(imageName): No content detected" + } + + return "\(imageName): \(parts.joined(separator: " | "))" + } +} + +/// Errors from Vision analysis +public enum VisionAnalysisError: Error, CustomStringConvertible, Sendable { + case invalidImageData + case analysisFailure(String) + case unsupportedFormat + + public var description: String { + switch self { + case .invalidImageData: + return "Invalid or corrupted image data" + case .analysisFailure(let reason): + return "Vision analysis failed: \(reason)" + case .unsupportedFormat: + return "Unsupported image format" + } + } +} + +/// Service for analyzing images using Apple's Vision framework +public actor VisionAnalysisService { + + /// Configuration for which analyses to perform + public struct AnalysisOptions: Sendable { + public var performOCR: Bool + public var performClassification: Bool + + public init(performOCR: Bool = true, performClassification: Bool = true) { + self.performOCR = performOCR + self.performClassification = performClassification + } + + public static let all = AnalysisOptions() + public static let textOnly = AnalysisOptions(performOCR: true, performClassification: false) + } + + public init() {} + + /// Analyze a single image + public func analyze( + imageData: Data, + options: AnalysisOptions = .all + ) async throws -> VisionAnalysisResult { + guard let cgImage = createCGImage(from: imageData) else { + throw VisionAnalysisError.invalidImageData + } + + var textContent = "" + var labels: [String] = [] + + // Perform OCR + if options.performOCR { + textContent = try await performTextRecognition(on: cgImage) + } + + // Perform image classification + if options.performClassification { + labels = try await performImageClassification(on: cgImage) + } + + // Build description + var descriptionParts: [String] = [] + if !textContent.isEmpty { + let truncatedText = textContent.count > 200 + ? String(textContent.prefix(200)) + "..." + : textContent + descriptionParts.append("Contains text: \"\(truncatedText)\"") + } + if !labels.isEmpty { + descriptionParts.append("Shows: \(labels.prefix(5).joined(separator: ", "))") + } + + let description = descriptionParts.isEmpty + ? "Image with no recognizable content" + : descriptionParts.joined(separator: ". ") + + return VisionAnalysisResult( + textContent: textContent, + labels: labels, + description: description + ) + } + + /// Analyze multiple images + public func analyzeMultiple( + images: [(data: Data, filename: String?)], + options: AnalysisOptions = .all + ) async throws -> [VisionAnalysisResult] { + var results: [VisionAnalysisResult] = [] + + for image in images { + let result = try await analyze(imageData: image.data, options: options) + results.append(result) + } + + return results + } + + /// Format multiple analyses as a combined context string for LLM + public func formatAnalysesAsPromptContext( + analyses: [(result: VisionAnalysisResult, filename: String?)] + ) -> String { + guard !analyses.isEmpty else { return "" } + + var lines: [String] = ["[Image Analysis]"] + + for (index, analysis) in analyses.enumerated() { + lines.append(analysis.result.formatAsContext( + imageIndex: index, + filename: analysis.filename + )) + } + + lines.append("[End Image Analysis]") + + return lines.joined(separator: "\n") + } + + // MARK: - Private Methods + + private func createCGImage(from data: Data) -> CGImage? { + #if canImport(AppKit) + guard let nsImage = NSImage(data: data), + let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) else { + // Try CIImage as fallback + guard let ciImage = CIImage(data: data) else { return nil } + let context = CIContext() + return context.createCGImage(ciImage, from: ciImage.extent) + } + return cgImage + #else + guard let ciImage = CIImage(data: data) else { return nil } + let context = CIContext() + return context.createCGImage(ciImage, from: ciImage.extent) + #endif + } + + private func performTextRecognition(on image: CGImage) async throws -> String { + try await withCheckedThrowingContinuation { continuation in + let request = VNRecognizeTextRequest { request, error in + if let error = error { + continuation.resume(throwing: VisionAnalysisError.analysisFailure(error.localizedDescription)) + return + } + + guard let observations = request.results as? [VNRecognizedTextObservation] else { + continuation.resume(returning: "") + return + } + + let recognizedText = observations.compactMap { observation in + observation.topCandidates(1).first?.string + }.joined(separator: "\n") + + continuation.resume(returning: recognizedText) + } + + request.recognitionLevel = .accurate + request.usesLanguageCorrection = true + + let handler = VNImageRequestHandler(cgImage: image, options: [:]) + + do { + try handler.perform([request]) + } catch { + continuation.resume(throwing: VisionAnalysisError.analysisFailure(error.localizedDescription)) + } + } + } + + private func performImageClassification(on image: CGImage) async throws -> [String] { + try await withCheckedThrowingContinuation { continuation in + let request = VNClassifyImageRequest { request, error in + if let error = error { + continuation.resume(throwing: VisionAnalysisError.analysisFailure(error.localizedDescription)) + return + } + + guard let observations = request.results as? [VNClassificationObservation] else { + continuation.resume(returning: []) + return + } + + // Filter to high-confidence labels and take top 10 + let labels = observations + .filter { $0.confidence > 0.3 } + .prefix(10) + .map { $0.identifier.replacingOccurrences(of: "_", with: " ") } + + continuation.resume(returning: Array(labels)) + } + + let handler = VNImageRequestHandler(cgImage: image, options: [:]) + + do { + try handler.perform([request]) + } catch { + continuation.resume(throwing: VisionAnalysisError.analysisFailure(error.localizedDescription)) + } + } + } +}