Add vision support, gRPC reflection toggle, and chat improvements
- Add Vision framework integration for image analysis (OCR, classification) - Add image attachment support in chat UI with drag & drop - Add recent images sidebar from Downloads/Desktop - Add copy to clipboard button for assistant responses - Add gRPC reflection service with toggle in settings - Create proper .proto file and generate Swift code - Add server restart when toggling reflection setting - Fix port number formatting in settings (remove comma grouping) - Update gRPC dependencies to v2.x 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Mathias Beaulieu-Duncan
@@ -4,28 +4,7 @@ import GRPCProtobuf
|
||||
import GRPCNIOTransportHTTP2
|
||||
|
||||
/// gRPC service provider for Apple Intelligence
|
||||
public struct AppleIntelligenceProvider: RegistrableRPCService {
|
||||
/// Service descriptor
|
||||
public static let serviceDescriptor = ServiceDescriptor(
|
||||
fullyQualifiedService: "appleintelligence.AppleIntelligence"
|
||||
)
|
||||
|
||||
/// Method descriptors
|
||||
enum Methods {
|
||||
static let complete = MethodDescriptor(
|
||||
service: AppleIntelligenceProvider.serviceDescriptor,
|
||||
method: "Complete"
|
||||
)
|
||||
static let streamComplete = MethodDescriptor(
|
||||
service: AppleIntelligenceProvider.serviceDescriptor,
|
||||
method: "StreamComplete"
|
||||
)
|
||||
static let health = MethodDescriptor(
|
||||
service: AppleIntelligenceProvider.serviceDescriptor,
|
||||
method: "Health"
|
||||
)
|
||||
}
|
||||
|
||||
public struct AppleIntelligenceProvider: Appleintelligence_AppleIntelligenceService.ServiceProtocol {
|
||||
/// The underlying AI service
|
||||
private let service: AppleIntelligenceService
|
||||
|
||||
@@ -37,123 +16,131 @@ public struct AppleIntelligenceProvider: RegistrableRPCService {
|
||||
self.apiKey = apiKey
|
||||
}
|
||||
|
||||
public func registerMethods<Transport: ServerTransport>(with router: inout RPCRouter<Transport>) {
|
||||
// Register Complete method (unary)
|
||||
router.registerHandler(
|
||||
forMethod: Methods.complete,
|
||||
deserializer: ProtobufDeserializer<Appleintelligence_CompletionRequest>(),
|
||||
serializer: ProtobufSerializer<Appleintelligence_CompletionResponse>()
|
||||
) { request, context in
|
||||
try self.validateApiKey(metadata: request.metadata)
|
||||
// MARK: - ServiceProtocol Implementation
|
||||
|
||||
// Collect the single message from the request stream
|
||||
var requestMessage: Appleintelligence_CompletionRequest?
|
||||
for try await message in request.messages {
|
||||
requestMessage = message
|
||||
break
|
||||
public func complete(
|
||||
request: GRPCCore.ServerRequest<Appleintelligence_CompletionRequest>,
|
||||
context: GRPCCore.ServerContext
|
||||
) async throws -> GRPCCore.ServerResponse<Appleintelligence_CompletionResponse> {
|
||||
try validateApiKey(metadata: request.metadata)
|
||||
|
||||
let message = request.message
|
||||
|
||||
// Convert protobuf images to service format
|
||||
let images = message.images.map { img in
|
||||
(data: img.data, filename: img.filename.isEmpty ? nil : img.filename)
|
||||
}
|
||||
|
||||
let (text, analyses) = try await service.complete(
|
||||
prompt: message.prompt,
|
||||
temperature: message.hasTemperature ? message.temperature : nil,
|
||||
maxTokens: message.hasMaxTokens ? Int(message.maxTokens) : nil,
|
||||
images: images
|
||||
)
|
||||
|
||||
var response = Appleintelligence_CompletionResponse()
|
||||
response.id = UUID().uuidString
|
||||
response.text = text
|
||||
response.finishReason = "stop"
|
||||
|
||||
// Include analysis results if requested
|
||||
if message.includeAnalysis {
|
||||
response.imageAnalyses = analyses.map { analysis in
|
||||
var protoAnalysis = Appleintelligence_ImageAnalysis()
|
||||
protoAnalysis.textContent = analysis.textContent
|
||||
protoAnalysis.labels = analysis.labels
|
||||
protoAnalysis.description_p = analysis.description
|
||||
return protoAnalysis
|
||||
}
|
||||
}
|
||||
|
||||
guard let message = requestMessage else {
|
||||
throw RPCError(code: .invalidArgument, message: "No request message received")
|
||||
}
|
||||
return ServerResponse(message: response)
|
||||
}
|
||||
|
||||
let text = try await self.service.complete(
|
||||
public func streamComplete(
|
||||
request: GRPCCore.ServerRequest<Appleintelligence_CompletionRequest>,
|
||||
context: GRPCCore.ServerContext
|
||||
) async throws -> GRPCCore.StreamingServerResponse<Appleintelligence_CompletionChunk> {
|
||||
try validateApiKey(metadata: request.metadata)
|
||||
|
||||
let message = request.message
|
||||
let completionId = UUID().uuidString
|
||||
|
||||
// Convert protobuf images to service format
|
||||
let images = message.images.map { img in
|
||||
(data: img.data, filename: img.filename.isEmpty ? nil : img.filename)
|
||||
}
|
||||
|
||||
return StreamingServerResponse { writer in
|
||||
let stream = await self.service.streamComplete(
|
||||
prompt: message.prompt,
|
||||
temperature: message.hasTemperature ? message.temperature : nil,
|
||||
maxTokens: message.hasMaxTokens ? Int(message.maxTokens) : nil
|
||||
maxTokens: message.hasMaxTokens ? Int(message.maxTokens) : nil,
|
||||
images: images
|
||||
)
|
||||
|
||||
var response = Appleintelligence_CompletionResponse()
|
||||
response.id = UUID().uuidString
|
||||
response.text = text
|
||||
response.finishReason = "stop"
|
||||
|
||||
return StreamingServerResponse(single: ServerResponse(message: response))
|
||||
}
|
||||
|
||||
// Register StreamComplete method (server streaming)
|
||||
router.registerHandler(
|
||||
forMethod: Methods.streamComplete,
|
||||
deserializer: ProtobufDeserializer<Appleintelligence_CompletionRequest>(),
|
||||
serializer: ProtobufSerializer<Appleintelligence_CompletionChunk>()
|
||||
) { request, context in
|
||||
try self.validateApiKey(metadata: request.metadata)
|
||||
|
||||
// Collect the single message from the request stream
|
||||
var requestMessage: Appleintelligence_CompletionRequest?
|
||||
for try await message in request.messages {
|
||||
requestMessage = message
|
||||
break
|
||||
}
|
||||
|
||||
guard let message = requestMessage else {
|
||||
throw RPCError(code: .invalidArgument, message: "No request message received")
|
||||
}
|
||||
|
||||
let completionId = UUID().uuidString
|
||||
let prompt = message.prompt
|
||||
let temperature = message.hasTemperature ? message.temperature : nil
|
||||
let maxTokens = message.hasMaxTokens ? Int(message.maxTokens) : nil
|
||||
|
||||
return StreamingServerResponse { writer in
|
||||
let stream = await self.service.streamComplete(
|
||||
prompt: prompt,
|
||||
temperature: temperature,
|
||||
maxTokens: maxTokens
|
||||
)
|
||||
|
||||
var lastContent = ""
|
||||
for try await partialResponse in stream {
|
||||
// Calculate the delta (new text since last response)
|
||||
let delta: String
|
||||
if partialResponse.hasPrefix(lastContent) {
|
||||
delta = String(partialResponse.dropFirst(lastContent.count))
|
||||
} else {
|
||||
delta = partialResponse
|
||||
}
|
||||
lastContent = partialResponse
|
||||
|
||||
if !delta.isEmpty {
|
||||
var chunk = Appleintelligence_CompletionChunk()
|
||||
chunk.id = completionId
|
||||
chunk.delta = delta
|
||||
chunk.isFinal = false
|
||||
try await writer.write(chunk)
|
||||
}
|
||||
var lastContent = ""
|
||||
var isFirstChunk = true
|
||||
for try await (partialResponse, analyses) in stream {
|
||||
// Calculate the delta (new text since last response)
|
||||
let delta: String
|
||||
if partialResponse.hasPrefix(lastContent) {
|
||||
delta = String(partialResponse.dropFirst(lastContent.count))
|
||||
} else {
|
||||
delta = partialResponse
|
||||
}
|
||||
lastContent = partialResponse
|
||||
|
||||
// Send final chunk
|
||||
var finalChunk = Appleintelligence_CompletionChunk()
|
||||
finalChunk.id = completionId
|
||||
finalChunk.delta = ""
|
||||
finalChunk.isFinal = true
|
||||
finalChunk.finishReason = "stop"
|
||||
try await writer.write(finalChunk)
|
||||
if !delta.isEmpty || isFirstChunk {
|
||||
var chunk = Appleintelligence_CompletionChunk()
|
||||
chunk.id = completionId
|
||||
chunk.delta = delta
|
||||
chunk.isFinal = false
|
||||
|
||||
return [:]
|
||||
// Include analyses in first chunk if requested
|
||||
if isFirstChunk && message.includeAnalysis, let analyses = analyses {
|
||||
chunk.imageAnalyses = analyses.map { analysis in
|
||||
var protoAnalysis = Appleintelligence_ImageAnalysis()
|
||||
protoAnalysis.textContent = analysis.textContent
|
||||
protoAnalysis.labels = analysis.labels
|
||||
protoAnalysis.description_p = analysis.description
|
||||
return protoAnalysis
|
||||
}
|
||||
}
|
||||
|
||||
try await writer.write(chunk)
|
||||
isFirstChunk = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Register Health method (unary)
|
||||
router.registerHandler(
|
||||
forMethod: Methods.health,
|
||||
deserializer: ProtobufDeserializer<Appleintelligence_HealthRequest>(),
|
||||
serializer: ProtobufSerializer<Appleintelligence_HealthResponse>()
|
||||
) { request, context in
|
||||
// Consume request messages (empty for health check)
|
||||
for try await _ in request.messages {}
|
||||
// Send final chunk
|
||||
var finalChunk = Appleintelligence_CompletionChunk()
|
||||
finalChunk.id = completionId
|
||||
finalChunk.delta = ""
|
||||
finalChunk.isFinal = true
|
||||
finalChunk.finishReason = "stop"
|
||||
try await writer.write(finalChunk)
|
||||
|
||||
let isHealthy = await self.service.isAvailable
|
||||
let modelStatus = await self.service.getModelStatus()
|
||||
|
||||
var response = Appleintelligence_HealthResponse()
|
||||
response.healthy = isHealthy
|
||||
response.modelStatus = modelStatus
|
||||
|
||||
return StreamingServerResponse(single: ServerResponse(message: response))
|
||||
return [:]
|
||||
}
|
||||
}
|
||||
|
||||
public func health(
|
||||
request: GRPCCore.ServerRequest<Appleintelligence_HealthRequest>,
|
||||
context: GRPCCore.ServerContext
|
||||
) async throws -> GRPCCore.ServerResponse<Appleintelligence_HealthResponse> {
|
||||
let isHealthy = await service.isAvailable
|
||||
let modelStatus = await service.getModelStatus()
|
||||
|
||||
var response = Appleintelligence_HealthResponse()
|
||||
response.healthy = isHealthy
|
||||
response.modelStatus = modelStatus
|
||||
|
||||
return ServerResponse(message: response)
|
||||
}
|
||||
|
||||
// MARK: - Private Helpers
|
||||
|
||||
/// Validate API key if configured
|
||||
private func validateApiKey(metadata: Metadata) throws {
|
||||
guard let expectedKey = apiKey else {
|
||||
|
||||
Reference in New Issue
Block a user