feat(task_worker): Enhance image upload with conversion and pre-caching

This commit is contained in:
cogwheel
2025-12-25 20:29:38 +05:30
parent 1447ddd93c
commit f594982d6a
7 changed files with 433 additions and 286 deletions

View File

@@ -1023,49 +1023,11 @@ bool validateFileCount(int currentCount, int newFilesCount, int? maxCount) {
return (currentCount + newFilesCount) <= maxCount; return (currentCount + newFilesCount) <= maxCount;
} }
// Helper function to get file content as base64
Future<String?> _getFileAsBase64(dynamic api, String fileId) async {
// Check if this is already a data URL (for images)
if (fileId.startsWith('data:')) {
return fileId;
}
try {
// First, get file info to determine if it's an image
final fileInfo = await api.getFileInfo(fileId);
// Try different fields for filename - check all possible field names
final fileName =
fileInfo['filename'] ??
fileInfo['meta']?['name'] ??
fileInfo['name'] ??
fileInfo['file_name'] ??
fileInfo['original_name'] ??
fileInfo['original_filename'] ??
'';
final ext = fileName.toLowerCase().split('.').last;
// Only process image files (including SVG)
if (!['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'].contains(ext)) {
return null;
}
// Get file content as base64 string
final fileContent = await api.getFileContent(fileId);
// The API service returns base64 string directly
return fileContent;
} catch (e) {
return null;
}
}
// Small internal helper to convert a message with attachments into the // Small internal helper to convert a message with attachments into the
// OpenWebUI content payload format (text + image_url + files). // OpenWebUI content payload format (text + image_url + files).
// - Adds text first (if non-empty) // - Adds text first (if non-empty)
// - Handles images as inline base64 data URLs (matching web client behavior) // - Images (base64 or server-stored) go into content array as image_url
// - Includes non-image attachments in a 'files' array for server-side resolution // - Non-image files go into files array for RAG/server-side resolution
Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
required dynamic api, required dynamic api,
required String role, required String role,
@@ -1078,15 +1040,14 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
contentArray.add({'type': 'text', 'text': cleanedText}); contentArray.add({'type': 'text', 'text': cleanedText});
} }
// Collect all files in OpenWebUI format for the files array // Collect non-image files for the files array
final allFiles = <Map<String, dynamic>>[]; final allFiles = <Map<String, dynamic>>[];
for (final attachmentId in attachmentIds) { for (final attachmentId in attachmentIds) {
try { try {
// Check if this is an image data URL (stored locally, matching web client) // Check if this is a base64 data URL (legacy or inline)
// Web client stores images as base64 data URLs, not server file IDs
if (attachmentId.startsWith('data:image/')) { if (attachmentId.startsWith('data:image/')) {
// This is an inline image data URL - add directly to content array // Inline image data URL - add directly to content array for LLM vision
contentArray.add({ contentArray.add({
'type': 'image_url', 'type': 'image_url',
'image_url': {'url': attachmentId}, 'image_url': {'url': attachmentId},
@@ -1094,46 +1055,43 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
continue; continue;
} }
// For server-stored files, fetch info // For server-stored files, fetch info to determine type
final fileInfo = await api.getFileInfo(attachmentId); final fileInfo = await api.getFileInfo(attachmentId);
final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'Unknown'; final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'Unknown';
final fileSize = fileInfo['size']; final fileSize = fileInfo['size'] ?? fileInfo['meta']?['size'];
final contentType =
fileInfo['meta']?['content_type'] ?? fileInfo['content_type'] ?? '';
final base64Data = await _getFileAsBase64(api, attachmentId); // Check if this is an image file
if (base64Data != null) { final isImage = contentType.toString().startsWith('image/');
// This is an image file from server - add to content array only
if (base64Data.startsWith('data:')) { if (isImage) {
contentArray.add({ // Images must be in content array as image_url for LLM vision
'type': 'image_url', // Fetch the image content from server and convert to base64 data URL
'image_url': {'url': base64Data}, try {
}); final fileContent = await api.getFileContent(attachmentId);
} else { String dataUrl;
final ext = fileName.toLowerCase().split('.').last; if (fileContent.startsWith('data:')) {
String mimeType = 'image/png'; dataUrl = fileContent;
if (ext == 'jpg' || ext == 'jpeg') { } else {
mimeType = 'image/jpeg'; // Determine MIME type from content type or file extension
} else if (ext == 'gif') { String mimeType = contentType.isNotEmpty
mimeType = 'image/gif'; ? contentType.toString()
} else if (ext == 'webp') { : _getMimeTypeFromFileName(fileName);
mimeType = 'image/webp'; dataUrl = 'data:$mimeType;base64,$fileContent';
} else if (ext == 'svg') {
mimeType = 'image/svg+xml';
} }
final dataUrl = 'data:$mimeType;base64,$base64Data';
contentArray.add({ contentArray.add({
'type': 'image_url', 'type': 'image_url',
'image_url': {'url': dataUrl}, 'image_url': {'url': dataUrl},
}); });
} catch (_) {
// If we can't fetch the image, skip it
} }
// Note: Images are handled in content array above, no need to duplicate in files array
// This prevents duplicate display in the WebUI
} else { } else {
// This is a non-image file - match web client format // Non-image files go to files array for RAG/server-side processing
allFiles.add({ allFiles.add({
'type': 'file', 'type': 'file',
'id': attachmentId, // Required for RAG system to lookup file content 'id': attachmentId,
'url': '/api/v1/files/$attachmentId', 'url': '/api/v1/files/$attachmentId',
'name': fileName, 'name': fileName,
if (fileSize != null) 'size': fileSize, if (fileSize != null) 'size': fileSize,
@@ -1154,6 +1112,19 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
return messageMap; return messageMap;
} }
String _getMimeTypeFromFileName(String fileName) {
final ext = fileName.toLowerCase().split('.').last;
return switch (ext) {
'jpg' || 'jpeg' => 'image/jpeg',
'png' => 'image/png',
'gif' => 'image/gif',
'webp' => 'image/webp',
'svg' => 'image/svg+xml',
'bmp' => 'image/bmp',
_ => 'image/png',
};
}
List<Map<String, dynamic>> _contextAttachmentsToFiles( List<Map<String, dynamic>> _contextAttachmentsToFiles(
List<ChatContextAttachment> attachments, List<ChatContextAttachment> attachments,
) { ) {
@@ -1751,107 +1722,138 @@ Future<void> _sendMessageInternal(
throw Exception('No API service or model selected'); throw Exception('No API service or model selected');
} }
Map<String, dynamic>? userSettingsData; // Get context attachments synchronously (no API calls)
String? userSystemPrompt;
if (!reviewerMode && api != null) {
try {
userSettingsData = await api.getUserSettings();
userSystemPrompt = _extractSystemPromptFromSettings(userSettingsData);
} catch (_) {}
}
// Check if we need to create a new conversation first
var activeConversation = ref.read(activeConversationProvider);
// Create user message first
// Build the files array to match web client format for persistence:
// - Images stored as {type: 'image', url: 'data:...'} (matching web client)
// - Server files stored as {type: 'file', id: '...', name: '...', url: '...'}
// - Context attachments (web/youtube/knowledge)
final contextAttachments = ref.read(contextAttachmentsProvider); final contextAttachments = ref.read(contextAttachmentsProvider);
final contextFiles = _contextAttachmentsToFiles(contextAttachments); final contextFiles = _contextAttachmentsToFiles(contextAttachments);
// Convert attachments to files format for web client compatibility // All attachments are now server file IDs (images uploaded like OpenWebUI)
// Process in parallel for better performance (fixes #310 - loading indicator) // Legacy base64 support kept for backwards compatibility
// while preserving original attachment order final legacyBase64Images = <Map<String, dynamic>>[];
final attachmentFiles = <Map<String, dynamic>>[]; final serverFileIds = <String>[];
if (attachments != null && !reviewerMode && api != null) {
// Process all attachments in parallel while preserving order if (attachments != null) {
final fileInfoFutures = attachments.map((attachment) async {
// Data URLs are images - return immediately (no API call needed)
if (attachment.startsWith('data:image/')) {
return <String, dynamic>{'type': 'image', 'url': attachment};
}
// Server file ID - fetch info
try {
final fileInfo = await api.getFileInfo(attachment);
final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'file';
final fileSize = fileInfo['size'] ?? fileInfo['meta']?['size'];
final collectionName =
fileInfo['meta']?['collection_name'] ?? fileInfo['collection_name'];
return <String, dynamic>{
'type': 'file',
'id': attachment,
'name': fileName,
'url': '/api/v1/files/$attachment',
if (fileSize != null) 'size': fileSize,
if (collectionName != null) 'collection_name': collectionName,
};
} catch (_) {
// If we can't fetch info, store minimal file entry
return <String, dynamic>{
'type': 'file',
'id': attachment,
'name': 'file',
'url': '/api/v1/files/$attachment',
};
}
});
// Future.wait preserves order - results match input order
final results = await Future.wait(fileInfoFutures);
attachmentFiles.addAll(results);
} else if (attachments != null) {
// Reviewer mode or no API - only handle images (server files need API)
for (final attachment in attachments) { for (final attachment in attachments) {
if (attachment.startsWith('data:image/')) { if (attachment.startsWith('data:image/')) {
attachmentFiles.add({'type': 'image', 'url': attachment}); // Legacy base64 format - keep for backwards compatibility
legacyBase64Images.add({'type': 'image', 'url': attachment});
} else { } else {
DebugLogger.log( // Server file ID (both images and documents)
'Ignoring non-image attachment in reviewer mode: $attachment', serverFileIds.add(attachment);
scope: 'chat/providers',
);
} }
} }
} }
// Combine attachment files and context files // Build initial user files with legacy base64 and context (server files added later)
final List<Map<String, dynamic>>? userFiles = final List<Map<String, dynamic>>? initialUserFiles =
(attachmentFiles.isNotEmpty || contextFiles.isNotEmpty) (legacyBase64Images.isNotEmpty || contextFiles.isNotEmpty)
? [...attachmentFiles, ...contextFiles] ? [...legacyBase64Images, ...contextFiles]
: null; : null;
final userMessage = ChatMessage( // Create user message - files will be updated after fetching server info
id: const Uuid().v4(), final userMessageId = const Uuid().v4();
var userMessage = ChatMessage(
id: userMessageId,
role: 'user', role: 'user',
content: message, content: message,
timestamp: DateTime.now(), timestamp: DateTime.now(),
model: selectedModel.id, model: selectedModel.id,
attachmentIds: attachments, attachmentIds: attachments,
files: userFiles, files: initialUserFiles,
); );
// Add user message to UI immediately for instant feedback
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
// Add assistant placeholder immediately to show typing indicator right away
final String assistantMessageId = const Uuid().v4();
final assistantPlaceholder = ChatMessage(
id: assistantMessageId,
role: 'assistant',
content: '',
timestamp: DateTime.now(),
model: selectedModel.id,
isStreaming: true,
);
ref.read(chatMessagesProvider.notifier).addMessage(assistantPlaceholder);
// Now do async work in parallel: user settings + server file info
String? userSystemPrompt;
Map<String, dynamic>? userSettingsData;
final serverFiles = <Map<String, dynamic>>[];
if (!reviewerMode && api != null) {
// Fetch user settings and server file info in parallel
final settingsFuture = api.getUserSettings().catchError((_) => null);
final fileInfoFutures = serverFileIds.map((fileId) async {
try {
final fileInfo = await api.getFileInfo(fileId);
final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'file';
final fileSize = fileInfo['size'] ?? fileInfo['meta']?['size'];
final contentType =
fileInfo['meta']?['content_type'] ?? fileInfo['content_type'] ?? '';
final collectionName =
fileInfo['meta']?['collection_name'] ?? fileInfo['collection_name'];
// Determine type: 'image' for image content types, 'file' for others
// .toString() for safety against malformed API responses returning non-String
final isImage = contentType.toString().startsWith('image/');
return <String, dynamic>{
'type': isImage ? 'image' : 'file',
'id': fileId,
'name': fileName,
'url': '/api/v1/files/$fileId', // Full URL for conversation parsing compatibility
if (fileSize != null) 'size': fileSize,
if (collectionName != null) 'collection_name': collectionName,
if (contentType.isNotEmpty) 'content_type': contentType,
};
} catch (_) {
return <String, dynamic>{
'type': 'file',
'id': fileId,
'name': 'file',
'url': '/api/v1/files/$fileId',
};
}
});
// Wait for all async work to complete in parallel
final fileInfoResults = await Future.wait(fileInfoFutures);
userSettingsData = await settingsFuture;
if (userSettingsData != null) {
userSystemPrompt = _extractSystemPromptFromSettings(userSettingsData);
}
serverFiles.addAll(fileInfoResults);
// Update user message with server file info if needed
if (serverFiles.isNotEmpty || legacyBase64Images.isNotEmpty) {
final allFiles = [...legacyBase64Images, ...serverFiles, ...contextFiles];
userMessage = userMessage.copyWith(files: allFiles);
ref
.read(chatMessagesProvider.notifier)
.updateMessageById(
userMessageId,
(ChatMessage m) => m.copyWith(files: allFiles),
);
}
}
// Check if we need to create a new conversation first
var activeConversation = ref.read(activeConversationProvider);
if (activeConversation == null) { if (activeConversation == null) {
// Check if there's a pending folder ID for this new conversation // Check if there's a pending folder ID for this new conversation
final pendingFolderId = ref.read(pendingFolderIdProvider); final pendingFolderId = ref.read(pendingFolderIdProvider);
// Create new conversation with the first message included // Create new conversation with user message AND assistant placeholder
// so the listener doesn't remove the placeholder when setting active
final localConversation = Conversation( final localConversation = Conversation(
id: const Uuid().v4(), id: const Uuid().v4(),
title: 'New Chat', title: 'New Chat',
createdAt: DateTime.now(), createdAt: DateTime.now(),
updatedAt: DateTime.now(), updatedAt: DateTime.now(),
systemPrompt: userSystemPrompt, systemPrompt: userSystemPrompt,
messages: [userMessage], // Include the user message messages: [userMessage, assistantPlaceholder],
folderId: pendingFolderId, folderId: pendingFolderId,
); );
@@ -1860,11 +1862,16 @@ Future<void> _sendMessageInternal(
activeConversation = localConversation; activeConversation = localConversation;
if (!reviewerMode) { if (!reviewerMode) {
// Try to create on server with the first message included // Try to create on server - use lightweight message without large
// base64 image data to avoid timeout (images sent in chat request)
try { try {
final lightweightMessage = userMessage.copyWith(
attachmentIds: null,
files: null,
);
final serverConversation = await api.createConversation( final serverConversation = await api.createConversation(
title: 'New Chat', title: 'New Chat',
messages: [userMessage], // Include the first message in creation messages: [lightweightMessage],
model: selectedModel.id, model: selectedModel.id,
systemPrompt: userSystemPrompt, systemPrompt: userSystemPrompt,
folderId: pendingFolderId, folderId: pendingFolderId,
@@ -1873,21 +1880,18 @@ Future<void> _sendMessageInternal(
// Clear the pending folder ID after successful creation // Clear the pending folder ID after successful creation
ref.read(pendingFolderIdProvider.notifier).clear(); ref.read(pendingFolderIdProvider.notifier).clear();
// Keep local messages (user + assistant placeholder) instead of server
// messages, since we're in the middle of sending and streaming
final currentMessages = ref.read(chatMessagesProvider);
final updatedConversation = localConversation.copyWith( final updatedConversation = localConversation.copyWith(
id: serverConversation.id, id: serverConversation.id,
systemPrompt: serverConversation.systemPrompt ?? userSystemPrompt, systemPrompt: serverConversation.systemPrompt ?? userSystemPrompt,
messages: serverConversation.messages.isNotEmpty messages: currentMessages,
? serverConversation.messages
: [userMessage],
folderId: serverConversation.folderId ?? pendingFolderId, folderId: serverConversation.folderId ?? pendingFolderId,
); );
ref.read(activeConversationProvider.notifier).set(updatedConversation); ref.read(activeConversationProvider.notifier).set(updatedConversation);
activeConversation = updatedConversation; activeConversation = updatedConversation;
// Set messages in the messages provider to keep UI in sync
ref.read(chatMessagesProvider.notifier).clearMessages();
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
ref ref
.read(conversationsProvider.notifier) .read(conversationsProvider.notifier)
.upsertConversation( .upsertConversation(
@@ -1914,22 +1918,13 @@ Future<void> _sendMessageInternal(
} }
}); });
} catch (e) { } catch (e) {
// Still add the message locally
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
// Clear the pending folder ID on failure to prevent stale state // Clear the pending folder ID on failure to prevent stale state
ref.read(pendingFolderIdProvider.notifier).clear(); ref.read(pendingFolderIdProvider.notifier).clear();
} }
} else { } else {
// Add message for reviewer mode
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
// Clear the pending folder ID even in reviewer mode // Clear the pending folder ID even in reviewer mode
ref.read(pendingFolderIdProvider.notifier).clear(); ref.read(pendingFolderIdProvider.notifier).clear();
} }
} else {
// Add user message to existing conversation
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
} }
if (activeConversation != null && if (activeConversation != null &&
@@ -1941,19 +1936,6 @@ Future<void> _sendMessageInternal(
activeConversation = updated; activeConversation = updated;
} }
// Add assistant placeholder immediately after user message to show typing
// indicator right away (fixes #310 - loading animation not showing)
final String assistantMessageId = const Uuid().v4();
final assistantPlaceholder = ChatMessage(
id: assistantMessageId,
role: 'assistant',
content: '',
timestamp: DateTime.now(),
model: selectedModel.id,
isStreaming: true,
);
ref.read(chatMessagesProvider.notifier).addMessage(assistantPlaceholder);
// Reviewer mode: simulate a response locally and return // Reviewer mode: simulate a response locally and return
if (reviewerMode) { if (reviewerMode) {
// Check if there are attachments // Check if there are attachments

View File

@@ -8,6 +8,7 @@ import 'package:flutter_image_compress/flutter_image_compress.dart';
import 'package:image_picker/image_picker.dart'; import 'package:image_picker/image_picker.dart';
import 'package:path/path.dart' as path; import 'package:path/path.dart' as path;
import '../../../core/providers/app_providers.dart'; import '../../../core/providers/app_providers.dart';
import '../../../core/services/worker_manager.dart';
import '../../../core/utils/debug_logger.dart'; import '../../../core/utils/debug_logger.dart';
/// Size threshold for optimizing images to WebP (200KB). /// Size threshold for optimizing images to WebP (200KB).
@@ -38,17 +39,10 @@ const Set<String> _alwaysConvertFormats = {
}; };
/// Formats that benefit from WebP conversion when large. /// Formats that benefit from WebP conversion when large.
const Set<String> _optimizableFormats = { const Set<String> _optimizableFormats = {'.jpg', '.jpeg', '.png'};
'.jpg',
'.jpeg',
'.png',
};
/// Formats that should never be converted (animation, already optimal). /// Formats that should never be converted (animation, already optimal).
const Set<String> _preserveFormats = { const Set<String> _preserveFormats = {'.gif', '.webp'};
'.gif',
'.webp',
};
/// All supported image formats (both standard and those requiring conversion). /// All supported image formats (both standard and those requiring conversion).
const Set<String> allSupportedImageFormats = { const Set<String> allSupportedImageFormats = {
@@ -71,6 +65,30 @@ bool _shouldPreserve(String extension) {
return _preserveFormats.contains(extension); return _preserveFormats.contains(extension);
} }
/// Top-level function for base64 encoding in an isolate.
String _encodeToDataUrlWorker(Map<String, dynamic> payload) {
final bytes = payload['bytes'] as List<int>;
final mimeType = payload['mimeType'] as String;
return 'data:$mimeType;base64,${base64Encode(bytes)}';
}
/// Helper to encode bytes to data URL, using isolate when worker is provided.
Future<String> _encodeToDataUrl(
List<int> bytes,
String mimeType,
WorkerManager? worker,
) async {
if (worker != null && bytes.length > 50 * 1024) {
// Use isolate for files > 50KB
return worker.schedule(_encodeToDataUrlWorker, {
'bytes': bytes,
'mimeType': mimeType,
}, debugLabel: 'base64-encode');
}
// Small files: encode on main thread
return 'data:$mimeType;base64,${base64Encode(bytes)}';
}
/// Converts an image file to a base64 data URL with smart optimization. /// Converts an image file to a base64 data URL with smart optimization.
/// This is a standalone utility used by both FileAttachmentService and TaskWorker. /// This is a standalone utility used by both FileAttachmentService and TaskWorker.
/// ///
@@ -81,8 +99,14 @@ bool _shouldPreserve(String extension) {
/// - GIF → Preserve (maintains animation) /// - GIF → Preserve (maintains animation)
/// - WebP → Preserve (already optimal) /// - WebP → Preserve (already optimal)
/// ///
/// If [worker] is provided, base64 encoding runs in a background isolate
/// to avoid blocking the UI thread for large images.
///
/// Returns null if conversion fails for formats requiring conversion. /// Returns null if conversion fails for formats requiring conversion.
Future<String?> convertImageFileToDataUrl(File imageFile) async { Future<String?> convertImageFileToDataUrl(
File imageFile, {
WorkerManager? worker,
}) async {
try { try {
final ext = path.extension(imageFile.path).toLowerCase(); final ext = path.extension(imageFile.path).toLowerCase();
final fileSize = await imageFile.length(); final fileSize = await imageFile.length();
@@ -97,7 +121,7 @@ Future<String?> convertImageFileToDataUrl(File imageFile) async {
final convertedBytes = await _convertToWebP(imageFile); final convertedBytes = await _convertToWebP(imageFile);
if (convertedBytes != null) { if (convertedBytes != null) {
return 'data:image/webp;base64,${base64Encode(convertedBytes)}'; return _encodeToDataUrl(convertedBytes, 'image/webp', worker);
} }
DebugLogger.warning( DebugLogger.warning(
@@ -110,7 +134,7 @@ Future<String?> convertImageFileToDataUrl(File imageFile) async {
if (_shouldPreserve(ext)) { if (_shouldPreserve(ext)) {
final bytes = await imageFile.readAsBytes(); final bytes = await imageFile.readAsBytes();
final mimeType = ext == '.gif' ? 'image/gif' : 'image/webp'; final mimeType = ext == '.gif' ? 'image/gif' : 'image/webp';
return 'data:$mimeType;base64,${base64Encode(bytes)}'; return _encodeToDataUrl(bytes, mimeType, worker);
} }
// Optimizable formats (JPEG, PNG) - convert if large // Optimizable formats (JPEG, PNG) - convert if large
@@ -134,7 +158,7 @@ Future<String?> convertImageFileToDataUrl(File imageFile) async {
'saved': savings, 'saved': savings,
}, },
); );
return 'data:image/webp;base64,${base64Encode(convertedBytes)}'; return _encodeToDataUrl(convertedBytes, 'image/webp', worker);
} }
// Fall through to pass-through if conversion fails // Fall through to pass-through if conversion fails
} }
@@ -146,7 +170,7 @@ Future<String?> convertImageFileToDataUrl(File imageFile) async {
mimeType = 'image/jpeg'; mimeType = 'image/jpeg';
} }
return 'data:$mimeType;base64,${base64Encode(bytes)}'; return _encodeToDataUrl(bytes, mimeType, worker);
} catch (e) { } catch (e) {
DebugLogger.error('convert-image-failed', scope: 'attachments', error: e); DebugLogger.error('convert-image-failed', scope: 'attachments', error: e);
return null; return null;
@@ -167,21 +191,14 @@ Future<List<int>?> _convertToWebP(File imageFile) async {
DebugLogger.log( DebugLogger.log(
'Image converted to WebP successfully', 'Image converted to WebP successfully',
scope: 'attachments', scope: 'attachments',
data: { data: {'originalPath': imageFile.path, 'resultSize': result.length},
'originalPath': imageFile.path,
'resultSize': result.length,
},
); );
return result; return result;
} }
return null; return null;
} catch (e) { } catch (e) {
DebugLogger.error( DebugLogger.error('webp-conversion-failed', scope: 'attachments', error: e);
'webp-conversion-failed',
scope: 'attachments',
error: e,
);
return null; return null;
} }
} }

View File

@@ -47,6 +47,7 @@ class _EnhancedAttachmentState extends ConsumerState<EnhancedAttachment> {
try { try {
// Data URL for images short-circuit to image widget // Data URL for images short-circuit to image widget
if (widget.attachmentId.startsWith('data:image/')) { if (widget.attachmentId.startsWith('data:image/')) {
if (!mounted) return;
setState(() { setState(() {
_isLoading = false; _isLoading = false;
_fileInfo = {'mime': 'image/inline'}; _fileInfo = {'mime': 'image/inline'};
@@ -56,6 +57,7 @@ class _EnhancedAttachmentState extends ConsumerState<EnhancedAttachment> {
final api = ref.read(apiServiceProvider); final api = ref.read(apiServiceProvider);
if (api is! ApiService) { if (api is! ApiService) {
if (!mounted) return;
setState(() { setState(() {
_isLoading = false; _isLoading = false;
_error = 'Service unavailable'; _error = 'Service unavailable';
@@ -64,11 +66,13 @@ class _EnhancedAttachmentState extends ConsumerState<EnhancedAttachment> {
} }
final info = await api.getFileInfo(widget.attachmentId); final info = await api.getFileInfo(widget.attachmentId);
if (!mounted) return;
setState(() { setState(() {
_fileInfo = info; _fileInfo = info;
_isLoading = false; _isLoading = false;
}); });
} catch (e) { } catch (e) {
if (!mounted) return;
setState(() { setState(() {
_error = 'Failed to load attachment'; _error = 'Failed to load attachment';
_isLoading = false; _isLoading = false;

View File

@@ -26,6 +26,16 @@ final _globalImageBytesCache = <String, Uint8List>{};
final _globalSvgStates = <String, bool>{}; final _globalSvgStates = <String, bool>{};
final _base64WhitespacePattern = RegExp(r'\s'); final _base64WhitespacePattern = RegExp(r'\s');
/// Pre-cache image bytes for instant display after upload.
/// Call this with the server file ID and image bytes after successful upload.
void preCacheImageBytes(String fileId, Uint8List bytes) {
if (fileId.isEmpty || bytes.isEmpty) return;
_globalImageBytesCache[fileId] = bytes;
_globalLoadingStates[fileId] = false;
// Detect SVG
_globalSvgStates[fileId] = _isSvgBytes(bytes);
}
Uint8List _decodeImageData(String data) { Uint8List _decodeImageData(String data) {
var payload = data; var payload = data;
if (payload.startsWith('data:')) { if (payload.startsWith('data:')) {
@@ -147,6 +157,21 @@ class _EnhancedImageAttachmentState
Future<void> _loadImage() async { Future<void> _loadImage() async {
final l10n = AppLocalizations.of(context)!; final l10n = AppLocalizations.of(context)!;
// Check bytes cache first (populated during upload for instant display)
final preCachedBytes = _globalImageBytesCache[widget.attachmentId];
if (preCachedBytes != null) {
final cachedIsSvg = _globalSvgStates[widget.attachmentId] ?? false;
if (mounted) {
setState(() {
_cachedBytes = preCachedBytes;
_isSvg = cachedIsSvg;
_isLoading = false;
});
}
return;
}
final cachedError = _globalErrorStates[widget.attachmentId]; final cachedError = _globalErrorStates[widget.attachmentId];
if (cachedError != null) { if (cachedError != null) {
if (mounted) { if (mounted) {
@@ -241,15 +266,25 @@ class _EnhancedImageAttachmentState
final fileInfo = await api.getFileInfo(attachmentId); final fileInfo = await api.getFileInfo(attachmentId);
final fileName = _extractFileName(fileInfo); final fileName = _extractFileName(fileInfo);
final ext = fileName.toLowerCase().split('.').last; final ext = fileName.toLowerCase().split('.').last;
final contentType = (fileInfo['meta']?['content_type'] ??
fileInfo['content_type'] ??
'')
.toString()
.toLowerCase();
if (!['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'].contains(ext)) { // Check both extension and content_type for image detection
final isImageByExt =
['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'bmp'].contains(ext);
final isImageByContentType = contentType.startsWith('image/');
if (!isImageByExt && !isImageByContentType) {
final error = l10n.notAnImageFile(fileName); final error = l10n.notAnImageFile(fileName);
_cacheError(error); _cacheError(error);
return; return;
} }
// Track if this is an SVG file based on extension // Track if this is an SVG file based on extension or content type
final isSvgFile = ext == 'svg'; final isSvgFile = ext == 'svg' || contentType.contains('svg');
final fileContent = await api.getFileContent(attachmentId); final fileContent = await api.getFileContent(attachmentId);

View File

@@ -1060,6 +1060,16 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
final isGenerating = ref.watch(isChatStreamingProvider); final isGenerating = ref.watch(isChatStreamingProvider);
final stopGeneration = ref.read(stopGenerationProvider); final stopGeneration = ref.read(stopGenerationProvider);
// Check if file uploads are in progress or complete
final attachedFiles = ref.watch(attachedFilesProvider);
final hasUploadsInProgress = attachedFiles.any(
(f) =>
f.status == FileUploadStatus.uploading ||
f.status == FileUploadStatus.pending,
);
final allUploadsComplete = attachedFiles.isEmpty ||
attachedFiles.every((f) => f.status == FileUploadStatus.completed);
final webSearchEnabled = ref.watch(webSearchEnabledProvider); final webSearchEnabled = ref.watch(webSearchEnabledProvider);
final imageGenEnabled = ref.watch(imageGenerationEnabledProvider); final imageGenEnabled = ref.watch(imageGenerationEnabledProvider);
final imageGenAvailable = ref.watch(imageGenerationAvailableProvider); final imageGenAvailable = ref.watch(imageGenerationAvailableProvider);
@@ -1349,6 +1359,8 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
isGenerating, isGenerating,
stopGeneration, stopGeneration,
voiceAvailable, voiceAvailable,
allUploadsComplete,
hasUploadsInProgress,
), ),
], ],
), ),
@@ -1416,6 +1428,8 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
isGenerating, isGenerating,
stopGeneration, stopGeneration,
voiceAvailable, voiceAvailable,
allUploadsComplete,
hasUploadsInProgress,
), ),
], ],
), ),
@@ -1825,12 +1839,16 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
bool isGenerating, bool isGenerating,
void Function() stopGeneration, void Function() stopGeneration,
bool voiceAvailable, bool voiceAvailable,
bool allUploadsComplete,
bool hasUploadsInProgress,
) { ) {
// Compact 44px touch target, circular radius, md icon size // Compact 44px touch target, circular radius, md icon size
const double buttonSize = TouchTarget.minimum; // 44.0 const double buttonSize = TouchTarget.minimum; // 44.0
const double radius = AppBorderRadius.round; // big to ensure circle const double radius = AppBorderRadius.round; // big to ensure circle
final enabled = !isGenerating && hasText && widget.enabled; // Don't allow sending until all uploads are complete
final enabled =
!isGenerating && hasText && widget.enabled && allUploadsComplete;
// Generating -> STOP variant // Generating -> STOP variant
if (isGenerating) { if (isGenerating) {
@@ -1947,17 +1965,26 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
: const [], : const [],
), ),
child: Center( child: Center(
child: Icon( child: hasUploadsInProgress
Platform.isIOS ? SizedBox(
? CupertinoIcons.arrow_up width: IconSize.large,
: Icons.arrow_upward, height: IconSize.large,
size: IconSize.large, child: CircularProgressIndicator(
color: enabled strokeWidth: 2.5,
? context.conduitTheme.buttonPrimaryText color: context.conduitTheme.textSecondary,
: context.conduitTheme.textPrimary.withValues(
alpha: Alpha.disabled,
), ),
), )
: Icon(
Platform.isIOS
? CupertinoIcons.arrow_up
: Icons.arrow_upward,
size: IconSize.large,
color: enabled
? context.conduitTheme.buttonPrimaryText
: context.conduitTheme.textPrimary.withValues(
alpha: Alpha.disabled,
),
),
), ),
), ),
), ),

View File

@@ -130,7 +130,8 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
Widget _buildFileImageLayout(List<dynamic> imageFiles, int imageCount) { Widget _buildFileImageLayout(List<dynamic> imageFiles, int imageCount) {
if (imageCount == 1) { if (imageCount == 1) {
final String imageUrl = imageFiles[0]['url'] as String; final file = imageFiles[0];
final String imageUrl = file['url'] as String;
return Row( return Row(
key: ValueKey('user_file_single_$imageUrl'), key: ValueKey('user_file_single_$imageUrl'),
mainAxisAlignment: MainAxisAlignment.end, mainAxisAlignment: MainAxisAlignment.end,
@@ -154,7 +155,7 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
maxHeight: 350, maxHeight: 350,
), ),
disableAnimation: widget.isStreaming, disableAnimation: widget.isStreaming,
httpHeaders: _headersForFile(imageFiles[0]), httpHeaders: _headersForFile(file),
), ),
), ),
), ),
@@ -173,7 +174,8 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
mainAxisSize: MainAxisSize.min, mainAxisSize: MainAxisSize.min,
children: imageFiles.asMap().entries.map((entry) { children: imageFiles.asMap().entries.map((entry) {
final index = entry.key; final index = entry.key;
final String imageUrl = entry.value['url'] as String; final file = entry.value;
final String imageUrl = file['url'] as String;
return Padding( return Padding(
padding: EdgeInsets.only(left: index == 0 ? 0 : Spacing.xs), padding: EdgeInsets.only(left: index == 0 ? 0 : Spacing.xs),
child: Container( child: Container(
@@ -196,7 +198,7 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
maxHeight: 180, maxHeight: 180,
), ),
disableAnimation: widget.isStreaming, disableAnimation: widget.isStreaming,
httpHeaders: _headersForFile(entry.value), httpHeaders: _headersForFile(file),
), ),
), ),
), ),

View File

@@ -1,14 +1,18 @@
import 'dart:async'; import 'dart:async';
import 'dart:convert'; import 'dart:convert';
import 'dart:io'; import 'dart:io';
import 'package:flutter_image_compress/flutter_image_compress.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'dart:typed_data';
import '../../../core/providers/app_providers.dart'; import '../../../core/providers/app_providers.dart';
import '../../../core/services/attachment_upload_queue.dart'; import '../../../core/services/attachment_upload_queue.dart';
import '../../../core/services/worker_manager.dart';
import '../../../core/utils/debug_logger.dart'; import '../../../core/utils/debug_logger.dart';
import '../../../features/chat/providers/chat_providers.dart' as chat; import '../../../features/chat/providers/chat_providers.dart' as chat;
import '../../../features/chat/providers/context_attachments_provider.dart'; import '../../../features/chat/providers/context_attachments_provider.dart';
import '../../../features/chat/services/file_attachment_service.dart'; import '../../../features/chat/services/file_attachment_service.dart';
import '../../../features/chat/widgets/enhanced_image_attachment.dart';
import 'outbound_task.dart'; import 'outbound_task.dart';
class TaskWorker { class TaskWorker {
@@ -77,14 +81,9 @@ class TaskWorker {
final lowerName = task.fileName.toLowerCase(); final lowerName = task.fileName.toLowerCase();
final bool isImage = allSupportedImageFormats.any(lowerName.endsWith); final bool isImage = allSupportedImageFormats.any(lowerName.endsWith);
// For images: read as base64 locally (matching web client behavior) // Upload all files (including images) to server
// Web client never uploads images to /api/v1/files/ // This mirrors OpenWebUI's approach: images are uploaded to /api/v1/files/
if (isImage) { // and the server resolves them when sending to LLM
await _handleImageAsBase64(task);
return;
}
// For non-images: upload to server
final uploader = AttachmentUploadQueue(); final uploader = AttachmentUploadQueue();
try { try {
final api = _ref.read(apiServiceProvider); final api = _ref.read(apiServiceProvider);
@@ -93,15 +92,47 @@ class TaskWorker {
} }
} catch (_) {} } catch (_) {}
// For images: convert unsupported formats and optimize large JPEG/PNG
String uploadPath = task.filePath;
String uploadFileName = task.fileName;
String? uploadMimeType = task.mimeType;
if (isImage) {
final shouldConvert = await _shouldConvertImage(lowerName, task.fileSize);
if (shouldConvert) {
final convertedPath = await _convertImageForUpload(task);
if (convertedPath != null) {
uploadPath = convertedPath;
// Update filename to .webp extension since we converted the format
final baseName = task.fileName.contains('.')
? task.fileName.substring(0, task.fileName.lastIndexOf('.'))
: task.fileName;
uploadFileName = '$baseName.webp';
uploadMimeType = 'image/webp';
}
}
}
// Read image bytes before upload for instant display cache
Uint8List? imageBytes;
if (isImage) {
try {
imageBytes = await File(uploadPath).readAsBytes();
} catch (_) {}
}
final id = await uploader.enqueue( final id = await uploader.enqueue(
filePath: task.filePath, filePath: uploadPath,
fileName: task.fileName, fileName: uploadFileName,
fileSize: task.fileSize ?? 0, fileSize: task.fileSize ?? 0,
mimeType: task.mimeType, mimeType: uploadMimeType,
checksum: task.checksum, checksum: task.checksum,
); );
final completer = Completer<void>(); final completer = Completer<void>();
// Capture values for use in closure
final displayFileName = uploadFileName;
final cachedBytes = imageBytes;
final tempFilePath = uploadPath != task.filePath ? uploadPath : null;
late final StreamSubscription<List<QueuedAttachment>> sub; late final StreamSubscription<List<QueuedAttachment>> sub;
sub = uploader.queueStream.listen((items) { sub = uploader.queueStream.listen((items) {
QueuedAttachment? entry; QueuedAttachment? entry;
@@ -124,9 +155,17 @@ class TaskWorker {
QueuedAttachmentStatus.failed => FileUploadStatus.failed, QueuedAttachmentStatus.failed => FileUploadStatus.failed,
QueuedAttachmentStatus.cancelled => FileUploadStatus.failed, QueuedAttachmentStatus.cancelled => FileUploadStatus.failed,
}; };
// Pre-cache image bytes for instant display when upload completes
if (status == FileUploadStatus.completed &&
entry.fileId != null &&
cachedBytes != null) {
preCacheImageBytes(entry.fileId!, cachedBytes);
}
final newState = FileUploadState( final newState = FileUploadState(
file: File(task.filePath), file: File(task.filePath),
fileName: task.fileName, fileName: displayFileName,
fileSize: task.fileSize ?? existing.fileSize, fileSize: task.fileSize ?? existing.fileSize,
progress: status == FileUploadStatus.completed progress: status == FileUploadStatus.completed
? 1.0 ? 1.0
@@ -134,7 +173,7 @@ class TaskWorker {
status: status, status: status,
fileId: entry.fileId ?? existing.fileId, fileId: entry.fileId ?? existing.fileId,
error: entry.lastError, error: entry.lastError,
isImage: false, isImage: isImage,
); );
_ref _ref
.read(attachedFilesProvider.notifier) .read(attachedFilesProvider.notifier)
@@ -146,6 +185,12 @@ class TaskWorker {
case QueuedAttachmentStatus.failed: case QueuedAttachmentStatus.failed:
case QueuedAttachmentStatus.cancelled: case QueuedAttachmentStatus.cancelled:
sub.cancel(); sub.cancel();
// Clean up temp file from image conversion
if (tempFilePath != null) {
try {
File(tempFilePath).parent.deleteSync(recursive: true);
} catch (_) {}
}
completer.complete(); completer.complete();
break; break;
default: default:
@@ -160,73 +205,104 @@ class TaskWorker {
try { try {
sub.cancel(); sub.cancel();
} catch (_) {} } catch (_) {}
// Clean up temp file on timeout
if (tempFilePath != null) {
try {
File(tempFilePath).parent.deleteSync(recursive: true);
} catch (_) {}
}
// Update state to failed on timeout
try {
final current = _ref.read(attachedFilesProvider);
final idx = current.indexWhere((f) => f.file.path == task.filePath);
if (idx != -1) {
final existing = current[idx];
final newState = FileUploadState(
file: File(task.filePath),
fileName: displayFileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: 0.0,
status: FileUploadStatus.failed,
error: 'Upload timed out',
isImage: isImage,
);
_ref
.read(attachedFilesProvider.notifier)
.updateFileState(task.filePath, newState);
}
} catch (_) {}
DebugLogger.warning('UploadMediaTask timed out: ${task.fileName}'); DebugLogger.warning('UploadMediaTask timed out: ${task.fileName}');
return; return;
}, },
); );
} }
/// Handles image files by reading as base64 locally (matching web client) /// Check if image should be converted to WebP before upload
Future<void> _handleImageAsBase64(UploadMediaTask task) async { /// - Always convert: HEIC, RAW formats, BMP (unsupported or inefficient)
/// - Optimize if large: JPEG, PNG > 500KB
/// - Never convert: WebP (already optimal), GIF (may be animated)
Future<bool> _shouldConvertImage(String lowerName, int? fileSize) async {
// Always convert these formats (unsupported by some backends or inefficient)
const alwaysConvert = {
'.heic', '.heif', '.dng', '.raw', '.cr2', '.nef', '.arw', '.orf', '.rw2', '.bmp',
};
if (alwaysConvert.any(lowerName.endsWith)) {
return true;
}
// Never convert these (already optimal or special format)
const neverConvert = {'.webp', '.gif'};
if (neverConvert.any(lowerName.endsWith)) {
return false;
}
// Optimize large JPEG/PNG (> 500KB)
const optimizeThreshold = 500 * 1024; // 500KB
const optimizableFormats = {'.jpg', '.jpeg', '.png'};
if (optimizableFormats.any(lowerName.endsWith)) {
final size = fileSize ?? 0;
return size > optimizeThreshold;
}
return false;
}
/// Convert image to WebP for upload if needed
Future<String?> _convertImageForUpload(UploadMediaTask task) async {
try { try {
final file = File(task.filePath); final file = File(task.filePath);
final base64DataUrl = await convertImageFileToDataUrl(file); final result = await FlutterImageCompress.compressWithFile(
file.absolute.path,
if (base64DataUrl == null) { format: CompressFormat.webp,
throw Exception('Failed to convert image to base64'); quality: 85,
}
// Update attachment state with base64 data URL
final current = _ref.read(attachedFilesProvider);
final idx = current.indexWhere((f) => f.file.path == task.filePath);
if (idx != -1) {
final existing = current[idx];
final newState = FileUploadState(
file: file,
fileName: task.fileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: 1.0,
status: FileUploadStatus.completed,
fileId: base64DataUrl,
isImage: true,
base64DataUrl: base64DataUrl,
);
_ref
.read(attachedFilesProvider.notifier)
.updateFileState(task.filePath, newState);
}
DebugLogger.log(
'image-base64-complete',
scope: 'tasks/upload',
data: {
'fileName': task.fileName,
'dataUrlLength': base64DataUrl.length,
},
); );
if (result != null && result.isNotEmpty) {
// Write to temp file for upload
final tempDir = await Directory.systemTemp.createTemp('conduit_img_');
final tempFile = File('${tempDir.path}/converted.webp');
await tempFile.writeAsBytes(result);
DebugLogger.log(
'Converted image for upload',
scope: 'tasks/upload',
data: {
'original': task.filePath,
'converted': tempFile.path,
'originalSize': await file.length(),
'convertedSize': result.length,
},
);
return tempFile.path;
}
} catch (e) { } catch (e) {
DebugLogger.error('image-base64-failed', scope: 'tasks/upload', error: e); DebugLogger.error('image-conversion-failed', scope: 'tasks/upload', error: e);
// Update state to failed
try {
final current = _ref.read(attachedFilesProvider);
final idx = current.indexWhere((f) => f.file.path == task.filePath);
if (idx != -1) {
final existing = current[idx];
final newState = FileUploadState(
file: File(task.filePath),
fileName: task.fileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: 0.0,
status: FileUploadStatus.failed,
error: e.toString(),
isImage: true,
);
_ref
.read(attachedFilesProvider.notifier)
.updateFileState(task.filePath, newState);
}
} catch (_) {}
} }
return null;
} }
Future<void> _performExecuteToolCall(ExecuteToolCallTask task) async { Future<void> _performExecuteToolCall(ExecuteToolCallTask task) async {
@@ -322,7 +398,11 @@ class TaskWorker {
// Convert image to base64 data URL locally (matching web client behavior) // Convert image to base64 data URL locally (matching web client behavior)
try { try {
final file = File(task.filePath); final file = File(task.filePath);
final base64DataUrl = await convertImageFileToDataUrl(file); final worker = _ref.read(workerManagerProvider);
final base64DataUrl = await convertImageFileToDataUrl(
file,
worker: worker,
);
if (base64DataUrl == null) { if (base64DataUrl == null) {
throw Exception('Failed to convert image to base64'); throw Exception('Failed to convert image to base64');