Merge pull request #328 from cogwheel0/feat-improve-attachment-processing

feat-improve-attachment-processing
This commit is contained in:
cogwheel
2026-01-01 10:33:16 +05:30
committed by GitHub
9 changed files with 559 additions and 373 deletions

View File

@@ -1023,49 +1023,11 @@ bool validateFileCount(int currentCount, int newFilesCount, int? maxCount) {
return (currentCount + newFilesCount) <= maxCount;
}
// Helper function to get file content as base64
Future<String?> _getFileAsBase64(dynamic api, String fileId) async {
// Check if this is already a data URL (for images)
if (fileId.startsWith('data:')) {
return fileId;
}
try {
// First, get file info to determine if it's an image
final fileInfo = await api.getFileInfo(fileId);
// Try different fields for filename - check all possible field names
final fileName =
fileInfo['filename'] ??
fileInfo['meta']?['name'] ??
fileInfo['name'] ??
fileInfo['file_name'] ??
fileInfo['original_name'] ??
fileInfo['original_filename'] ??
'';
final ext = fileName.toLowerCase().split('.').last;
// Only process image files (including SVG)
if (!['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'].contains(ext)) {
return null;
}
// Get file content as base64 string
final fileContent = await api.getFileContent(fileId);
// The API service returns base64 string directly
return fileContent;
} catch (e) {
return null;
}
}
// Small internal helper to convert a message with attachments into the
// OpenWebUI content payload format (text + image_url + files).
// - Adds text first (if non-empty)
// - Handles images as inline base64 data URLs (matching web client behavior)
// - Includes non-image attachments in a 'files' array for server-side resolution
// - Images (base64 or server-stored) go into content array as image_url
// - Non-image files go into files array for RAG/server-side resolution
Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
required dynamic api,
required String role,
@@ -1078,15 +1040,14 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
contentArray.add({'type': 'text', 'text': cleanedText});
}
// Collect all files in OpenWebUI format for the files array
// Collect non-image files for the files array
final allFiles = <Map<String, dynamic>>[];
for (final attachmentId in attachmentIds) {
try {
// Check if this is an image data URL (stored locally, matching web client)
// Web client stores images as base64 data URLs, not server file IDs
// Check if this is a base64 data URL (legacy or inline)
if (attachmentId.startsWith('data:image/')) {
// This is an inline image data URL - add directly to content array
// Inline image data URL - add directly to content array for LLM vision
contentArray.add({
'type': 'image_url',
'image_url': {'url': attachmentId},
@@ -1094,46 +1055,43 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
continue;
}
// For server-stored files, fetch info
// For server-stored files, fetch info to determine type
final fileInfo = await api.getFileInfo(attachmentId);
final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'Unknown';
final fileSize = fileInfo['size'];
final fileSize = fileInfo['size'] ?? fileInfo['meta']?['size'];
final contentType =
fileInfo['meta']?['content_type'] ?? fileInfo['content_type'] ?? '';
final base64Data = await _getFileAsBase64(api, attachmentId);
if (base64Data != null) {
// This is an image file from server - add to content array only
if (base64Data.startsWith('data:')) {
contentArray.add({
'type': 'image_url',
'image_url': {'url': base64Data},
});
} else {
final ext = fileName.toLowerCase().split('.').last;
String mimeType = 'image/png';
if (ext == 'jpg' || ext == 'jpeg') {
mimeType = 'image/jpeg';
} else if (ext == 'gif') {
mimeType = 'image/gif';
} else if (ext == 'webp') {
mimeType = 'image/webp';
} else if (ext == 'svg') {
mimeType = 'image/svg+xml';
// Check if this is an image file
final isImage = contentType.toString().startsWith('image/');
if (isImage) {
// Images must be in content array as image_url for LLM vision
// Fetch the image content from server and convert to base64 data URL
try {
final fileContent = await api.getFileContent(attachmentId);
String dataUrl;
if (fileContent.startsWith('data:')) {
dataUrl = fileContent;
} else {
// Determine MIME type from content type or file extension
String mimeType = contentType.isNotEmpty
? contentType.toString()
: _getMimeTypeFromFileName(fileName);
dataUrl = 'data:$mimeType;base64,$fileContent';
}
final dataUrl = 'data:$mimeType;base64,$base64Data';
contentArray.add({
'type': 'image_url',
'image_url': {'url': dataUrl},
});
} catch (_) {
// If we can't fetch the image, skip it
}
// Note: Images are handled in content array above, no need to duplicate in files array
// This prevents duplicate display in the WebUI
} else {
// This is a non-image file - match web client format
// Non-image files go to files array for RAG/server-side processing
allFiles.add({
'type': 'file',
'id': attachmentId, // Required for RAG system to lookup file content
'id': attachmentId,
'url': '/api/v1/files/$attachmentId',
'name': fileName,
if (fileSize != null) 'size': fileSize,
@@ -1154,6 +1112,19 @@ Future<Map<String, dynamic>> _buildMessagePayloadWithAttachments({
return messageMap;
}
String _getMimeTypeFromFileName(String fileName) {
final ext = fileName.toLowerCase().split('.').last;
return switch (ext) {
'jpg' || 'jpeg' => 'image/jpeg',
'png' => 'image/png',
'gif' => 'image/gif',
'webp' => 'image/webp',
'svg' => 'image/svg+xml',
'bmp' => 'image/bmp',
_ => 'image/png',
};
}
List<Map<String, dynamic>> _contextAttachmentsToFiles(
List<ChatContextAttachment> attachments,
) {
@@ -1751,104 +1722,138 @@ Future<void> _sendMessageInternal(
throw Exception('No API service or model selected');
}
Map<String, dynamic>? userSettingsData;
String? userSystemPrompt;
if (!reviewerMode && api != null) {
try {
userSettingsData = await api.getUserSettings();
userSystemPrompt = _extractSystemPromptFromSettings(userSettingsData);
} catch (_) {}
}
// Check if we need to create a new conversation first
var activeConversation = ref.read(activeConversationProvider);
// Create user message first
// Build the files array to match web client format for persistence:
// - Images stored as {type: 'image', url: 'data:...'} (matching web client)
// - Server files stored as {type: 'file', id: '...', name: '...', url: '...'}
// - Context attachments (web/youtube/knowledge)
// Get context attachments synchronously (no API calls)
final contextAttachments = ref.read(contextAttachmentsProvider);
final contextFiles = _contextAttachmentsToFiles(contextAttachments);
// Convert attachments to files format for web client compatibility
final attachmentFiles = <Map<String, dynamic>>[];
if (attachments != null && !reviewerMode && api != null) {
for (final attachment in attachments) {
// Data URLs are images - store inline
if (attachment.startsWith('data:image/')) {
attachmentFiles.add({'type': 'image', 'url': attachment});
} else {
// Server file ID - fetch info and create file entry
// Match web client format: {type, id, name, url, size, collection_name}
try {
final fileInfo = await api.getFileInfo(attachment);
final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'file';
final fileSize = fileInfo['size'] ?? fileInfo['meta']?['size'];
final collectionName =
fileInfo['meta']?['collection_name'] ??
fileInfo['collection_name'];
attachmentFiles.add({
'type': 'file',
'id': attachment,
'name': fileName,
'url': '/api/v1/files/$attachment',
if (fileSize != null) 'size': fileSize,
if (collectionName != null) 'collection_name': collectionName,
});
} catch (_) {
// If we can't fetch info, store minimal file entry with placeholder name
attachmentFiles.add({
'type': 'file',
'id': attachment,
'name': 'file',
'url': '/api/v1/files/$attachment',
});
}
}
}
} else if (attachments != null) {
// Reviewer mode or no API - only handle images (server files need API)
// All attachments are now server file IDs (images uploaded like OpenWebUI)
// Legacy base64 support kept for backwards compatibility
final legacyBase64Images = <Map<String, dynamic>>[];
final serverFileIds = <String>[];
if (attachments != null) {
for (final attachment in attachments) {
if (attachment.startsWith('data:image/')) {
attachmentFiles.add({'type': 'image', 'url': attachment});
// Legacy base64 format - keep for backwards compatibility
legacyBase64Images.add({'type': 'image', 'url': attachment});
} else {
DebugLogger.log(
'Ignoring non-image attachment in reviewer mode: $attachment',
scope: 'chat/providers',
);
// Server file ID (both images and documents)
serverFileIds.add(attachment);
}
}
}
// Combine attachment files and context files
final List<Map<String, dynamic>>? userFiles =
(attachmentFiles.isNotEmpty || contextFiles.isNotEmpty)
? [...attachmentFiles, ...contextFiles]
// Build initial user files with legacy base64 and context (server files added later)
final List<Map<String, dynamic>>? initialUserFiles =
(legacyBase64Images.isNotEmpty || contextFiles.isNotEmpty)
? [...legacyBase64Images, ...contextFiles]
: null;
final userMessage = ChatMessage(
id: const Uuid().v4(),
// Create user message - files will be updated after fetching server info
final userMessageId = const Uuid().v4();
var userMessage = ChatMessage(
id: userMessageId,
role: 'user',
content: message,
timestamp: DateTime.now(),
model: selectedModel.id,
attachmentIds: attachments,
files: userFiles,
files: initialUserFiles,
);
// Add user message to UI immediately for instant feedback
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
// Add assistant placeholder immediately to show typing indicator right away
final String assistantMessageId = const Uuid().v4();
final assistantPlaceholder = ChatMessage(
id: assistantMessageId,
role: 'assistant',
content: '',
timestamp: DateTime.now(),
model: selectedModel.id,
isStreaming: true,
);
ref.read(chatMessagesProvider.notifier).addMessage(assistantPlaceholder);
// Now do async work in parallel: user settings + server file info
String? userSystemPrompt;
Map<String, dynamic>? userSettingsData;
final serverFiles = <Map<String, dynamic>>[];
if (!reviewerMode && api != null) {
// Fetch user settings and server file info in parallel
final settingsFuture = api.getUserSettings().catchError((_) => null);
final fileInfoFutures = serverFileIds.map((fileId) async {
try {
final fileInfo = await api.getFileInfo(fileId);
final fileName = fileInfo['filename'] ?? fileInfo['name'] ?? 'file';
final fileSize = fileInfo['size'] ?? fileInfo['meta']?['size'];
final contentType =
fileInfo['meta']?['content_type'] ?? fileInfo['content_type'] ?? '';
final collectionName =
fileInfo['meta']?['collection_name'] ?? fileInfo['collection_name'];
// Determine type: 'image' for image content types, 'file' for others
// .toString() for safety against malformed API responses returning non-String
final isImage = contentType.toString().startsWith('image/');
return <String, dynamic>{
'type': isImage ? 'image' : 'file',
'id': fileId,
'name': fileName,
'url': '/api/v1/files/$fileId', // Full URL for conversation parsing compatibility
if (fileSize != null) 'size': fileSize,
if (collectionName != null) 'collection_name': collectionName,
if (contentType.isNotEmpty) 'content_type': contentType,
};
} catch (_) {
return <String, dynamic>{
'type': 'file',
'id': fileId,
'name': 'file',
'url': '/api/v1/files/$fileId',
};
}
});
// Wait for all async work to complete in parallel
final fileInfoResults = await Future.wait(fileInfoFutures);
userSettingsData = await settingsFuture;
if (userSettingsData != null) {
userSystemPrompt = _extractSystemPromptFromSettings(userSettingsData);
}
serverFiles.addAll(fileInfoResults);
// Update user message with server file info if needed
if (serverFiles.isNotEmpty || legacyBase64Images.isNotEmpty) {
final allFiles = [...legacyBase64Images, ...serverFiles, ...contextFiles];
userMessage = userMessage.copyWith(files: allFiles);
ref
.read(chatMessagesProvider.notifier)
.updateMessageById(
userMessageId,
(ChatMessage m) => m.copyWith(files: allFiles),
);
}
}
// Check if we need to create a new conversation first
var activeConversation = ref.read(activeConversationProvider);
if (activeConversation == null) {
// Check if there's a pending folder ID for this new conversation
final pendingFolderId = ref.read(pendingFolderIdProvider);
// Create new conversation with the first message included
// Create new conversation with user message AND assistant placeholder
// so the listener doesn't remove the placeholder when setting active
final localConversation = Conversation(
id: const Uuid().v4(),
title: 'New Chat',
createdAt: DateTime.now(),
updatedAt: DateTime.now(),
systemPrompt: userSystemPrompt,
messages: [userMessage], // Include the user message
messages: [userMessage, assistantPlaceholder],
folderId: pendingFolderId,
);
@@ -1857,11 +1862,16 @@ Future<void> _sendMessageInternal(
activeConversation = localConversation;
if (!reviewerMode) {
// Try to create on server with the first message included
// Try to create on server - use lightweight message without large
// base64 image data to avoid timeout (images sent in chat request)
try {
final lightweightMessage = userMessage.copyWith(
attachmentIds: null,
files: null,
);
final serverConversation = await api.createConversation(
title: 'New Chat',
messages: [userMessage], // Include the first message in creation
messages: [lightweightMessage],
model: selectedModel.id,
systemPrompt: userSystemPrompt,
folderId: pendingFolderId,
@@ -1870,21 +1880,18 @@ Future<void> _sendMessageInternal(
// Clear the pending folder ID after successful creation
ref.read(pendingFolderIdProvider.notifier).clear();
// Keep local messages (user + assistant placeholder) instead of server
// messages, since we're in the middle of sending and streaming
final currentMessages = ref.read(chatMessagesProvider);
final updatedConversation = localConversation.copyWith(
id: serverConversation.id,
systemPrompt: serverConversation.systemPrompt ?? userSystemPrompt,
messages: serverConversation.messages.isNotEmpty
? serverConversation.messages
: [userMessage],
messages: currentMessages,
folderId: serverConversation.folderId ?? pendingFolderId,
);
ref.read(activeConversationProvider.notifier).set(updatedConversation);
activeConversation = updatedConversation;
// Set messages in the messages provider to keep UI in sync
ref.read(chatMessagesProvider.notifier).clearMessages();
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
ref
.read(conversationsProvider.notifier)
.upsertConversation(
@@ -1911,22 +1918,13 @@ Future<void> _sendMessageInternal(
}
});
} catch (e) {
// Still add the message locally
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
// Clear the pending folder ID on failure to prevent stale state
ref.read(pendingFolderIdProvider.notifier).clear();
}
} else {
// Add message for reviewer mode
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
// Clear the pending folder ID even in reviewer mode
ref.read(pendingFolderIdProvider.notifier).clear();
}
} else {
// Add user message to existing conversation
ref.read(chatMessagesProvider.notifier).addMessage(userMessage);
}
if (activeConversation != null &&
@@ -1938,21 +1936,8 @@ Future<void> _sendMessageInternal(
activeConversation = updated;
}
// We'll add the assistant message placeholder after we get the message ID from the API (or immediately in reviewer mode)
// Reviewer mode: simulate a response locally and return
if (reviewerMode) {
// Add assistant message placeholder
final assistantMessage = ChatMessage(
id: const Uuid().v4(),
role: 'assistant',
content: '',
timestamp: DateTime.now(),
model: selectedModel.id,
isStreaming: true,
);
ref.read(chatMessagesProvider.notifier).addMessage(assistantMessage);
// Check if there are attachments
String? filename;
if (attachments != null && attachments.isNotEmpty) {
@@ -2066,21 +2051,8 @@ Future<void> _sendMessageInternal(
: null;
try {
// Pre-seed assistant skeleton on server to ensure correct chain
// Generate assistant message id now (must be consistent across client/server)
final String assistantMessageId = const Uuid().v4();
// Add assistant placeholder locally before sending
final assistantPlaceholder = ChatMessage(
id: assistantMessageId,
role: 'assistant',
content: '',
timestamp: DateTime.now(),
model: selectedModel.id,
isStreaming: true,
);
ref.read(chatMessagesProvider.notifier).addMessage(assistantPlaceholder);
// Assistant placeholder was already added above (after user message)
// to show typing indicator immediately. Sync conversation state to server.
// Sync conversation state to ensure WebUI can load conversation history
try {
final activeConvForSeed = ref.read(activeConversationProvider);

View File

@@ -8,8 +8,13 @@ import 'package:flutter_image_compress/flutter_image_compress.dart';
import 'package:image_picker/image_picker.dart';
import 'package:path/path.dart' as path;
import '../../../core/providers/app_providers.dart';
import '../../../core/services/worker_manager.dart';
import '../../../core/utils/debug_logger.dart';
/// Size threshold for optimizing images to WebP (200KB).
/// Images larger than this will be converted to WebP for better compression.
const int _webpOptimizationThreshold = 200 * 1024;
/// Standard web image formats that LLMs can process directly.
const Set<String> _standardImageFormats = {
'.jpg',
@@ -17,17 +22,12 @@ const Set<String> _standardImageFormats = {
'.png',
'.gif',
'.webp',
'.bmp',
};
/// iOS-specific formats that need conversion to JPEG before LLM submission.
const Set<String> _iosImageFormats = {
/// Formats that should always be converted to WebP (not widely supported).
const Set<String> _alwaysConvertFormats = {
'.heic',
'.heif',
};
/// RAW image formats that need conversion to JPEG before LLM submission.
const Set<String> _rawImageFormats = {
'.dng',
'.raw',
'.cr2',
@@ -35,101 +35,170 @@ const Set<String> _rawImageFormats = {
'.arw',
'.orf',
'.rw2',
'.bmp',
};
/// Formats that benefit from WebP conversion when large.
const Set<String> _optimizableFormats = {'.jpg', '.jpeg', '.png'};
/// Formats that should never be converted (animation, already optimal).
const Set<String> _preserveFormats = {'.gif', '.webp'};
/// All supported image formats (both standard and those requiring conversion).
const Set<String> allSupportedImageFormats = {
..._standardImageFormats,
..._iosImageFormats,
..._rawImageFormats,
..._alwaysConvertFormats,
};
/// Returns true if the extension requires conversion to a standard format.
bool _needsConversion(String extension) {
return _iosImageFormats.contains(extension) ||
_rawImageFormats.contains(extension);
/// Returns true if the extension always requires conversion to WebP.
bool _alwaysNeedsConversion(String extension) {
return _alwaysConvertFormats.contains(extension);
}
/// Converts an image file to a base64 data URL.
/// Returns true if the format can benefit from WebP optimization.
bool _canOptimize(String extension) {
return _optimizableFormats.contains(extension);
}
/// Returns true if the format should be preserved as-is.
bool _shouldPreserve(String extension) {
return _preserveFormats.contains(extension);
}
/// Top-level function for base64 encoding in an isolate.
String _encodeToDataUrlWorker(Map<String, dynamic> payload) {
final bytes = payload['bytes'] as List<int>;
final mimeType = payload['mimeType'] as String;
return 'data:$mimeType;base64,${base64Encode(bytes)}';
}
/// Helper to encode bytes to data URL, using isolate when worker is provided.
Future<String> _encodeToDataUrl(
List<int> bytes,
String mimeType,
WorkerManager? worker,
) async {
if (worker != null && bytes.length > 50 * 1024) {
// Use isolate for files > 50KB
return worker.schedule(_encodeToDataUrlWorker, {
'bytes': bytes,
'mimeType': mimeType,
}, debugLabel: 'base64-encode');
}
// Small files: encode on main thread
return 'data:$mimeType;base64,${base64Encode(bytes)}';
}
/// Converts an image file to a base64 data URL with smart optimization.
/// This is a standalone utility used by both FileAttachmentService and TaskWorker.
///
/// Handles iOS-specific formats (HEIC, HEIF) and RAW formats (DNG, CR2, etc.)
/// by converting them to JPEG before encoding.
/// Optimization strategy:
/// - HEIC/HEIF/RAW/BMP → Always convert to WebP
/// - Large JPEG/PNG (>200KB) → Convert to WebP for better compression
/// - Small JPEG/PNG (<200KB) → Pass through as-is
/// - GIF → Preserve (maintains animation)
/// - WebP → Preserve (already optimal)
///
/// Returns null if conversion fails.
Future<String?> convertImageFileToDataUrl(File imageFile) async {
/// If [worker] is provided, base64 encoding runs in a background isolate
/// to avoid blocking the UI thread for large images.
///
/// Returns null if conversion fails for formats requiring conversion.
Future<String?> convertImageFileToDataUrl(
File imageFile, {
WorkerManager? worker,
}) async {
try {
final ext = path.extension(imageFile.path).toLowerCase();
final fileSize = await imageFile.length();
// Check if we need to convert the image format
if (_needsConversion(ext)) {
// Formats that must always be converted (HEIC, RAW, BMP, etc.)
if (_alwaysNeedsConversion(ext)) {
DebugLogger.log(
'Converting image from $ext to JPEG',
'Converting image from $ext to WebP (required)',
scope: 'attachments',
data: {'path': imageFile.path},
data: {'path': imageFile.path, 'size': fileSize},
);
final convertedBytes = await _convertImageToJpeg(imageFile);
final convertedBytes = await _convertToWebP(imageFile);
if (convertedBytes != null) {
return 'data:image/jpeg;base64,${base64Encode(convertedBytes)}';
return _encodeToDataUrl(convertedBytes, 'image/webp', worker);
}
// Conversion failed - return null rather than sending unusable raw data
DebugLogger.warning(
'Conversion failed for $ext format, cannot process image',
);
return null;
}
// Standard format - read directly
final bytes = await imageFile.readAsBytes();
// Formats that should be preserved as-is (GIF, WebP)
if (_shouldPreserve(ext)) {
final bytes = await imageFile.readAsBytes();
final mimeType = ext == '.gif' ? 'image/gif' : 'image/webp';
return _encodeToDataUrl(bytes, mimeType, worker);
}
// Optimizable formats (JPEG, PNG) - convert if large
if (_canOptimize(ext) && fileSize > _webpOptimizationThreshold) {
DebugLogger.log(
'Optimizing large image from $ext to WebP',
scope: 'attachments',
data: {'path': imageFile.path, 'size': fileSize},
);
final convertedBytes = await _convertToWebP(imageFile);
if (convertedBytes != null) {
final savings = fileSize - convertedBytes.length;
final savingsPercent = (savings / fileSize * 100).toStringAsFixed(1);
DebugLogger.log(
'WebP optimization saved $savingsPercent%',
scope: 'attachments',
data: {
'originalSize': fileSize,
'newSize': convertedBytes.length,
'saved': savings,
},
);
return _encodeToDataUrl(convertedBytes, 'image/webp', worker);
}
// Fall through to pass-through if conversion fails
}
// Pass through as-is (small images or unknown formats)
final bytes = await imageFile.readAsBytes();
String mimeType = 'image/png';
if (ext == '.jpg' || ext == '.jpeg') {
mimeType = 'image/jpeg';
} else if (ext == '.gif') {
mimeType = 'image/gif';
} else if (ext == '.webp') {
mimeType = 'image/webp';
}
return 'data:$mimeType;base64,${base64Encode(bytes)}';
return _encodeToDataUrl(bytes, mimeType, worker);
} catch (e) {
DebugLogger.error('convert-image-failed', scope: 'attachments', error: e);
return null;
}
}
/// Converts an image file to JPEG bytes using flutter_image_compress.
/// This handles iOS-specific formats (HEIC, HEIF) and RAW formats (DNG, etc.)
Future<List<int>?> _convertImageToJpeg(File imageFile) async {
/// Converts an image file to WebP bytes using flutter_image_compress.
/// WebP provides better compression than JPEG while maintaining quality.
Future<List<int>?> _convertToWebP(File imageFile) async {
try {
// Use flutter_image_compress for native iOS/Android conversion
final result = await FlutterImageCompress.compressWithFile(
imageFile.absolute.path,
format: CompressFormat.jpeg,
quality: 90,
format: CompressFormat.webp,
quality: 85,
);
if (result != null && result.isNotEmpty) {
DebugLogger.log(
'Image converted successfully',
'Image converted to WebP successfully',
scope: 'attachments',
data: {
'originalPath': imageFile.path,
'resultSize': result.length,
},
data: {'originalPath': imageFile.path, 'resultSize': result.length},
);
return result;
}
return null;
} catch (e) {
DebugLogger.error(
'image-conversion-failed',
scope: 'attachments',
error: e,
);
DebugLogger.error('webp-conversion-failed', scope: 'attachments', error: e);
return null;
}
}
@@ -163,7 +232,7 @@ String _deriveDisplayName({
String _timestampedName({required String prefix, required String extension}) {
final DateTime now = DateTime.now();
String two(int value) => value.toString().padLeft(2, '0');
final String ext = extension.isNotEmpty ? extension : '.jpg';
final String ext = extension.isNotEmpty ? extension : '.webp';
final String timestamp =
'${now.year}${two(now.month)}${two(now.day)}_${two(now.hour)}${two(now.minute)}${two(now.second)}';
return '${prefix}_$timestamp$ext';
@@ -295,7 +364,9 @@ class FileAttachmentService {
}
}
// Compress image similar to OpenWebUI's implementation
/// Compresses and resizes an image data URL.
/// Uses PNG format for the resize operation (dart:ui limitation),
/// then converts to WebP for optimal file size.
Future<String> compressImage(
String imageDataUrl,
int? maxWidth,
@@ -314,7 +385,7 @@ class FileAttachmentService {
: imageDataUrl,
},
);
return imageDataUrl; // Return original if format is invalid
return imageDataUrl;
}
final data = parts[1];
final bytes = base64Decode(data);
@@ -330,7 +401,7 @@ class FileAttachmentService {
// Calculate new dimensions maintaining aspect ratio
if (maxWidth != null && maxHeight != null) {
if (width <= maxWidth && height <= maxHeight) {
return imageDataUrl; // No compression needed
return imageDataUrl;
}
if (width / height > maxWidth / maxHeight) {
@@ -342,19 +413,19 @@ class FileAttachmentService {
}
} else if (maxWidth != null) {
if (width <= maxWidth) {
return imageDataUrl; // No compression needed
return imageDataUrl;
}
height = ((maxWidth * height) / width).round();
width = maxWidth;
} else if (maxHeight != null) {
if (height <= maxHeight) {
return imageDataUrl; // No compression needed
return imageDataUrl;
}
width = ((maxHeight * width) / height).round();
height = maxHeight;
}
// Create compressed image
// Create resized image (dart:ui only supports PNG output)
final recorder = ui.PictureRecorder();
final canvas = Canvas(recorder);
@@ -366,22 +437,28 @@ class FileAttachmentService {
);
final picture = recorder.endRecording();
final compressedImage = await picture.toImage(width, height);
final byteData = await compressedImage.toByteData(
final resizedImage = await picture.toImage(width, height);
final byteData = await resizedImage.toByteData(
format: ui.ImageByteFormat.png,
);
final compressedBytes = byteData!.buffer.asUint8List();
final pngBytes = byteData!.buffer.asUint8List();
// Convert back to data URL
final compressedBase64 = base64Encode(compressedBytes);
return 'data:image/png;base64,$compressedBase64';
// Convert PNG to WebP for better compression
final webpBytes = await FlutterImageCompress.compressWithList(
pngBytes,
format: CompressFormat.webp,
quality: 85,
);
final compressedBase64 = base64Encode(webpBytes);
return 'data:image/webp;base64,$compressedBase64';
} catch (e) {
DebugLogger.error(
'compress-failed',
scope: 'attachments/image',
error: e,
);
return imageDataUrl; // Return original if compression fails
return imageDataUrl;
}
}

View File

@@ -593,7 +593,6 @@ class _AssistantMessageWidgetState extends ConsumerState<AssistantMessageWidget>
Widget _buildSegmentedContent() {
final children = <Widget>[];
bool firstToolSpacerAdded = false;
bool hasNonTextSegment = false;
int idx = 0;
for (final seg in _segments) {
if (seg.isTool && seg.toolCall != null) {
@@ -603,16 +602,10 @@ class _AssistantMessageWidgetState extends ConsumerState<AssistantMessageWidget>
firstToolSpacerAdded = true;
}
children.add(_buildToolCallTile(seg.toolCall!));
hasNonTextSegment = true;
} else if (seg.isReasoning && seg.reasoning != null) {
children.add(_buildReasoningTile(seg.reasoning!, idx));
hasNonTextSegment = true;
} else if ((seg.text ?? '').trim().isNotEmpty) {
// Add spacing before text content if it follows non-text segments
if (hasNonTextSegment) {
children.add(const SizedBox(height: Spacing.sm));
hasNonTextSegment = false;
}
// No extra spacing needed - reasoning/tool tiles have bottom padding
children.add(_buildEnhancedMarkdownContent(seg.text!));
}
idx++;
@@ -704,12 +697,19 @@ class _AssistantMessageWidgetState extends ConsumerState<AssistantMessageWidget>
return false;
}
final hasVisibleStatus = widget.message.statusHistory
// Check if there's a pending (not done) visible status - those have shimmer
// so we don't need the typing indicator. But if all visible statuses are
// done (e.g., "Retrieved 1 source"), show typing indicator to indicate
// the model is still working on generating a response.
final visibleStatuses = widget.message.statusHistory
.where((status) => status.hidden != true)
.isNotEmpty;
if (hasVisibleStatus) {
.toList();
final hasPendingStatus = visibleStatuses.any((status) => status.done != true);
if (hasPendingStatus) {
// Pending status has shimmer effect, no need for typing indicator
return false;
}
// If all statuses are done but no content yet, show typing indicator
final hasFollowUps = widget.message.followUps.isNotEmpty;
if (hasFollowUps) {

View File

@@ -47,6 +47,7 @@ class _EnhancedAttachmentState extends ConsumerState<EnhancedAttachment> {
try {
// Data URL for images short-circuit to image widget
if (widget.attachmentId.startsWith('data:image/')) {
if (!mounted) return;
setState(() {
_isLoading = false;
_fileInfo = {'mime': 'image/inline'};
@@ -56,6 +57,7 @@ class _EnhancedAttachmentState extends ConsumerState<EnhancedAttachment> {
final api = ref.read(apiServiceProvider);
if (api is! ApiService) {
if (!mounted) return;
setState(() {
_isLoading = false;
_error = 'Service unavailable';
@@ -64,11 +66,13 @@ class _EnhancedAttachmentState extends ConsumerState<EnhancedAttachment> {
}
final info = await api.getFileInfo(widget.attachmentId);
if (!mounted) return;
setState(() {
_fileInfo = info;
_isLoading = false;
});
} catch (e) {
if (!mounted) return;
setState(() {
_error = 'Failed to load attachment';
_isLoading = false;

View File

@@ -26,6 +26,16 @@ final _globalImageBytesCache = <String, Uint8List>{};
final _globalSvgStates = <String, bool>{};
final _base64WhitespacePattern = RegExp(r'\s');
/// Pre-cache image bytes for instant display after upload.
/// Call this with the server file ID and image bytes after successful upload.
void preCacheImageBytes(String fileId, Uint8List bytes) {
if (fileId.isEmpty || bytes.isEmpty) return;
_globalImageBytesCache[fileId] = bytes;
_globalLoadingStates[fileId] = false;
// Detect SVG
_globalSvgStates[fileId] = _isSvgBytes(bytes);
}
Uint8List _decodeImageData(String data) {
var payload = data;
if (payload.startsWith('data:')) {
@@ -147,6 +157,21 @@ class _EnhancedImageAttachmentState
Future<void> _loadImage() async {
final l10n = AppLocalizations.of(context)!;
// Check bytes cache first (populated during upload for instant display)
final preCachedBytes = _globalImageBytesCache[widget.attachmentId];
if (preCachedBytes != null) {
final cachedIsSvg = _globalSvgStates[widget.attachmentId] ?? false;
if (mounted) {
setState(() {
_cachedBytes = preCachedBytes;
_isSvg = cachedIsSvg;
_isLoading = false;
});
}
return;
}
final cachedError = _globalErrorStates[widget.attachmentId];
if (cachedError != null) {
if (mounted) {
@@ -241,15 +266,25 @@ class _EnhancedImageAttachmentState
final fileInfo = await api.getFileInfo(attachmentId);
final fileName = _extractFileName(fileInfo);
final ext = fileName.toLowerCase().split('.').last;
final contentType = (fileInfo['meta']?['content_type'] ??
fileInfo['content_type'] ??
'')
.toString()
.toLowerCase();
if (!['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg'].contains(ext)) {
// Check both extension and content_type for image detection
final isImageByExt =
['jpg', 'jpeg', 'png', 'gif', 'webp', 'svg', 'bmp'].contains(ext);
final isImageByContentType = contentType.startsWith('image/');
if (!isImageByExt && !isImageByContentType) {
final error = l10n.notAnImageFile(fileName);
_cacheError(error);
return;
}
// Track if this is an SVG file based on extension
final isSvgFile = ext == 'svg';
// Track if this is an SVG file based on extension or content type
final isSvgFile = ext == 'svg' || contentType.contains('svg');
final fileContent = await api.getFileContent(attachmentId);

View File

@@ -1060,6 +1060,16 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
final isGenerating = ref.watch(isChatStreamingProvider);
final stopGeneration = ref.read(stopGenerationProvider);
// Check if file uploads are in progress or complete
final attachedFiles = ref.watch(attachedFilesProvider);
final hasUploadsInProgress = attachedFiles.any(
(f) =>
f.status == FileUploadStatus.uploading ||
f.status == FileUploadStatus.pending,
);
final allUploadsComplete = attachedFiles.isEmpty ||
attachedFiles.every((f) => f.status == FileUploadStatus.completed);
final webSearchEnabled = ref.watch(webSearchEnabledProvider);
final imageGenEnabled = ref.watch(imageGenerationEnabledProvider);
final imageGenAvailable = ref.watch(imageGenerationAvailableProvider);
@@ -1349,6 +1359,8 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
isGenerating,
stopGeneration,
voiceAvailable,
allUploadsComplete,
hasUploadsInProgress,
),
],
),
@@ -1416,6 +1428,8 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
isGenerating,
stopGeneration,
voiceAvailable,
allUploadsComplete,
hasUploadsInProgress,
),
],
),
@@ -1825,12 +1839,16 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
bool isGenerating,
void Function() stopGeneration,
bool voiceAvailable,
bool allUploadsComplete,
bool hasUploadsInProgress,
) {
// Compact 44px touch target, circular radius, md icon size
const double buttonSize = TouchTarget.minimum; // 44.0
const double radius = AppBorderRadius.round; // big to ensure circle
final enabled = !isGenerating && hasText && widget.enabled;
// Don't allow sending until all uploads are complete
final enabled =
!isGenerating && hasText && widget.enabled && allUploadsComplete;
// Generating -> STOP variant
if (isGenerating) {
@@ -1947,17 +1965,26 @@ class _ModernChatInputState extends ConsumerState<ModernChatInput>
: const [],
),
child: Center(
child: Icon(
Platform.isIOS
? CupertinoIcons.arrow_up
: Icons.arrow_upward,
size: IconSize.large,
color: enabled
? context.conduitTheme.buttonPrimaryText
: context.conduitTheme.textPrimary.withValues(
alpha: Alpha.disabled,
child: hasUploadsInProgress
? SizedBox(
width: IconSize.large,
height: IconSize.large,
child: CircularProgressIndicator(
strokeWidth: 2.5,
color: context.conduitTheme.textSecondary,
),
),
)
: Icon(
Platform.isIOS
? CupertinoIcons.arrow_up
: Icons.arrow_upward,
size: IconSize.large,
color: enabled
? context.conduitTheme.buttonPrimaryText
: context.conduitTheme.textPrimary.withValues(
alpha: Alpha.disabled,
),
),
),
),
),

View File

@@ -130,7 +130,8 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
Widget _buildFileImageLayout(List<dynamic> imageFiles, int imageCount) {
if (imageCount == 1) {
final String imageUrl = imageFiles[0]['url'] as String;
final file = imageFiles[0];
final String imageUrl = file['url'] as String;
return Row(
key: ValueKey('user_file_single_$imageUrl'),
mainAxisAlignment: MainAxisAlignment.end,
@@ -154,7 +155,7 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
maxHeight: 350,
),
disableAnimation: widget.isStreaming,
httpHeaders: _headersForFile(imageFiles[0]),
httpHeaders: _headersForFile(file),
),
),
),
@@ -173,7 +174,8 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
mainAxisSize: MainAxisSize.min,
children: imageFiles.asMap().entries.map((entry) {
final index = entry.key;
final String imageUrl = entry.value['url'] as String;
final file = entry.value;
final String imageUrl = file['url'] as String;
return Padding(
padding: EdgeInsets.only(left: index == 0 ? 0 : Spacing.xs),
child: Container(
@@ -196,7 +198,7 @@ class _UserMessageBubbleState extends ConsumerState<UserMessageBubble> {
maxHeight: 180,
),
disableAnimation: widget.isStreaming,
httpHeaders: _headersForFile(entry.value),
httpHeaders: _headersForFile(file),
),
),
),

View File

@@ -1,14 +1,18 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:flutter_image_compress/flutter_image_compress.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'dart:typed_data';
import '../../../core/providers/app_providers.dart';
import '../../../core/services/attachment_upload_queue.dart';
import '../../../core/services/worker_manager.dart';
import '../../../core/utils/debug_logger.dart';
import '../../../features/chat/providers/chat_providers.dart' as chat;
import '../../../features/chat/providers/context_attachments_provider.dart';
import '../../../features/chat/services/file_attachment_service.dart';
import '../../../features/chat/widgets/enhanced_image_attachment.dart';
import 'outbound_task.dart';
class TaskWorker {
@@ -77,14 +81,9 @@ class TaskWorker {
final lowerName = task.fileName.toLowerCase();
final bool isImage = allSupportedImageFormats.any(lowerName.endsWith);
// For images: read as base64 locally (matching web client behavior)
// Web client never uploads images to /api/v1/files/
if (isImage) {
await _handleImageAsBase64(task);
return;
}
// For non-images: upload to server
// Upload all files (including images) to server
// This mirrors OpenWebUI's approach: images are uploaded to /api/v1/files/
// and the server resolves them when sending to LLM
final uploader = AttachmentUploadQueue();
try {
final api = _ref.read(apiServiceProvider);
@@ -93,15 +92,47 @@ class TaskWorker {
}
} catch (_) {}
// For images: convert unsupported formats and optimize large JPEG/PNG
String uploadPath = task.filePath;
String uploadFileName = task.fileName;
String? uploadMimeType = task.mimeType;
if (isImage) {
final shouldConvert = await _shouldConvertImage(lowerName, task.fileSize);
if (shouldConvert) {
final convertedPath = await _convertImageForUpload(task);
if (convertedPath != null) {
uploadPath = convertedPath;
// Update filename to .webp extension since we converted the format
final baseName = task.fileName.contains('.')
? task.fileName.substring(0, task.fileName.lastIndexOf('.'))
: task.fileName;
uploadFileName = '$baseName.webp';
uploadMimeType = 'image/webp';
}
}
}
// Read image bytes before upload for instant display cache
Uint8List? imageBytes;
if (isImage) {
try {
imageBytes = await File(uploadPath).readAsBytes();
} catch (_) {}
}
final id = await uploader.enqueue(
filePath: task.filePath,
fileName: task.fileName,
filePath: uploadPath,
fileName: uploadFileName,
fileSize: task.fileSize ?? 0,
mimeType: task.mimeType,
mimeType: uploadMimeType,
checksum: task.checksum,
);
final completer = Completer<void>();
// Capture values for use in closure
final displayFileName = uploadFileName;
final cachedBytes = imageBytes;
final tempFilePath = uploadPath != task.filePath ? uploadPath : null;
late final StreamSubscription<List<QueuedAttachment>> sub;
sub = uploader.queueStream.listen((items) {
QueuedAttachment? entry;
@@ -124,9 +155,17 @@ class TaskWorker {
QueuedAttachmentStatus.failed => FileUploadStatus.failed,
QueuedAttachmentStatus.cancelled => FileUploadStatus.failed,
};
// Pre-cache image bytes for instant display when upload completes
if (status == FileUploadStatus.completed &&
entry.fileId != null &&
cachedBytes != null) {
preCacheImageBytes(entry.fileId!, cachedBytes);
}
final newState = FileUploadState(
file: File(task.filePath),
fileName: task.fileName,
fileName: displayFileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: status == FileUploadStatus.completed
? 1.0
@@ -134,7 +173,7 @@ class TaskWorker {
status: status,
fileId: entry.fileId ?? existing.fileId,
error: entry.lastError,
isImage: false,
isImage: isImage,
);
_ref
.read(attachedFilesProvider.notifier)
@@ -146,6 +185,12 @@ class TaskWorker {
case QueuedAttachmentStatus.failed:
case QueuedAttachmentStatus.cancelled:
sub.cancel();
// Clean up temp file from image conversion
if (tempFilePath != null) {
try {
File(tempFilePath).parent.deleteSync(recursive: true);
} catch (_) {}
}
completer.complete();
break;
default:
@@ -160,73 +205,104 @@ class TaskWorker {
try {
sub.cancel();
} catch (_) {}
// Clean up temp file on timeout
if (tempFilePath != null) {
try {
File(tempFilePath).parent.deleteSync(recursive: true);
} catch (_) {}
}
// Update state to failed on timeout
try {
final current = _ref.read(attachedFilesProvider);
final idx = current.indexWhere((f) => f.file.path == task.filePath);
if (idx != -1) {
final existing = current[idx];
final newState = FileUploadState(
file: File(task.filePath),
fileName: displayFileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: 0.0,
status: FileUploadStatus.failed,
error: 'Upload timed out',
isImage: isImage,
);
_ref
.read(attachedFilesProvider.notifier)
.updateFileState(task.filePath, newState);
}
} catch (_) {}
DebugLogger.warning('UploadMediaTask timed out: ${task.fileName}');
return;
},
);
}
/// Handles image files by reading as base64 locally (matching web client)
Future<void> _handleImageAsBase64(UploadMediaTask task) async {
/// Check if image should be converted to WebP before upload
/// - Always convert: HEIC, RAW formats, BMP (unsupported or inefficient)
/// - Optimize if large: JPEG, PNG > 500KB
/// - Never convert: WebP (already optimal), GIF (may be animated)
Future<bool> _shouldConvertImage(String lowerName, int? fileSize) async {
// Always convert these formats (unsupported by some backends or inefficient)
const alwaysConvert = {
'.heic', '.heif', '.dng', '.raw', '.cr2', '.nef', '.arw', '.orf', '.rw2', '.bmp',
};
if (alwaysConvert.any(lowerName.endsWith)) {
return true;
}
// Never convert these (already optimal or special format)
const neverConvert = {'.webp', '.gif'};
if (neverConvert.any(lowerName.endsWith)) {
return false;
}
// Optimize large JPEG/PNG (> 500KB)
const optimizeThreshold = 500 * 1024; // 500KB
const optimizableFormats = {'.jpg', '.jpeg', '.png'};
if (optimizableFormats.any(lowerName.endsWith)) {
final size = fileSize ?? 0;
return size > optimizeThreshold;
}
return false;
}
/// Convert image to WebP for upload if needed
Future<String?> _convertImageForUpload(UploadMediaTask task) async {
try {
final file = File(task.filePath);
final base64DataUrl = await convertImageFileToDataUrl(file);
if (base64DataUrl == null) {
throw Exception('Failed to convert image to base64');
}
// Update attachment state with base64 data URL
final current = _ref.read(attachedFilesProvider);
final idx = current.indexWhere((f) => f.file.path == task.filePath);
if (idx != -1) {
final existing = current[idx];
final newState = FileUploadState(
file: file,
fileName: task.fileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: 1.0,
status: FileUploadStatus.completed,
fileId: base64DataUrl,
isImage: true,
base64DataUrl: base64DataUrl,
);
_ref
.read(attachedFilesProvider.notifier)
.updateFileState(task.filePath, newState);
}
DebugLogger.log(
'image-base64-complete',
scope: 'tasks/upload',
data: {
'fileName': task.fileName,
'dataUrlLength': base64DataUrl.length,
},
final result = await FlutterImageCompress.compressWithFile(
file.absolute.path,
format: CompressFormat.webp,
quality: 85,
);
if (result != null && result.isNotEmpty) {
// Write to temp file for upload
final tempDir = await Directory.systemTemp.createTemp('conduit_img_');
final tempFile = File('${tempDir.path}/converted.webp');
await tempFile.writeAsBytes(result);
DebugLogger.log(
'Converted image for upload',
scope: 'tasks/upload',
data: {
'original': task.filePath,
'converted': tempFile.path,
'originalSize': await file.length(),
'convertedSize': result.length,
},
);
return tempFile.path;
}
} catch (e) {
DebugLogger.error('image-base64-failed', scope: 'tasks/upload', error: e);
// Update state to failed
try {
final current = _ref.read(attachedFilesProvider);
final idx = current.indexWhere((f) => f.file.path == task.filePath);
if (idx != -1) {
final existing = current[idx];
final newState = FileUploadState(
file: File(task.filePath),
fileName: task.fileName,
fileSize: task.fileSize ?? existing.fileSize,
progress: 0.0,
status: FileUploadStatus.failed,
error: e.toString(),
isImage: true,
);
_ref
.read(attachedFilesProvider.notifier)
.updateFileState(task.filePath, newState);
}
} catch (_) {}
DebugLogger.error('image-conversion-failed', scope: 'tasks/upload', error: e);
}
return null;
}
Future<void> _performExecuteToolCall(ExecuteToolCallTask task) async {
@@ -322,7 +398,11 @@ class TaskWorker {
// Convert image to base64 data URL locally (matching web client behavior)
try {
final file = File(task.filePath);
final base64DataUrl = await convertImageFileToDataUrl(file);
final worker = _ref.read(workerManagerProvider);
final base64DataUrl = await convertImageFileToDataUrl(
file,
worker: worker,
);
if (base64DataUrl == null) {
throw Exception('Failed to convert image to base64');

View File

@@ -142,7 +142,7 @@ class CitationBadge extends StatelessWidget {
SourceHelper.launchSourceUrl(url);
}
},
borderRadius: BorderRadius.circular(AppBorderRadius.chip),
borderRadius: BorderRadius.circular(20),
child: Container(
padding: const EdgeInsets.symmetric(
horizontal: Spacing.sm,
@@ -150,33 +150,22 @@ class CitationBadge extends StatelessWidget {
),
margin: const EdgeInsets.symmetric(horizontal: 2),
decoration: BoxDecoration(
color: theme.surfaceContainer.withValues(alpha: 0.6),
borderRadius: BorderRadius.circular(AppBorderRadius.chip),
color: theme.surfaceContainer.withValues(alpha: 0.3),
borderRadius: BorderRadius.circular(20),
border: Border.all(
color: theme.cardBorder.withValues(alpha: 0.5),
width: BorderWidth.thin,
color: theme.dividerColor.withValues(alpha: 0.5),
width: 1,
),
),
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
Icon(
Icons.link_rounded,
size: 10,
color: theme.textSecondary.withValues(alpha: 0.7),
),
const SizedBox(width: Spacing.xxs),
Text(
displayTitle,
style: TextStyle(
fontSize: AppTypography.labelSmall,
fontWeight: FontWeight.w500,
color: theme.textSecondary,
),
maxLines: 1,
overflow: TextOverflow.ellipsis,
),
],
child: Text(
displayTitle,
style: TextStyle(
fontSize: AppTypography.labelSmall,
fontWeight: FontWeight.w500,
color: theme.textSecondary,
),
maxLines: 1,
overflow: TextOverflow.ellipsis,
),
),
),