diff --git a/lib/core/services/api_service.dart b/lib/core/services/api_service.dart index ef1a21b..bcc3c80 100644 --- a/lib/core/services/api_service.dart +++ b/lib/core/services/api_service.dart @@ -16,6 +16,7 @@ import '../error/api_error_interceptor.dart'; // Tool-call details are parsed in the UI layer to render collapsible blocks import 'persistent_streaming_service.dart'; import '../utils/debug_logger.dart'; +import '../utils/openwebui_source_parser.dart'; const bool _traceApiLogs = false; const bool _traceConversationParsing = false; @@ -1284,64 +1285,11 @@ class ApiService { } List _parseSourcesField(dynamic raw) { - if (raw is List) { - return raw - .whereType() - .map((entry) { - try { - // Convert Map to Map safely - final Map entryMap = {}; - entry.forEach((key, value) { - entryMap[key.toString()] = value; - }); - - // Handle nested source structure from OpenWebUI - // Sources can have structure like: { "source": { "name": "...", "id": "..." }, "document": [...], "metadata": [...] } - final sourceData = entryMap['source']; - if (sourceData is Map) { - // Extract the actual source information from nested structure - final Map sourceMap = {}; - sourceData.forEach((key, value) { - sourceMap[key.toString()] = value; - }); - - // Add additional metadata from the outer structure if available - if (entryMap.containsKey('document') && - entryMap['document'] is List) { - final documents = entryMap['document'] as List; - if (documents.isNotEmpty) { - sourceMap['snippet'] = documents.first?.toString(); - } - } - - if (entryMap.containsKey('metadata') && - entryMap['metadata'] is List) { - final metadata = entryMap['metadata'] as List; - if (metadata.isNotEmpty && metadata.first is Map) { - sourceMap['metadata'] = metadata.first; - } - } - - return ChatSourceReference.fromJson(sourceMap); - } else { - // Fallback: treat the entire entry as a source (for backward compatibility) - return ChatSourceReference.fromJson(entryMap); - } - } catch (e) { - // Log the error and skip this entry - DebugLogger.log( - 'source-parse-error', - scope: 'api/chat', - data: {'error': e.toString(), 'entry': entry.toString()}, - ); - return null; - } - }) - .where((item) => item != null) - .cast() - .toList(growable: false); + try { + return parseOpenWebUISourceList(raw); + } catch (_) { + return const []; } - return const []; } // Create new conversation using OpenWebUI API diff --git a/lib/core/services/streaming_helper.dart b/lib/core/services/streaming_helper.dart index c1cc1fc..168c188 100644 --- a/lib/core/services/streaming_helper.dart +++ b/lib/core/services/streaming_helper.dart @@ -12,6 +12,7 @@ import 'navigation_service.dart'; import '../../shared/widgets/themed_dialogs.dart'; import '../../shared/theme/theme_extensions.dart'; import '../utils/debug_logger.dart'; +import '../utils/openwebui_source_parser.dart'; // Keep local verbosity toggle for socket logs const bool kSocketVerboseLogging = false; @@ -705,38 +706,17 @@ ActiveSocketStream attachUnifiedChunkedStreaming({ } catch (_) {} } else { try { - // Handle nested source structure from OpenWebUI - final sourceData = map['source']; - final ChatSourceReference source; - - if (sourceData is Map) { - // Extract the actual source information from nested structure - final sourceMap = Map.from(sourceData); - - // Add additional metadata from the outer structure if available - if (map.containsKey('document') && map['document'] is List) { - final documents = map['document'] as List; - if (documents.isNotEmpty) { - sourceMap['snippet'] = documents.first?.toString(); + final sources = parseOpenWebUISourceList([map]); + if (sources.isNotEmpty) { + final targetId = _resolveTargetMessageId( + messageId, + getMessages, + ); + if (targetId != null) { + for (final source in sources) { + appendSourceReference(targetId, source); } } - - if (map.containsKey('metadata') && map['metadata'] is List) { - final metadata = map['metadata'] as List; - if (metadata.isNotEmpty && metadata.first is Map) { - sourceMap['metadata'] = metadata.first; - } - } - - source = ChatSourceReference.fromJson(sourceMap); - } else { - // Fallback: treat the entire map as a source (for backward compatibility) - source = ChatSourceReference.fromJson(map); - } - - final targetId = _resolveTargetMessageId(messageId, getMessages); - if (targetId != null) { - appendSourceReference(targetId, source); } } catch (_) {} } diff --git a/lib/core/utils/openwebui_source_parser.dart b/lib/core/utils/openwebui_source_parser.dart new file mode 100644 index 0000000..00f326b --- /dev/null +++ b/lib/core/utils/openwebui_source_parser.dart @@ -0,0 +1,228 @@ +import '../models/chat_message.dart'; + +/// Parses OpenWebUI style source payloads into flattened chat source references. +List parseOpenWebUISourceList(dynamic raw) { + if (raw is! List) { + return const []; + } + + final aggregated = {}; + var fallbackIndex = 0; + + for (final entry in raw) { + if (entry is! Map) { + continue; + } + + final entryMap = _asStringKeyMap(entry); + if (entryMap == null) { + continue; + } + + final baseSource = + _asStringKeyMap(entryMap['source']) ?? {}; + entryMap.remove('source'); + + for (final key in ['id', 'name', 'title', 'url', 'link', 'type']) { + final value = entryMap[key]; + if (value != null && baseSource[key] == null) { + baseSource[key] = value; + } + } + + final documents = entryMap['document'] is List + ? (entryMap['document'] as List) + : const []; + final metadataRaw = entryMap['metadata']; + final metadataList = metadataRaw is List + ? metadataRaw + : metadataRaw is Map + ? [metadataRaw] + : const []; + final distances = entryMap['distances'] is List + ? (entryMap['distances'] as List) + : const []; + + final counts = [ + documents.length, + metadataList.length, + distances.length, + ].where((len) => len > 0).toList(); + final loopCount = counts.isEmpty + ? 1 + : counts.reduce((value, element) => value > element ? value : element); + + for (var index = 0; index < loopCount; index++) { + final document = index < documents.length ? documents[index] : null; + final metadata = index < metadataList.length ? metadataList[index] : null; + final distance = index < distances.length ? distances[index] : null; + + final metadataMap = _asStringKeyMap(metadata) ?? {}; + + final idCandidate = _firstNonEmpty([ + metadataMap['source'], + metadataMap['id'], + baseSource['id'], + entryMap['id'], + ]); + + final key = idCandidate?.isNotEmpty == true + ? idCandidate! + : '__fallback_${fallbackIndex++}'; + + final accumulator = aggregated.putIfAbsent( + key, + () => _CitationAccumulator( + key: key, + source: Map.from(baseSource), + ), + ); + + accumulator.explicitId ??= idCandidate?.toString(); + accumulator.explicitType ??= _firstNonEmpty([ + baseSource['type'], + entryMap['type'], + metadataMap['type'], + ])?.toString(); + + final metadataName = _firstNonEmpty([ + metadataMap['name'], + metadataMap['title'], + ])?.toString(); + if (metadataName != null && metadataName.isNotEmpty) { + accumulator.source['name'] = metadataName; + accumulator.source['title'] ??= metadataName; + } + + if (_looksLikeUrl(idCandidate)) { + accumulator.source['url'] ??= idCandidate; + accumulator.source['name'] ??= idCandidate; + } + + final metadataUrl = _firstNonEmpty([ + metadataMap['url'], + metadataMap['link'], + metadataMap['source'], + accumulator.source['url'], + ])?.toString(); + if (_looksLikeUrl(metadataUrl)) { + accumulator.source['url'] = metadataUrl; + } + + final snippet = _extractSnippet(document); + if (snippet != null && snippet.isNotEmpty) { + accumulator.documents.add(snippet); + } + + if (metadataMap.isNotEmpty) { + accumulator.metadata.add(metadataMap); + } + + if (distance != null) { + accumulator.distances.add(distance); + } + } + } + + final results = []; + + for (final accumulator in aggregated.values) { + final id = accumulator.explicitId; + final title = _firstNonEmpty([ + accumulator.source['name'], + accumulator.source['title'], + id, + ])?.toString(); + + final urlCandidate = _firstNonEmpty([ + accumulator.source['url'], + id, + ])?.toString(); + final url = _looksLikeUrl(urlCandidate) ? urlCandidate : null; + + final snippet = accumulator.documents.firstWhere( + (doc) => doc.trim().isNotEmpty, + orElse: () => '', + ); + + final metadata = { + if (accumulator.metadata.isNotEmpty) 'items': accumulator.metadata, + if (accumulator.documents.isNotEmpty) 'documents': accumulator.documents, + if (accumulator.distances.isNotEmpty) 'distances': accumulator.distances, + if (accumulator.source.isNotEmpty) 'source': accumulator.source, + }; + + metadata.removeWhere((key, value) { + if (value == null) return true; + if (value is List && value.isEmpty) return true; + if (value is Map && value.isEmpty) return true; + return false; + }); + + results.add( + ChatSourceReference( + id: (id != null && id.startsWith('__fallback_')) ? null : id, + title: title, + url: url, + snippet: snippet.isNotEmpty ? snippet : null, + type: accumulator.explicitType, + metadata: metadata.isNotEmpty ? metadata : null, + ), + ); + } + + return results; +} + +Map? _asStringKeyMap(dynamic value) { + if (value is Map) { + final map = {}; + value.forEach((key, entryValue) { + map[key.toString()] = entryValue; + }); + return map; + } + return null; +} + +String? _firstNonEmpty(Iterable values) { + for (final value in values) { + if (value == null) { + continue; + } + final stringValue = value.toString(); + if (stringValue.isNotEmpty) { + return stringValue; + } + } + return null; +} + +bool _looksLikeUrl(String? value) { + if (value == null) { + return false; + } + return value.startsWith('http://') || value.startsWith('https://'); +} + +String? _extractSnippet(dynamic document) { + if (document == null) { + return null; + } + if (document is String) { + return document.trim(); + } + return document.toString().trim().isNotEmpty ? document.toString() : null; +} + +class _CitationAccumulator { + _CitationAccumulator({required this.key, required this.source}); + + final String key; + final Map source; + final List documents = []; + final List> metadata = []; + final List distances = []; + String? explicitId; + String? explicitType; +} diff --git a/lib/features/chat/widgets/sources/openwebui_sources.dart b/lib/features/chat/widgets/sources/openwebui_sources.dart index 585097e..808051e 100644 --- a/lib/features/chat/widgets/sources/openwebui_sources.dart +++ b/lib/features/chat/widgets/sources/openwebui_sources.dart @@ -337,9 +337,6 @@ class _OpenWebUISourcesWidgetState extends State { source.metadata!['link']?.toString(); } } - debugPrint( - '_getSourceUrl: source.url=${source.url}, metadata_source=${source.metadata?['source']}, final_url=$url', - ); return url; }