fix: sources count
This commit is contained in:
cogwheel0
@@ -16,6 +16,7 @@ import '../error/api_error_interceptor.dart';
|
||||
// Tool-call details are parsed in the UI layer to render collapsible blocks
|
||||
import 'persistent_streaming_service.dart';
|
||||
import '../utils/debug_logger.dart';
|
||||
import '../utils/openwebui_source_parser.dart';
|
||||
|
||||
const bool _traceApiLogs = false;
|
||||
const bool _traceConversationParsing = false;
|
||||
@@ -1284,64 +1285,11 @@ class ApiService {
|
||||
}
|
||||
|
||||
List<ChatSourceReference> _parseSourcesField(dynamic raw) {
|
||||
if (raw is List) {
|
||||
return raw
|
||||
.whereType<Map>()
|
||||
.map((entry) {
|
||||
try {
|
||||
// Convert Map to Map<String, dynamic> safely
|
||||
final Map<String, dynamic> entryMap = {};
|
||||
entry.forEach((key, value) {
|
||||
entryMap[key.toString()] = value;
|
||||
});
|
||||
|
||||
// Handle nested source structure from OpenWebUI
|
||||
// Sources can have structure like: { "source": { "name": "...", "id": "..." }, "document": [...], "metadata": [...] }
|
||||
final sourceData = entryMap['source'];
|
||||
if (sourceData is Map) {
|
||||
// Extract the actual source information from nested structure
|
||||
final Map<String, dynamic> sourceMap = {};
|
||||
sourceData.forEach((key, value) {
|
||||
sourceMap[key.toString()] = value;
|
||||
});
|
||||
|
||||
// Add additional metadata from the outer structure if available
|
||||
if (entryMap.containsKey('document') &&
|
||||
entryMap['document'] is List) {
|
||||
final documents = entryMap['document'] as List;
|
||||
if (documents.isNotEmpty) {
|
||||
sourceMap['snippet'] = documents.first?.toString();
|
||||
}
|
||||
}
|
||||
|
||||
if (entryMap.containsKey('metadata') &&
|
||||
entryMap['metadata'] is List) {
|
||||
final metadata = entryMap['metadata'] as List;
|
||||
if (metadata.isNotEmpty && metadata.first is Map) {
|
||||
sourceMap['metadata'] = metadata.first;
|
||||
}
|
||||
}
|
||||
|
||||
return ChatSourceReference.fromJson(sourceMap);
|
||||
} else {
|
||||
// Fallback: treat the entire entry as a source (for backward compatibility)
|
||||
return ChatSourceReference.fromJson(entryMap);
|
||||
}
|
||||
} catch (e) {
|
||||
// Log the error and skip this entry
|
||||
DebugLogger.log(
|
||||
'source-parse-error',
|
||||
scope: 'api/chat',
|
||||
data: {'error': e.toString(), 'entry': entry.toString()},
|
||||
);
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.where((item) => item != null)
|
||||
.cast<ChatSourceReference>()
|
||||
.toList(growable: false);
|
||||
try {
|
||||
return parseOpenWebUISourceList(raw);
|
||||
} catch (_) {
|
||||
return const <ChatSourceReference>[];
|
||||
}
|
||||
return const <ChatSourceReference>[];
|
||||
}
|
||||
|
||||
// Create new conversation using OpenWebUI API
|
||||
|
||||
@@ -12,6 +12,7 @@ import 'navigation_service.dart';
|
||||
import '../../shared/widgets/themed_dialogs.dart';
|
||||
import '../../shared/theme/theme_extensions.dart';
|
||||
import '../utils/debug_logger.dart';
|
||||
import '../utils/openwebui_source_parser.dart';
|
||||
|
||||
// Keep local verbosity toggle for socket logs
|
||||
const bool kSocketVerboseLogging = false;
|
||||
@@ -705,38 +706,17 @@ ActiveSocketStream attachUnifiedChunkedStreaming({
|
||||
} catch (_) {}
|
||||
} else {
|
||||
try {
|
||||
// Handle nested source structure from OpenWebUI
|
||||
final sourceData = map['source'];
|
||||
final ChatSourceReference source;
|
||||
|
||||
if (sourceData is Map<String, dynamic>) {
|
||||
// Extract the actual source information from nested structure
|
||||
final sourceMap = Map<String, dynamic>.from(sourceData);
|
||||
|
||||
// Add additional metadata from the outer structure if available
|
||||
if (map.containsKey('document') && map['document'] is List) {
|
||||
final documents = map['document'] as List;
|
||||
if (documents.isNotEmpty) {
|
||||
sourceMap['snippet'] = documents.first?.toString();
|
||||
final sources = parseOpenWebUISourceList([map]);
|
||||
if (sources.isNotEmpty) {
|
||||
final targetId = _resolveTargetMessageId(
|
||||
messageId,
|
||||
getMessages,
|
||||
);
|
||||
if (targetId != null) {
|
||||
for (final source in sources) {
|
||||
appendSourceReference(targetId, source);
|
||||
}
|
||||
}
|
||||
|
||||
if (map.containsKey('metadata') && map['metadata'] is List) {
|
||||
final metadata = map['metadata'] as List;
|
||||
if (metadata.isNotEmpty && metadata.first is Map) {
|
||||
sourceMap['metadata'] = metadata.first;
|
||||
}
|
||||
}
|
||||
|
||||
source = ChatSourceReference.fromJson(sourceMap);
|
||||
} else {
|
||||
// Fallback: treat the entire map as a source (for backward compatibility)
|
||||
source = ChatSourceReference.fromJson(map);
|
||||
}
|
||||
|
||||
final targetId = _resolveTargetMessageId(messageId, getMessages);
|
||||
if (targetId != null) {
|
||||
appendSourceReference(targetId, source);
|
||||
}
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
228
lib/core/utils/openwebui_source_parser.dart
Normal file
228
lib/core/utils/openwebui_source_parser.dart
Normal file
@@ -0,0 +1,228 @@
|
||||
import '../models/chat_message.dart';
|
||||
|
||||
/// Parses OpenWebUI style source payloads into flattened chat source references.
|
||||
List<ChatSourceReference> parseOpenWebUISourceList(dynamic raw) {
|
||||
if (raw is! List) {
|
||||
return const <ChatSourceReference>[];
|
||||
}
|
||||
|
||||
final aggregated = <String, _CitationAccumulator>{};
|
||||
var fallbackIndex = 0;
|
||||
|
||||
for (final entry in raw) {
|
||||
if (entry is! Map) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final entryMap = _asStringKeyMap(entry);
|
||||
if (entryMap == null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
final baseSource =
|
||||
_asStringKeyMap(entryMap['source']) ?? <String, dynamic>{};
|
||||
entryMap.remove('source');
|
||||
|
||||
for (final key in ['id', 'name', 'title', 'url', 'link', 'type']) {
|
||||
final value = entryMap[key];
|
||||
if (value != null && baseSource[key] == null) {
|
||||
baseSource[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
final documents = entryMap['document'] is List
|
||||
? (entryMap['document'] as List)
|
||||
: const [];
|
||||
final metadataRaw = entryMap['metadata'];
|
||||
final metadataList = metadataRaw is List
|
||||
? metadataRaw
|
||||
: metadataRaw is Map
|
||||
? [metadataRaw]
|
||||
: const [];
|
||||
final distances = entryMap['distances'] is List
|
||||
? (entryMap['distances'] as List)
|
||||
: const [];
|
||||
|
||||
final counts = <int>[
|
||||
documents.length,
|
||||
metadataList.length,
|
||||
distances.length,
|
||||
].where((len) => len > 0).toList();
|
||||
final loopCount = counts.isEmpty
|
||||
? 1
|
||||
: counts.reduce((value, element) => value > element ? value : element);
|
||||
|
||||
for (var index = 0; index < loopCount; index++) {
|
||||
final document = index < documents.length ? documents[index] : null;
|
||||
final metadata = index < metadataList.length ? metadataList[index] : null;
|
||||
final distance = index < distances.length ? distances[index] : null;
|
||||
|
||||
final metadataMap = _asStringKeyMap(metadata) ?? <String, dynamic>{};
|
||||
|
||||
final idCandidate = _firstNonEmpty([
|
||||
metadataMap['source'],
|
||||
metadataMap['id'],
|
||||
baseSource['id'],
|
||||
entryMap['id'],
|
||||
]);
|
||||
|
||||
final key = idCandidate?.isNotEmpty == true
|
||||
? idCandidate!
|
||||
: '__fallback_${fallbackIndex++}';
|
||||
|
||||
final accumulator = aggregated.putIfAbsent(
|
||||
key,
|
||||
() => _CitationAccumulator(
|
||||
key: key,
|
||||
source: Map<String, dynamic>.from(baseSource),
|
||||
),
|
||||
);
|
||||
|
||||
accumulator.explicitId ??= idCandidate?.toString();
|
||||
accumulator.explicitType ??= _firstNonEmpty([
|
||||
baseSource['type'],
|
||||
entryMap['type'],
|
||||
metadataMap['type'],
|
||||
])?.toString();
|
||||
|
||||
final metadataName = _firstNonEmpty([
|
||||
metadataMap['name'],
|
||||
metadataMap['title'],
|
||||
])?.toString();
|
||||
if (metadataName != null && metadataName.isNotEmpty) {
|
||||
accumulator.source['name'] = metadataName;
|
||||
accumulator.source['title'] ??= metadataName;
|
||||
}
|
||||
|
||||
if (_looksLikeUrl(idCandidate)) {
|
||||
accumulator.source['url'] ??= idCandidate;
|
||||
accumulator.source['name'] ??= idCandidate;
|
||||
}
|
||||
|
||||
final metadataUrl = _firstNonEmpty([
|
||||
metadataMap['url'],
|
||||
metadataMap['link'],
|
||||
metadataMap['source'],
|
||||
accumulator.source['url'],
|
||||
])?.toString();
|
||||
if (_looksLikeUrl(metadataUrl)) {
|
||||
accumulator.source['url'] = metadataUrl;
|
||||
}
|
||||
|
||||
final snippet = _extractSnippet(document);
|
||||
if (snippet != null && snippet.isNotEmpty) {
|
||||
accumulator.documents.add(snippet);
|
||||
}
|
||||
|
||||
if (metadataMap.isNotEmpty) {
|
||||
accumulator.metadata.add(metadataMap);
|
||||
}
|
||||
|
||||
if (distance != null) {
|
||||
accumulator.distances.add(distance);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final results = <ChatSourceReference>[];
|
||||
|
||||
for (final accumulator in aggregated.values) {
|
||||
final id = accumulator.explicitId;
|
||||
final title = _firstNonEmpty([
|
||||
accumulator.source['name'],
|
||||
accumulator.source['title'],
|
||||
id,
|
||||
])?.toString();
|
||||
|
||||
final urlCandidate = _firstNonEmpty([
|
||||
accumulator.source['url'],
|
||||
id,
|
||||
])?.toString();
|
||||
final url = _looksLikeUrl(urlCandidate) ? urlCandidate : null;
|
||||
|
||||
final snippet = accumulator.documents.firstWhere(
|
||||
(doc) => doc.trim().isNotEmpty,
|
||||
orElse: () => '',
|
||||
);
|
||||
|
||||
final metadata = <String, dynamic>{
|
||||
if (accumulator.metadata.isNotEmpty) 'items': accumulator.metadata,
|
||||
if (accumulator.documents.isNotEmpty) 'documents': accumulator.documents,
|
||||
if (accumulator.distances.isNotEmpty) 'distances': accumulator.distances,
|
||||
if (accumulator.source.isNotEmpty) 'source': accumulator.source,
|
||||
};
|
||||
|
||||
metadata.removeWhere((key, value) {
|
||||
if (value == null) return true;
|
||||
if (value is List && value.isEmpty) return true;
|
||||
if (value is Map && value.isEmpty) return true;
|
||||
return false;
|
||||
});
|
||||
|
||||
results.add(
|
||||
ChatSourceReference(
|
||||
id: (id != null && id.startsWith('__fallback_')) ? null : id,
|
||||
title: title,
|
||||
url: url,
|
||||
snippet: snippet.isNotEmpty ? snippet : null,
|
||||
type: accumulator.explicitType,
|
||||
metadata: metadata.isNotEmpty ? metadata : null,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
Map<String, dynamic>? _asStringKeyMap(dynamic value) {
|
||||
if (value is Map) {
|
||||
final map = <String, dynamic>{};
|
||||
value.forEach((key, entryValue) {
|
||||
map[key.toString()] = entryValue;
|
||||
});
|
||||
return map;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
String? _firstNonEmpty(Iterable<dynamic> values) {
|
||||
for (final value in values) {
|
||||
if (value == null) {
|
||||
continue;
|
||||
}
|
||||
final stringValue = value.toString();
|
||||
if (stringValue.isNotEmpty) {
|
||||
return stringValue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
bool _looksLikeUrl(String? value) {
|
||||
if (value == null) {
|
||||
return false;
|
||||
}
|
||||
return value.startsWith('http://') || value.startsWith('https://');
|
||||
}
|
||||
|
||||
String? _extractSnippet(dynamic document) {
|
||||
if (document == null) {
|
||||
return null;
|
||||
}
|
||||
if (document is String) {
|
||||
return document.trim();
|
||||
}
|
||||
return document.toString().trim().isNotEmpty ? document.toString() : null;
|
||||
}
|
||||
|
||||
class _CitationAccumulator {
|
||||
_CitationAccumulator({required this.key, required this.source});
|
||||
|
||||
final String key;
|
||||
final Map<String, dynamic> source;
|
||||
final List<String> documents = [];
|
||||
final List<Map<String, dynamic>> metadata = [];
|
||||
final List<dynamic> distances = [];
|
||||
String? explicitId;
|
||||
String? explicitType;
|
||||
}
|
||||
Reference in New Issue
Block a user