Line data Source code
1 : import 'dart:convert';
2 :
3 : import 'package:matrix/matrix_api_lite.dart';
4 : import 'package:matrix/src/event.dart';
5 : import 'package:matrix/src/timeline.dart';
6 :
7 : extension TimelineExportExtension on Timeline {
8 : /// Exports timeline events from a Matrix room within a specified date range.
9 : ///
10 : /// The export process provides progress updates through the returned stream with the following information:
11 : /// - Total number of events exported
12 : /// - Count of unable-to-decrypt (UTD) events
13 : /// - Count of media events (images, audio, video, files)
14 : /// - Number of unique users involved
15 : ///
16 : /// ```dart
17 : /// // Example usage:
18 : /// final timeline = room.timeline;
19 : /// final oneWeekAgo = DateTime.now().subtract(Duration(days: 7));
20 : ///
21 : /// // Export last week's messages, excluding encrypted events
22 : /// await for (final result in timeline.export(
23 : /// from: oneWeekAgo,
24 : /// filter: (event) => event?.type != EventTypes.Encrypted,
25 : /// )) {
26 : /// if (result is ExportProgress) {
27 : /// print('Progress: ${result.totalEvents} events exported');
28 : /// } else if (result is ExportComplete) {
29 : /// print('Export completed with ${result.events.length} events');
30 : /// } else if (result is ExportError) {
31 : /// print('Export failed: ${result.error}');
32 : /// }
33 : /// }
34 : /// ```
35 : ///
36 : /// [from] Optional start date to filter events. If null, exports from the beginning.
37 : /// [until] Optional end date to filter events. If null, exports up to the latest event.
38 : /// [filter] Optional function to filter events. Return true to include the event.
39 : /// [requestHistoryCount] Optional. The number of events to request from the server at once.
40 : ///
41 : /// Returns a [Stream] of [ExportResult] which can be:
42 : /// - [ExportProgress]: Provides progress updates during export
43 : /// - [ExportComplete]: Contains the final list of exported events
44 : /// - [ExportError]: Contains error information if export fails
45 2 : Stream<ExportResult> export({
46 : DateTime? from,
47 : DateTime? until,
48 : bool Function(Event)? filter,
49 : int requestHistoryCount = 500,
50 : }) async* {
51 2 : final eventsToExport = <Event>[];
52 : var utdEventsCount = 0;
53 : var mediaEventsCount = 0;
54 : final users = <String>{};
55 :
56 : try {
57 2 : yield ExportProgress(
58 : source: ExportSource.timeline,
59 : totalEvents: 0,
60 : utdEvents: 0,
61 : mediaEvents: 0,
62 : users: 0,
63 : );
64 :
65 2 : void exportEvent(Event event) {
66 2 : eventsToExport.add(event);
67 :
68 4 : if (event.type == EventTypes.Encrypted &&
69 4 : event.messageType == MessageTypes.BadEncrypted) {
70 2 : utdEventsCount++;
71 4 : } else if (event.type == EventTypes.Message &&
72 : {
73 2 : MessageTypes.Sticker,
74 2 : MessageTypes.Image,
75 2 : MessageTypes.Audio,
76 2 : MessageTypes.Video,
77 2 : MessageTypes.File,
78 4 : }.contains(event.messageType)) {
79 2 : mediaEventsCount++;
80 : }
81 4 : users.add(event.senderId);
82 : }
83 :
84 : // From the timeline
85 8 : if (until == null || events.last.originServerTs.isBefore(until)) {
86 4 : for (final event in events) {
87 4 : if (from != null && event.originServerTs.isBefore(from)) break;
88 4 : if (until != null && event.originServerTs.isAfter(until)) continue;
89 0 : if (filter != null && !filter(event)) continue;
90 2 : exportEvent(event);
91 : }
92 : }
93 2 : yield ExportProgress(
94 : source: ExportSource.timeline,
95 2 : totalEvents: eventsToExport.length,
96 : utdEvents: utdEventsCount,
97 : mediaEvents: mediaEventsCount,
98 2 : users: users.length,
99 : );
100 :
101 8 : if (from != null && events.last.originServerTs.isBefore(from)) {
102 0 : yield ExportComplete(
103 : events: eventsToExport,
104 0 : totalEvents: eventsToExport.length,
105 : utdEvents: utdEventsCount,
106 : mediaEvents: mediaEventsCount,
107 0 : users: users.length,
108 : );
109 : return;
110 : }
111 :
112 : // From the database
113 : final eventsFromStore =
114 14 : await room.client.database.getEventList(room, start: events.length);
115 2 : if (eventsFromStore.isNotEmpty) {
116 : if (until == null ||
117 6 : eventsFromStore.last.originServerTs.isBefore(until)) {
118 4 : for (final event in eventsFromStore) {
119 4 : if (from != null && event.originServerTs.isBefore(from)) break;
120 4 : if (until != null && event.originServerTs.isAfter(until)) continue;
121 0 : if (filter != null && !filter(event)) continue;
122 2 : exportEvent(event);
123 : }
124 : }
125 2 : yield ExportProgress(
126 : source: ExportSource.database,
127 2 : totalEvents: eventsToExport.length,
128 : utdEvents: utdEventsCount,
129 : mediaEvents: mediaEventsCount,
130 2 : users: users.length,
131 : );
132 :
133 : if (from != null &&
134 6 : eventsFromStore.last.originServerTs.isBefore(from)) {
135 2 : yield ExportComplete(
136 : events: eventsToExport,
137 2 : totalEvents: eventsToExport.length,
138 : utdEvents: utdEventsCount,
139 : mediaEvents: mediaEventsCount,
140 2 : users: users.length,
141 : );
142 : return;
143 : }
144 : }
145 :
146 : // From the server
147 4 : var prevBatch = room.prev_batch;
148 6 : final encryption = room.client.encryption;
149 : do {
150 : if (prevBatch == null) break;
151 : try {
152 6 : final resp = await room.client.getRoomEvents(
153 4 : room.id,
154 : Direction.b,
155 : from: prevBatch,
156 : limit: requestHistoryCount,
157 6 : filter: jsonEncode(StateFilter(lazyLoadMembers: true).toJson()),
158 : );
159 4 : if (resp.chunk.isEmpty) break;
160 :
161 4 : for (final matrixEvent in resp.chunk) {
162 4 : var event = Event.fromMatrixEvent(matrixEvent, room);
163 4 : if (event.type == EventTypes.Encrypted && encryption != null) {
164 0 : event = await encryption.decryptRoomEvent(event);
165 0 : if (event.type == EventTypes.Encrypted &&
166 0 : event.messageType == MessageTypes.BadEncrypted &&
167 0 : event.content['can_request_session'] == true) {
168 : // Await requestKey() here to ensure decrypted message bodies
169 0 : await event.requestKey().catchError((_) {});
170 : }
171 : }
172 0 : if (from != null && event.originServerTs.isBefore(from)) break;
173 0 : if (until != null && event.originServerTs.isAfter(until)) continue;
174 0 : if (filter != null && !filter(event)) continue;
175 2 : exportEvent(event);
176 : }
177 2 : yield ExportProgress(
178 : source: ExportSource.server,
179 2 : totalEvents: eventsToExport.length,
180 : utdEvents: utdEventsCount,
181 : mediaEvents: mediaEventsCount,
182 2 : users: users.length,
183 : );
184 :
185 2 : prevBatch = resp.end;
186 6 : if (resp.chunk.length < requestHistoryCount) break;
187 :
188 0 : if (from != null && resp.chunk.last.originServerTs.isBefore(from)) {
189 : break;
190 : }
191 2 : } on MatrixException catch (e) {
192 : // We have no permission anymore to request the history, so we stop here
193 : // and return the events we have so far
194 4 : if (e.error == MatrixError.M_FORBIDDEN) {
195 : break;
196 : }
197 : // If it's not a forbidden error, we yield an [ExportError]
198 : rethrow;
199 : }
200 : } while (true);
201 :
202 2 : yield ExportComplete(
203 : events: eventsToExport,
204 2 : totalEvents: eventsToExport.length,
205 : utdEvents: utdEventsCount,
206 : mediaEvents: mediaEventsCount,
207 2 : users: users.length,
208 : );
209 : } catch (e) {
210 0 : yield ExportError(
211 0 : error: e.toString(),
212 0 : totalEvents: eventsToExport.length,
213 : utdEvents: utdEventsCount,
214 : mediaEvents: mediaEventsCount,
215 0 : users: users.length,
216 : );
217 : }
218 : }
219 : }
220 :
221 : /// Base class for export results
222 : sealed class ExportResult {
223 : /// Total events count
224 : final int totalEvents;
225 :
226 : /// Unable-to-decrypt events count
227 : final int utdEvents;
228 :
229 : /// Media events count
230 : final int mediaEvents;
231 :
232 : /// Users count
233 : final int users;
234 :
235 2 : ExportResult({
236 : required this.totalEvents,
237 : required this.utdEvents,
238 : required this.mediaEvents,
239 : required this.users,
240 : });
241 : }
242 :
243 : enum ExportSource {
244 : timeline,
245 : database,
246 : server,
247 : }
248 :
249 : /// Represents progress during export
250 : final class ExportProgress extends ExportResult {
251 : /// Export source
252 : final ExportSource source;
253 :
254 2 : ExportProgress({
255 : required this.source,
256 : required super.totalEvents,
257 : required super.utdEvents,
258 : required super.mediaEvents,
259 : required super.users,
260 : });
261 : }
262 :
263 : /// Represents successful completion with exported events
264 : final class ExportComplete extends ExportResult {
265 : final List<Event> events;
266 2 : ExportComplete({
267 : required this.events,
268 : required super.totalEvents,
269 : required super.utdEvents,
270 : required super.mediaEvents,
271 : required super.users,
272 : });
273 : }
274 :
275 : /// Represents an error during export
276 : final class ExportError extends ExportResult {
277 : final String error;
278 0 : ExportError({
279 : required this.error,
280 : required super.totalEvents,
281 : required super.utdEvents,
282 : required super.mediaEvents,
283 : required super.users,
284 : });
285 : }
|
