Line data Source code
1 : import 'dart:convert';
2 :
3 : import 'package:matrix/matrix_api_lite.dart';
4 : import 'package:matrix/src/event.dart';
5 : import 'package:matrix/src/timeline.dart';
6 :
7 : extension TimelineExportExtension on Timeline {
8 : /// Exports timeline events from a Matrix room within a specified date range.
9 : ///
10 : /// The export process provides progress updates through the returned stream with the following information:
11 : /// - Total number of events exported
12 : /// - Count of unable-to-decrypt (UTD) events
13 : /// - Count of media events (images, audio, video, files)
14 : /// - Number of unique users involved
15 : ///
16 : /// ```dart
17 : /// // Example usage:
18 : /// final timeline = room.timeline;
19 : /// final oneWeekAgo = DateTime.now().subtract(Duration(days: 7));
20 : ///
21 : /// // Export last week's messages, excluding encrypted events
22 : /// await for (final result in timeline.export(
23 : /// from: oneWeekAgo,
24 : /// filter: (event) => event?.type != EventTypes.Encrypted,
25 : /// )) {
26 : /// if (result is ExportProgress) {
27 : /// print('Progress: ${result.totalEvents} events exported');
28 : /// } else if (result is ExportComplete) {
29 : /// print('Export completed with ${result.events.length} events');
30 : /// } else if (result is ExportError) {
31 : /// print('Export failed: ${result.error}');
32 : /// }
33 : /// }
34 : /// ```
35 : ///
36 : /// [from] Optional start date to filter events. If null, exports from the beginning.
37 : /// [until] Optional end date to filter events. If null, exports up to the latest event.
38 : /// [filter] Optional function to filter events. Return true to include the event.
39 : /// [requestHistoryCount] Optional. The number of events to request from the server at once.
40 : ///
41 : /// Returns a [Stream] of [ExportResult] which can be:
42 : /// - [ExportProgress]: Provides progress updates during export
43 : /// - [ExportComplete]: Contains the final list of exported events
44 : /// - [ExportError]: Contains error information if export fails
45 2 : Stream<ExportResult> export({
46 : DateTime? from,
47 : DateTime? until,
48 : bool Function(Event)? filter,
49 : int requestHistoryCount = 500,
50 : }) async* {
51 2 : final eventsToExport = <Event>[];
52 : var utdEventsCount = 0;
53 : var mediaEventsCount = 0;
54 : final users = <String>{};
55 :
56 : try {
57 2 : yield ExportProgress(
58 : source: ExportSource.timeline,
59 : totalEvents: 0,
60 : utdEvents: 0,
61 : mediaEvents: 0,
62 : users: 0,
63 : );
64 :
65 2 : void exportEvent(Event event) {
66 2 : eventsToExport.add(event);
67 :
68 4 : if (event.type == EventTypes.Encrypted &&
69 4 : event.messageType == MessageTypes.BadEncrypted) {
70 2 : utdEventsCount++;
71 4 : } else if (event.type == EventTypes.Message &&
72 : {
73 2 : MessageTypes.Sticker,
74 2 : MessageTypes.Image,
75 2 : MessageTypes.Audio,
76 2 : MessageTypes.Video,
77 2 : MessageTypes.File,
78 4 : }.contains(event.messageType)) {
79 2 : mediaEventsCount++;
80 : }
81 4 : users.add(event.senderId);
82 : }
83 :
84 : // From the timeline
85 8 : if (until == null || events.last.originServerTs.isBefore(until)) {
86 4 : for (final event in events) {
87 4 : if (from != null && event.originServerTs.isBefore(from)) break;
88 4 : if (until != null && event.originServerTs.isAfter(until)) continue;
89 0 : if (filter != null && !filter(event)) continue;
90 2 : exportEvent(event);
91 : }
92 : }
93 2 : yield ExportProgress(
94 : source: ExportSource.timeline,
95 2 : totalEvents: eventsToExport.length,
96 : utdEvents: utdEventsCount,
97 : mediaEvents: mediaEventsCount,
98 2 : users: users.length,
99 : );
100 :
101 8 : if (from != null && events.last.originServerTs.isBefore(from)) {
102 0 : yield ExportComplete(
103 : events: eventsToExport,
104 0 : totalEvents: eventsToExport.length,
105 : utdEvents: utdEventsCount,
106 : mediaEvents: mediaEventsCount,
107 0 : users: users.length,
108 : );
109 : return;
110 : }
111 :
112 : // From the database
113 6 : final eventsFromStore = await room.client.database
114 8 : ?.getEventList(room, start: events.length) ??
115 0 : [];
116 2 : if (eventsFromStore.isNotEmpty) {
117 : if (until == null ||
118 6 : eventsFromStore.last.originServerTs.isBefore(until)) {
119 4 : for (final event in eventsFromStore) {
120 4 : if (from != null && event.originServerTs.isBefore(from)) break;
121 4 : if (until != null && event.originServerTs.isAfter(until)) continue;
122 0 : if (filter != null && !filter(event)) continue;
123 2 : exportEvent(event);
124 : }
125 : }
126 2 : yield ExportProgress(
127 : source: ExportSource.database,
128 2 : totalEvents: eventsToExport.length,
129 : utdEvents: utdEventsCount,
130 : mediaEvents: mediaEventsCount,
131 2 : users: users.length,
132 : );
133 :
134 : if (from != null &&
135 6 : eventsFromStore.last.originServerTs.isBefore(from)) {
136 2 : yield ExportComplete(
137 : events: eventsToExport,
138 2 : totalEvents: eventsToExport.length,
139 : utdEvents: utdEventsCount,
140 : mediaEvents: mediaEventsCount,
141 2 : users: users.length,
142 : );
143 : return;
144 : }
145 : }
146 :
147 : // From the server
148 4 : var prevBatch = room.prev_batch;
149 6 : final encryption = room.client.encryption;
150 : do {
151 : if (prevBatch == null) break;
152 : try {
153 6 : final resp = await room.client.getRoomEvents(
154 4 : room.id,
155 : Direction.b,
156 : from: prevBatch,
157 : limit: requestHistoryCount,
158 6 : filter: jsonEncode(StateFilter(lazyLoadMembers: true).toJson()),
159 : );
160 4 : if (resp.chunk.isEmpty) break;
161 :
162 4 : for (final matrixEvent in resp.chunk) {
163 4 : var event = Event.fromMatrixEvent(matrixEvent, room);
164 4 : if (event.type == EventTypes.Encrypted && encryption != null) {
165 0 : event = await encryption.decryptRoomEvent(event);
166 0 : if (event.type == EventTypes.Encrypted &&
167 0 : event.messageType == MessageTypes.BadEncrypted &&
168 0 : event.content['can_request_session'] == true) {
169 : // Await requestKey() here to ensure decrypted message bodies
170 0 : await event.requestKey().catchError((_) {});
171 : }
172 : }
173 0 : if (from != null && event.originServerTs.isBefore(from)) break;
174 0 : if (until != null && event.originServerTs.isAfter(until)) continue;
175 0 : if (filter != null && !filter(event)) continue;
176 2 : exportEvent(event);
177 : }
178 2 : yield ExportProgress(
179 : source: ExportSource.server,
180 2 : totalEvents: eventsToExport.length,
181 : utdEvents: utdEventsCount,
182 : mediaEvents: mediaEventsCount,
183 2 : users: users.length,
184 : );
185 :
186 2 : prevBatch = resp.end;
187 6 : if (resp.chunk.length < requestHistoryCount) break;
188 :
189 0 : if (from != null && resp.chunk.last.originServerTs.isBefore(from)) {
190 : break;
191 : }
192 2 : } on MatrixException catch (e) {
193 : // We have no permission anymore to request the history, so we stop here
194 : // and return the events we have so far
195 4 : if (e.error == MatrixError.M_FORBIDDEN) {
196 : break;
197 : }
198 : // If it's not a forbidden error, we yield an [ExportError]
199 : rethrow;
200 : }
201 : } while (true);
202 :
203 2 : yield ExportComplete(
204 : events: eventsToExport,
205 2 : totalEvents: eventsToExport.length,
206 : utdEvents: utdEventsCount,
207 : mediaEvents: mediaEventsCount,
208 2 : users: users.length,
209 : );
210 : } catch (e) {
211 0 : yield ExportError(
212 0 : error: e.toString(),
213 0 : totalEvents: eventsToExport.length,
214 : utdEvents: utdEventsCount,
215 : mediaEvents: mediaEventsCount,
216 0 : users: users.length,
217 : );
218 : }
219 : }
220 : }
221 :
222 : /// Base class for export results
223 : sealed class ExportResult {
224 : /// Total events count
225 : final int totalEvents;
226 :
227 : /// Unable-to-decrypt events count
228 : final int utdEvents;
229 :
230 : /// Media events count
231 : final int mediaEvents;
232 :
233 : /// Users count
234 : final int users;
235 :
236 2 : ExportResult({
237 : required this.totalEvents,
238 : required this.utdEvents,
239 : required this.mediaEvents,
240 : required this.users,
241 : });
242 : }
243 :
244 : enum ExportSource {
245 : timeline,
246 : database,
247 : server,
248 : }
249 :
250 : /// Represents progress during export
251 : final class ExportProgress extends ExportResult {
252 : /// Export source
253 : final ExportSource source;
254 :
255 2 : ExportProgress({
256 : required this.source,
257 : required super.totalEvents,
258 : required super.utdEvents,
259 : required super.mediaEvents,
260 : required super.users,
261 : });
262 : }
263 :
264 : /// Represents successful completion with exported events
265 : final class ExportComplete extends ExportResult {
266 : final List<Event> events;
267 2 : ExportComplete({
268 : required this.events,
269 : required super.totalEvents,
270 : required super.utdEvents,
271 : required super.mediaEvents,
272 : required super.users,
273 : });
274 : }
275 :
276 : /// Represents an error during export
277 : final class ExportError extends ExportResult {
278 : final String error;
279 0 : ExportError({
280 : required this.error,
281 : required super.totalEvents,
282 : required super.utdEvents,
283 : required super.mediaEvents,
284 : required super.users,
285 : });
286 : }
|
