Line data Source code
1 : import 'dart:async';
2 : import 'dart:convert';
3 : import 'dart:core';
4 :
5 : import 'package:collection/collection.dart';
6 : import 'package:sdp_transform/sdp_transform.dart' as sdp_transform;
7 : import 'package:webrtc_interface/webrtc_interface.dart';
8 :
9 : import 'package:matrix/matrix.dart';
10 : import 'package:matrix/src/utils/cached_stream_controller.dart';
11 : import 'package:matrix/src/utils/crypto/crypto.dart';
12 : import 'package:matrix/src/voip/models/call_options.dart';
13 : import 'package:matrix/src/voip/models/voip_id.dart';
14 : import 'package:matrix/src/voip/utils/stream_helper.dart';
15 :
16 : /// The parent highlevel voip class, this trnslates matrix events to webrtc methods via
17 : /// `CallSession` or `GroupCallSession` methods
18 : class VoIP {
19 : // used only for internal tests, all txids for call events will be overwritten to this
20 : static String? customTxid;
21 :
22 : /// set to true if you want to use the ratcheting mechanism with your keyprovider
23 : /// remember to set the window size correctly on your keyprovider
24 : ///
25 : /// at client level because reinitializing a `GroupCallSession` and its `KeyProvider`
26 : /// everytime this changed would be a pain
27 : final bool enableSFUE2EEKeyRatcheting;
28 :
29 : /// cached turn creds
30 : TurnServerCredentials? _turnServerCredentials;
31 :
32 12 : Map<VoipId, CallSession> get calls => _calls;
33 : final Map<VoipId, CallSession> _calls = {};
34 :
35 12 : Map<VoipId, GroupCallSession> get groupCalls => _groupCalls;
36 : final Map<VoipId, GroupCallSession> _groupCalls = {};
37 :
38 : /// The stream is used to prepare for incoming peer calls in a mesh call
39 : /// For example, registering listeners
40 : final CachedStreamController<CallSession> onIncomingCallSetup =
41 : CachedStreamController();
42 :
43 : /// The stream is used to signal the start of an incoming peer call in a mesh call
44 : final CachedStreamController<CallSession> onIncomingCallStart =
45 : CachedStreamController();
46 :
47 : VoipId? currentCID;
48 : VoipId? currentGroupCID;
49 :
50 4 : String get localPartyId => currentSessionId;
51 :
52 : final Client client;
53 : final WebRTCDelegate delegate;
54 : final StreamController<GroupCallSession> onIncomingGroupCall =
55 : StreamController();
56 :
57 18 : CallParticipant? get localParticipant => client.isLogged()
58 6 : ? CallParticipant(
59 : this,
60 12 : userId: client.userID!,
61 12 : deviceId: client.deviceID,
62 : )
63 : : null;
64 :
65 : /// map of roomIds to the invites they are currently processing or in a call with
66 : /// used for handling glare in p2p calls
67 4 : Map<String, String> get incomingCallRoomId => _incomingCallRoomId;
68 : final Map<String, String> _incomingCallRoomId = {};
69 :
70 : /// the current instance of voip, changing this will drop any ongoing mesh calls
71 : /// with that sessionId
72 : late String currentSessionId;
73 :
74 : /// the following parameters are only used in livekit calls, but cannot be
75 : /// in the LivekitBackend class because that could be created from a pre-existing state event
76 :
77 : /// controls how many key indices can you have before looping back to index 0
78 : /// only used in livekit calls
79 : final int keyRingSize;
80 :
81 : // default values set in super constructor
82 : CallTimeouts? timeouts;
83 :
84 6 : VoIP(
85 : this.client,
86 : this.delegate, {
87 : this.enableSFUE2EEKeyRatcheting = false,
88 : this.keyRingSize = 16,
89 : this.timeouts,
90 6 : }) : super() {
91 12 : timeouts ??= CallTimeouts();
92 18 : currentSessionId = base64Encode(secureRandomBytes(16));
93 24 : Logs().v('set currentSessionId to $currentSessionId');
94 : // to populate groupCalls with already present calls
95 18 : for (final room in client.rooms) {
96 6 : final memsList = room.getCallMembershipsFromRoom(this);
97 6 : for (final mems in memsList.values) {
98 0 : for (final mem in mems) {
99 0 : unawaited(createGroupCallFromRoomStateEvent(mem));
100 : }
101 : }
102 : }
103 :
104 : /// handles events todevice and matrix events for invite, candidates, hangup, etc.
105 28 : client.onCallEvents.stream.listen((events) async {
106 4 : await _handleCallEvents(events);
107 : });
108 :
109 : // handles the com.famedly.call events.
110 24 : client.onRoomState.stream.listen(
111 6 : (update) async {
112 : final event = update.state;
113 6 : if (event is! Event) return;
114 18 : if (event.room.membership != Membership.join) return;
115 12 : if (event.type != EventTypes.GroupCallMember) return;
116 :
117 12 : final mems = event.room.getCallMembershipsFromEvent(event, this);
118 12 : for (final mem in mems) {
119 12 : unawaited(createGroupCallFromRoomStateEvent(mem));
120 : }
121 18 : for (final map in groupCalls.entries) {
122 30 : if (map.key.roomId == event.room.id) {
123 : // because we don't know which call got updated, just update all
124 : // group calls we have entered for that room
125 12 : await map.value.onMemberStateChanged();
126 : }
127 : }
128 : },
129 : );
130 :
131 24 : delegate.mediaDevices.ondevicechange = _onDeviceChange;
132 : }
133 :
134 4 : Future<void> _handleCallEvents(List<BasicEventWithSender> callEvents) async {
135 : // Call invites should be omitted for a call that is already answered,
136 : // has ended, is rejectd or replaced.
137 4 : final callEventsCopy = List<BasicEventWithSender>.from(callEvents);
138 8 : for (final callEvent in callEventsCopy) {
139 8 : final callId = callEvent.content.tryGet<String>('call_id');
140 :
141 8 : if (CallConstants.callEndedEventTypes.contains(callEvent.type)) {
142 0 : callEvents.removeWhere((event) {
143 0 : if (CallConstants.omitWhenCallEndedTypes.contains(event.type) &&
144 0 : event.content.tryGet<String>('call_id') == callId) {
145 0 : Logs().v(
146 0 : 'Ommit "${event.type}" event for an already terminated call',
147 : );
148 : return true;
149 : }
150 :
151 : return false;
152 : });
153 : }
154 :
155 : // checks for ended events and removes invites for that call id.
156 4 : if (callEvent is Event) {
157 : // removes expired invites
158 6 : final age = callEvent.unsigned?.tryGet<int>('age') ??
159 12 : (DateTime.now().millisecondsSinceEpoch -
160 8 : callEvent.originServerTs.millisecondsSinceEpoch);
161 :
162 8 : callEvents.removeWhere((element) {
163 8 : if (callEvent.type == EventTypes.CallInvite &&
164 2 : age >
165 4 : (callEvent.content.tryGet<int>('lifetime') ??
166 0 : timeouts!.callInviteLifetime.inMilliseconds)) {
167 4 : Logs().w(
168 4 : '[VOIP] Ommiting invite event ${callEvent.eventId} as age was older than lifetime',
169 : );
170 : return true;
171 : }
172 : return false;
173 : });
174 : }
175 : }
176 :
177 : // and finally call the respective methods on the clean callEvents list
178 8 : for (final callEvent in callEvents) {
179 4 : await _handleCallEvent(callEvent);
180 : }
181 : }
182 :
183 4 : Future<void> _handleCallEvent(BasicEventWithSender event) async {
184 : // member event updates handled in onRoomState for ease
185 8 : if (event.type == EventTypes.GroupCallMember) return;
186 :
187 : GroupCallSession? groupCallSession;
188 : Room? room;
189 4 : final remoteUserId = event.senderId;
190 : String? remoteDeviceId;
191 :
192 4 : if (event is Event) {
193 4 : room = event.room;
194 :
195 8 : if (event.type == EventTypes.GroupCallMemberReaction) {
196 4 : final isEphemeral = event.content.tryGet<bool>('is_ephemeral')!;
197 : if (isEphemeral &&
198 4 : event.originServerTs.isBefore(
199 4 : DateTime.now().subtract(CallConstants.ephemeralReactionTimeout),
200 : )) {
201 4 : Logs().d(
202 4 : '[VOIP] Ignoring ephemeral group call emoji reaction event of type ${event.type} because it is older than ${CallConstants.ephemeralReactionTimeout}',
203 : );
204 : return;
205 : }
206 :
207 : // well this is a bit of a mess, but we use normal Events for reactions,
208 : // therefore have to setup the deviceId here
209 4 : remoteDeviceId = event.content.tryGet<String>('device_id');
210 : } else {
211 : /// this can also be sent in p2p calls when they want to call a specific device
212 8 : remoteDeviceId = event.content.tryGet<String>('invitee_device_id');
213 : }
214 0 : } else if (event is ToDeviceEvent) {
215 0 : final roomId = event.content.tryGet<String>('room_id');
216 0 : final confId = event.content.tryGet<String>('conf_id');
217 :
218 : /// to-device events specifically, m.call.invite and encryption key sending and requesting
219 0 : remoteDeviceId = event.content.tryGet<String>('device_id');
220 :
221 : if (roomId != null && confId != null) {
222 0 : room = client.getRoomById(roomId);
223 0 : groupCallSession = groupCalls[VoipId(roomId: roomId, callId: confId)];
224 : } else {
225 0 : Logs().w(
226 0 : '[VOIP] Ignoring to_device event of type ${event.type} but did not find group call for id: $confId',
227 : );
228 : return;
229 : }
230 :
231 : if (!{
232 0 : EventTypes.GroupCallMemberEncryptionKeys,
233 0 : EventTypes.GroupCallMemberEncryptionKeysRequest,
234 0 : }.contains(event.type)) {
235 : // livekit calls have their own session deduplication logic so ignore sessionId deduplication for them
236 0 : final destSessionId = event.content.tryGet<String>('dest_session_id');
237 0 : if (destSessionId != currentSessionId) {
238 0 : Logs().w(
239 0 : '[VOIP] Ignoring to_device event of type ${event.type} did not match currentSessionId: $currentSessionId, dest_session_id was set to $destSessionId',
240 : );
241 : return;
242 : }
243 : } else if (groupCallSession == null || remoteDeviceId == null) {
244 0 : Logs().w(
245 0 : '[VOIP] _handleCallEvent ${event.type} recieved but either groupCall ${groupCallSession?.groupCallId} or deviceId $remoteDeviceId was null, ignoring',
246 : );
247 : return;
248 : }
249 : } else {
250 0 : Logs().w(
251 0 : '[VOIP] _handleCallEvent can only handle Event or ToDeviceEvent, it got ${event.runtimeType}',
252 : );
253 : return;
254 : }
255 :
256 : if (room == null) {
257 0 : Logs().w(
258 : '[VOIP] _handleCallEvent call event does not contain a room_id, ignoring',
259 : );
260 : return;
261 : }
262 :
263 4 : final content = event.content;
264 :
265 8 : if (client.userID != null &&
266 8 : client.deviceID != null &&
267 12 : remoteUserId == client.userID &&
268 6 : remoteDeviceId == client.deviceID) {
269 : // We don't want to ignore group call reactions from our own device because
270 : // we want to show them on the UI
271 6 : if (!{EventTypes.GroupCallMemberReaction}.contains(event.type)) {
272 0 : Logs().v(
273 0 : 'Ignoring call event ${event.type} for room ${room.id} from our own device',
274 : );
275 : return;
276 : }
277 : } else if (!{
278 4 : EventTypes.GroupCallMemberEncryptionKeys,
279 4 : EventTypes.GroupCallMemberEncryptionKeysRequest,
280 4 : EventTypes.GroupCallMemberReaction,
281 4 : EventTypes.Redaction,
282 8 : }.contains(event.type)) {
283 : // skip webrtc event checks on encryption_keys
284 2 : final callId = content['call_id'] as String?;
285 2 : final partyId = content['party_id'] as String?;
286 0 : if (callId == null && event.type.startsWith('m.call')) {
287 0 : Logs().w('Ignoring call event ${event.type} because call_id was null');
288 : return;
289 : }
290 : if (callId != null) {
291 8 : final call = calls[VoipId(roomId: room.id, callId: callId)];
292 : if (call == null &&
293 4 : !{EventTypes.CallInvite, EventTypes.GroupCallMemberInvite}
294 4 : .contains(event.type)) {
295 0 : Logs().w(
296 0 : 'Ignoring call event ${event.type} for room ${room.id} because we do not have the call',
297 : );
298 : return;
299 : } else if (call != null) {
300 : // multiple checks to make sure the events sent are from the expected party
301 8 : if (call.room.id != room.id) {
302 0 : Logs().w(
303 0 : 'Ignoring call event ${event.type} for room ${room.id} claiming to be for call in room ${call.room.id}',
304 : );
305 : return;
306 : }
307 6 : if (call.remoteUserId != null && call.remoteUserId != remoteUserId) {
308 0 : Logs().d(
309 0 : 'Ignoring call event ${event.type} for room ${room.id} from sender $remoteUserId, expected sender: ${call.remoteUserId}',
310 : );
311 : return;
312 : }
313 6 : if (call.remotePartyId != null && call.remotePartyId != partyId) {
314 0 : Logs().w(
315 0 : 'Ignoring call event ${event.type} for room ${room.id} from sender with a different party_id $partyId, expected party_id: ${call.remotePartyId}',
316 : );
317 : return;
318 : }
319 2 : if ((call.remotePartyId != null &&
320 6 : call.remotePartyId == localPartyId)) {
321 0 : Logs().v(
322 0 : 'Ignoring call event ${event.type} for room ${room.id} from our own partyId',
323 : );
324 : return;
325 : }
326 : }
327 : }
328 : }
329 8 : Logs().v(
330 16 : '[VOIP] Handling event of type: ${event.type}, content ${event.content} from sender ${event.senderId} rp: $remoteUserId:$remoteDeviceId',
331 : );
332 :
333 4 : switch (event.type) {
334 4 : case EventTypes.CallInvite:
335 4 : case EventTypes.GroupCallMemberInvite:
336 2 : await onCallInvite(room, remoteUserId, remoteDeviceId, content);
337 : break;
338 4 : case EventTypes.CallAnswer:
339 4 : case EventTypes.GroupCallMemberAnswer:
340 0 : await onCallAnswer(room, remoteUserId, remoteDeviceId, content);
341 : break;
342 4 : case EventTypes.CallCandidates:
343 4 : case EventTypes.GroupCallMemberCandidates:
344 2 : await onCallCandidates(room, content);
345 : break;
346 4 : case EventTypes.CallHangup:
347 4 : case EventTypes.GroupCallMemberHangup:
348 0 : await onCallHangup(room, content);
349 : break;
350 4 : case EventTypes.CallReject:
351 4 : case EventTypes.GroupCallMemberReject:
352 0 : await onCallReject(room, content);
353 : break;
354 4 : case EventTypes.CallNegotiate:
355 4 : case EventTypes.GroupCallMemberNegotiate:
356 0 : await onCallNegotiate(room, content);
357 : break;
358 : // case EventTypes.CallReplaces:
359 : // await onCallReplaces(room, content);
360 : // break;
361 4 : case EventTypes.CallSelectAnswer:
362 2 : case EventTypes.GroupCallMemberSelectAnswer:
363 2 : await onCallSelectAnswer(room, content);
364 : break;
365 2 : case EventTypes.CallSDPStreamMetadataChanged:
366 2 : case EventTypes.CallSDPStreamMetadataChangedPrefix:
367 2 : case EventTypes.GroupCallMemberSDPStreamMetadataChanged:
368 0 : await onSDPStreamMetadataChangedReceived(room, content);
369 : break;
370 2 : case EventTypes.CallAssertedIdentity:
371 2 : case EventTypes.CallAssertedIdentityPrefix:
372 2 : case EventTypes.GroupCallMemberAssertedIdentity:
373 0 : await onAssertedIdentityReceived(room, content);
374 : break;
375 2 : case EventTypes.GroupCallMemberEncryptionKeys:
376 0 : await groupCallSession!.backend.onCallEncryption(
377 : groupCallSession,
378 : remoteUserId,
379 : remoteDeviceId!,
380 : content,
381 : );
382 : break;
383 2 : case EventTypes.GroupCallMemberEncryptionKeysRequest:
384 0 : await groupCallSession!.backend.onCallEncryptionKeyRequest(
385 : groupCallSession,
386 : remoteUserId,
387 : remoteDeviceId!,
388 : content,
389 : );
390 : break;
391 2 : case EventTypes.GroupCallMemberReaction:
392 2 : await _handleReactionEvent(room, event as MatrixEvent);
393 : break;
394 2 : case EventTypes.Redaction:
395 2 : await _handleRedactionEvent(room, event);
396 : break;
397 : }
398 : }
399 :
400 0 : Future<void> _onDeviceChange(dynamic _) async {
401 0 : Logs().v('[VOIP] _onDeviceChange');
402 0 : for (final call in calls.values) {
403 0 : if (call.state == CallState.kConnected && !call.isGroupCall) {
404 0 : await call.updateMediaDeviceForCall();
405 : }
406 : }
407 0 : for (final groupCall in groupCalls.values) {
408 0 : if (groupCall.state == GroupCallState.entered) {
409 0 : await groupCall.backend.updateMediaDeviceForCalls();
410 : }
411 : }
412 : }
413 :
414 2 : Future<void> onCallInvite(
415 : Room room,
416 : String remoteUserId,
417 : String? remoteDeviceId,
418 : Map<String, dynamic> content,
419 : ) async {
420 4 : Logs().v(
421 12 : '[VOIP] onCallInvite $remoteUserId:$remoteDeviceId => ${client.userID}:${client.deviceID}, \ncontent => ${content.toString()}',
422 : );
423 :
424 2 : final String callId = content['call_id'];
425 2 : final int lifetime = content['lifetime'];
426 2 : final String? confId = content['conf_id'];
427 :
428 8 : final call = calls[VoipId(roomId: room.id, callId: callId)];
429 :
430 4 : Logs().d(
431 10 : '[glare] got new call ${content.tryGet('call_id')} and currently room id is mapped to ${incomingCallRoomId.tryGet(room.id)}',
432 : );
433 :
434 4 : if (call != null && call.state != CallState.kEnded) {
435 : // Session already exist.
436 6 : Logs().v('[VOIP] onCallInvite: Session [$callId] already exist.');
437 : return;
438 : }
439 :
440 2 : final inviteeUserId = content['invitee'];
441 0 : if (inviteeUserId != null && inviteeUserId != localParticipant?.userId) {
442 0 : Logs().w('[VOIP] Ignoring call, meant for user $inviteeUserId');
443 : return; // This invite was meant for another user in the room
444 : }
445 2 : final inviteeDeviceId = content['invitee_device_id'];
446 : if (inviteeDeviceId != null &&
447 0 : inviteeDeviceId != localParticipant?.deviceId) {
448 0 : Logs().w('[VOIP] Ignoring call, meant for device $inviteeDeviceId');
449 : return; // This invite was meant for another device in the room
450 : }
451 :
452 2 : if (content['capabilities'] != null) {
453 0 : final capabilities = CallCapabilities.fromJson(content['capabilities']);
454 0 : Logs().v(
455 0 : '[VOIP] CallCapabilities: dtmf => ${capabilities.dtmf}, transferee => ${capabilities.transferee}',
456 : );
457 : }
458 :
459 : var callType = CallType.kVoice;
460 : SDPStreamMetadata? sdpStreamMetadata;
461 2 : if (content[sdpStreamMetadataKey] != null) {
462 : sdpStreamMetadata =
463 0 : SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]);
464 0 : sdpStreamMetadata.sdpStreamMetadatas
465 0 : .forEach((streamId, SDPStreamPurpose purpose) {
466 0 : Logs().v(
467 0 : '[VOIP] [$streamId] => purpose: ${purpose.purpose}, audioMuted: ${purpose.audio_muted}, videoMuted: ${purpose.video_muted}',
468 : );
469 :
470 0 : if (!purpose.video_muted) {
471 : callType = CallType.kVideo;
472 : }
473 : });
474 : } else {
475 6 : callType = getCallType(content['offer']['sdp']);
476 : }
477 :
478 2 : final opts = CallOptions(
479 : voip: this,
480 : callId: callId,
481 : groupCallId: confId,
482 : dir: CallDirection.kIncoming,
483 : type: callType,
484 : room: room,
485 2 : localPartyId: localPartyId,
486 2 : iceServers: await getIceServers(),
487 : );
488 :
489 2 : final newCall = createNewCall(opts);
490 :
491 : /// both invitee userId and deviceId are set here because there can be
492 : /// multiple devices from same user in a call, so we specifiy who the
493 : /// invite is for
494 2 : newCall.remoteUserId = remoteUserId;
495 2 : newCall.remoteDeviceId = remoteDeviceId;
496 4 : newCall.remotePartyId = content['party_id'];
497 4 : newCall.remoteSessionId = content['sender_session_id'];
498 :
499 : // newCall.remoteSessionId = remoteParticipant.sessionId;
500 :
501 4 : if (!delegate.canHandleNewCall &&
502 : (confId == null ||
503 0 : currentGroupCID != VoipId(roomId: room.id, callId: confId))) {
504 0 : Logs().v(
505 : '[VOIP] onCallInvite: Unable to handle new calls, maybe user is busy.',
506 : );
507 : // no need to emit here because handleNewCall was never triggered yet
508 0 : await newCall.reject(reason: CallErrorCode.userBusy, shouldEmit: false);
509 0 : await delegate.handleMissedCall(newCall);
510 : return;
511 : }
512 :
513 2 : final offer = RTCSessionDescription(
514 4 : content['offer']['sdp'],
515 4 : content['offer']['type'],
516 : );
517 :
518 : /// play ringtone. We decided to play the ringtone before adding the call to
519 : /// the incoming call stream because getUserMedia from initWithInvite fails
520 : /// on firefox unless the tab is in focus. We should atleast be able to notify
521 : /// the user about an incoming call
522 : ///
523 : /// Autoplay on firefox still needs interaction, without which all notifications
524 : /// could be blocked.
525 : if (confId == null) {
526 4 : await delegate.playRingtone();
527 : }
528 :
529 : // When getUserMedia throws an exception, we handle it by terminating the call,
530 : // and all this happens inside initWithInvite. If we set currentCID after
531 : // initWithInvite, we might set it to callId even after it was reset to null
532 : // by terminate.
533 6 : currentCID = VoipId(roomId: room.id, callId: callId);
534 :
535 : if (confId == null) {
536 4 : await delegate.registerListeners(newCall);
537 : } else {
538 0 : onIncomingCallSetup.add(newCall);
539 : }
540 :
541 2 : await newCall.initWithInvite(
542 : callType,
543 : offer,
544 : sdpStreamMetadata,
545 : lifetime,
546 : confId != null,
547 : );
548 :
549 : // Popup CallingPage for incoming call.
550 2 : if (confId == null && !newCall.callHasEnded) {
551 4 : await delegate.handleNewCall(newCall);
552 : }
553 :
554 : if (confId != null) {
555 0 : onIncomingCallStart.add(newCall);
556 : }
557 : }
558 :
559 0 : Future<void> onCallAnswer(
560 : Room room,
561 : String remoteUserId,
562 : String? remoteDeviceId,
563 : Map<String, dynamic> content,
564 : ) async {
565 0 : Logs().v('[VOIP] onCallAnswer => ${content.toString()}');
566 0 : final String callId = content['call_id'];
567 :
568 0 : final call = calls[VoipId(roomId: room.id, callId: callId)];
569 : if (call != null) {
570 0 : if (!call.answeredByUs) {
571 0 : await delegate.stopRingtone();
572 : }
573 0 : if (call.state == CallState.kRinging) {
574 0 : await call.onAnsweredElsewhere();
575 : }
576 :
577 0 : if (call.room.id != room.id) {
578 0 : Logs().w(
579 0 : 'Ignoring call answer for room ${room.id} claiming to be for call in room ${call.room.id}',
580 : );
581 : return;
582 : }
583 :
584 0 : if (call.remoteUserId == null) {
585 0 : Logs().i(
586 : '[VOIP] you probably called the room without setting a userId in invite, setting the calls remote user id to what I get from m.call.answer now',
587 : );
588 0 : call.remoteUserId = remoteUserId;
589 : }
590 :
591 0 : if (call.remoteDeviceId == null) {
592 0 : Logs().i(
593 : '[VOIP] you probably called the room without setting a userId in invite, setting the calls remote user id to what I get from m.call.answer now',
594 : );
595 0 : call.remoteDeviceId = remoteDeviceId;
596 : }
597 0 : if (call.remotePartyId != null) {
598 0 : Logs().d(
599 0 : 'Ignoring call answer from party ${content['party_id']}, we are already with ${call.remotePartyId}',
600 : );
601 : return;
602 : } else {
603 0 : call.remotePartyId = content['party_id'];
604 : }
605 :
606 0 : final answer = RTCSessionDescription(
607 0 : content['answer']['sdp'],
608 0 : content['answer']['type'],
609 : );
610 :
611 : SDPStreamMetadata? metadata;
612 0 : if (content[sdpStreamMetadataKey] != null) {
613 0 : metadata = SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]);
614 : }
615 0 : await call.onAnswerReceived(answer, metadata);
616 : } else {
617 0 : Logs().v('[VOIP] onCallAnswer: Session [$callId] not found!');
618 : }
619 : }
620 :
621 2 : Future<void> onCallCandidates(Room room, Map<String, dynamic> content) async {
622 8 : Logs().v('[VOIP] onCallCandidates => ${content.toString()}');
623 2 : final String callId = content['call_id'];
624 8 : final call = calls[VoipId(roomId: room.id, callId: callId)];
625 : if (call != null) {
626 4 : await call.onCandidatesReceived(content['candidates']);
627 : } else {
628 0 : Logs().v('[VOIP] onCallCandidates: Session [$callId] not found!');
629 : }
630 : }
631 :
632 0 : Future<void> onCallHangup(Room room, Map<String, dynamic> content) async {
633 : // stop play ringtone, if this is an incoming call
634 0 : await delegate.stopRingtone();
635 0 : Logs().v('[VOIP] onCallHangup => ${content.toString()}');
636 0 : final String callId = content['call_id'];
637 :
638 0 : final call = calls[VoipId(roomId: room.id, callId: callId)];
639 : if (call != null) {
640 : // hangup in any case, either if the other party hung up or we did on another device
641 0 : await call.terminate(
642 : CallParty.kRemote,
643 0 : CallErrorCode.values.firstWhereOrNull(
644 0 : (element) => element.reason == content['reason'],
645 : ) ??
646 : CallErrorCode.userHangup,
647 : true,
648 : );
649 : } else {
650 0 : Logs().v('[VOIP] onCallHangup: Session [$callId] not found!');
651 : }
652 0 : if (callId == currentCID?.callId) {
653 0 : currentCID = null;
654 : }
655 : }
656 :
657 0 : Future<void> onCallReject(Room room, Map<String, dynamic> content) async {
658 0 : final String callId = content['call_id'];
659 0 : Logs().d('Reject received for call ID $callId');
660 :
661 0 : final call = calls[VoipId(roomId: room.id, callId: callId)];
662 : if (call != null) {
663 0 : await call.onRejectReceived(
664 0 : CallErrorCode.values.firstWhereOrNull(
665 0 : (element) => element.reason == content['reason'],
666 : ) ??
667 : CallErrorCode.userHangup,
668 : );
669 : } else {
670 0 : Logs().v('[VOIP] onCallReject: Session [$callId] not found!');
671 : }
672 : }
673 :
674 2 : Future<void> onCallSelectAnswer(
675 : Room room,
676 : Map<String, dynamic> content,
677 : ) async {
678 2 : final String callId = content['call_id'];
679 6 : Logs().d('SelectAnswer received for call ID $callId');
680 2 : final String selectedPartyId = content['selected_party_id'];
681 :
682 8 : final call = calls[VoipId(roomId: room.id, callId: callId)];
683 : if (call != null) {
684 8 : if (call.room.id != room.id) {
685 0 : Logs().w(
686 0 : 'Ignoring call select answer for room ${room.id} claiming to be for call in room ${call.room.id}',
687 : );
688 : return;
689 : }
690 2 : await call.onSelectAnswerReceived(selectedPartyId);
691 : }
692 : }
693 :
694 0 : Future<void> onSDPStreamMetadataChangedReceived(
695 : Room room,
696 : Map<String, dynamic> content,
697 : ) async {
698 0 : final String callId = content['call_id'];
699 0 : Logs().d('SDP Stream metadata received for call ID $callId');
700 :
701 0 : final call = calls[VoipId(roomId: room.id, callId: callId)];
702 : if (call != null) {
703 0 : if (content[sdpStreamMetadataKey] == null) {
704 0 : Logs().d('SDP Stream metadata is null');
705 : return;
706 : }
707 0 : await call.onSDPStreamMetadataReceived(
708 0 : SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]),
709 : );
710 : }
711 : }
712 :
713 0 : Future<void> onAssertedIdentityReceived(
714 : Room room,
715 : Map<String, dynamic> content,
716 : ) async {
717 0 : final String callId = content['call_id'];
718 0 : Logs().d('Asserted identity received for call ID $callId');
719 :
720 0 : final call = calls[VoipId(roomId: room.id, callId: callId)];
721 : if (call != null) {
722 0 : if (content['asserted_identity'] == null) {
723 0 : Logs().d('asserted_identity is null ');
724 : return;
725 : }
726 0 : call.onAssertedIdentityReceived(
727 0 : AssertedIdentity.fromJson(content['asserted_identity']),
728 : );
729 : }
730 : }
731 :
732 0 : Future<void> onCallNegotiate(Room room, Map<String, dynamic> content) async {
733 0 : final String callId = content['call_id'];
734 0 : Logs().d('Negotiate received for call ID $callId');
735 :
736 0 : final call = calls[VoipId(roomId: room.id, callId: callId)];
737 : if (call != null) {
738 : // ideally you also check the lifetime here and discard negotiation events
739 : // if age of the event was older than the lifetime but as to device events
740 : // do not have a unsigned age nor a origin_server_ts there's no easy way to
741 : // override this one function atm
742 :
743 0 : final description = content['description'];
744 : try {
745 : SDPStreamMetadata? metadata;
746 0 : if (content[sdpStreamMetadataKey] != null) {
747 0 : metadata = SDPStreamMetadata.fromJson(content[sdpStreamMetadataKey]);
748 : }
749 0 : await call.onNegotiateReceived(
750 : metadata,
751 0 : RTCSessionDescription(description['sdp'], description['type']),
752 : );
753 : } catch (e, s) {
754 0 : Logs().e('[VOIP] Failed to complete negotiation', e, s);
755 : }
756 : }
757 : }
758 :
759 2 : Future<void> _handleReactionEvent(
760 : Room room,
761 : MatrixEvent event,
762 : ) async {
763 2 : final content = event.content;
764 :
765 2 : final callId = content.tryGet<String>('call_id');
766 : if (callId == null) {
767 0 : Logs().w(
768 : '[VOIP] _handleReactionEvent: No call ID found in reaction content',
769 : );
770 : return;
771 : }
772 :
773 8 : final groupCall = groupCalls[VoipId(roomId: room.id, callId: callId)];
774 : if (groupCall == null) {
775 0 : Logs().w(
776 0 : '[VOIP] _handleReactionEvent: No respective group call found for room ${room.id}, call ID $callId',
777 : );
778 : return;
779 : }
780 :
781 : final membershipEventId = content
782 2 : .tryGetMap<String, String>('m.relates_to')
783 2 : ?.tryGet<String>('event_id');
784 2 : final deviceId = content.tryGet<String>('device_id');
785 :
786 : if (membershipEventId == null || deviceId == null) {
787 4 : Logs().w(
788 : '[VOIP] _handleReactionEvent: No event ID or device ID found in reaction content',
789 : );
790 : return;
791 : }
792 :
793 2 : final reactionKey = content.tryGet<String>('key');
794 2 : final isEphemeral = content.tryGet<bool>('is_ephemeral') ?? false;
795 :
796 : if (reactionKey == null) {
797 4 : Logs().w(
798 : '[VOIP] _handleReactionEvent: No reaction key found in reaction content',
799 : );
800 : return;
801 : }
802 :
803 : final memberships =
804 4 : room.getCallMembershipsForUser(event.senderId, deviceId, this);
805 2 : final membership = memberships.firstWhereOrNull(
806 2 : (m) =>
807 4 : m.callId == callId &&
808 4 : m.application == 'm.call' &&
809 4 : m.scope == 'm.room',
810 : );
811 :
812 2 : if (membership == null || membership.isExpired) {
813 0 : Logs().w(
814 0 : '[VOIP] _handleReactionEvent: No matching membership found or found expired for reaction from ${event.senderId}',
815 : );
816 : return;
817 : }
818 :
819 4 : if (membership.eventId != membershipEventId) {
820 0 : Logs().w(
821 0 : '[VOIP] _handleReactionEvent: Event ID mismatch, ignoring reaction on old event from ${event.senderId}',
822 : );
823 : return;
824 : }
825 :
826 2 : final participant = CallParticipant(
827 : this,
828 2 : userId: event.senderId,
829 : deviceId: deviceId,
830 : );
831 :
832 2 : final reaction = CallReactionAddedEvent(
833 : participant: participant,
834 : reactionKey: reactionKey,
835 : membershipEventId: membershipEventId,
836 2 : reactionEventId: event.eventId,
837 : isEphemeral: isEphemeral,
838 : );
839 :
840 4 : groupCall.matrixRTCEventStream.add(reaction);
841 :
842 4 : Logs().d(
843 2 : '[VOIP] _handleReactionEvent: Sent reaction event: $reaction',
844 : );
845 : }
846 :
847 2 : Future<void> _handleRedactionEvent(
848 : Room room,
849 : BasicEventWithSender event,
850 : ) async {
851 2 : final content = event.content;
852 4 : if (content.tryGet<String>('redacts_type') !=
853 : EventTypes.GroupCallMemberReaction) {
854 : // ignore it
855 4 : Logs().v(
856 4 : '[_handleRedactionEvent] Ignoring redaction event ${event.toJson()} because not a call reaction redaction',
857 : );
858 : return;
859 : }
860 2 : final redactedEventId = content.tryGet<String>('redacts');
861 :
862 : if (redactedEventId == null) {
863 0 : Logs().v(
864 : '[VOIP] _handleRedactionEvent: Missing sender or redacted event ID',
865 : );
866 : return;
867 : }
868 :
869 4 : final deviceId = event.content.tryGet<String>('device_id');
870 : if (deviceId == null) {
871 0 : Logs().w(
872 0 : '[VOIP] _handleRedactionEvent: Could not find device_id in redacted event $redactedEventId',
873 : );
874 : return;
875 : }
876 :
877 4 : Logs().d(
878 2 : '[VOIP] _handleRedactionEvent: Device ID from redacted event: $deviceId',
879 : );
880 :
881 : // Route to all active group calls in the room
882 6 : final groupCall = groupCalls.values.firstWhereOrNull(
883 2 : (m) =>
884 8 : m.room.id == room.id &&
885 4 : m.state == GroupCallState.entered &&
886 8 : m.groupCallId == event.content.tryGet('call_id') &&
887 4 : m.application == 'm.call' &&
888 4 : m.scope == 'm.room',
889 : );
890 :
891 : if (groupCall == null) {
892 0 : Logs().w(
893 0 : '[_handleRedactionEvent] could not find group call for event ${event.toJson()}',
894 : );
895 : return;
896 : }
897 :
898 2 : final participant = CallParticipant(
899 : this,
900 2 : userId: event.senderId,
901 : deviceId: deviceId,
902 : );
903 :
904 : // We don't know the specific reaction key from redaction events
905 : // The listeners can filter based on their current state
906 2 : final reactionEvent = CallReactionRemovedEvent(
907 : participant: participant,
908 : redactedEventId: redactedEventId,
909 : );
910 :
911 4 : groupCall.matrixRTCEventStream.add(reactionEvent);
912 : }
913 :
914 2 : CallType getCallType(String sdp) {
915 : try {
916 2 : final session = sdp_transform.parse(sdp);
917 8 : if (session['media'].indexWhere((e) => e['type'] == 'video') != -1) {
918 : return CallType.kVideo;
919 : }
920 : } catch (e, s) {
921 0 : Logs().e('[VOIP] Failed to getCallType', e, s);
922 : }
923 :
924 : return CallType.kVoice;
925 : }
926 :
927 6 : Future<List<Map<String, dynamic>>> getIceServers() async {
928 6 : if (_turnServerCredentials == null) {
929 : try {
930 18 : _turnServerCredentials = await client.getTurnServer();
931 : } catch (e) {
932 0 : Logs().v('[VOIP] getTurnServerCredentials error => ${e.toString()}');
933 : }
934 : }
935 :
936 6 : if (_turnServerCredentials == null) {
937 0 : return [];
938 : }
939 :
940 6 : return [
941 6 : {
942 12 : 'username': _turnServerCredentials!.username,
943 12 : 'credential': _turnServerCredentials!.password,
944 12 : 'urls': _turnServerCredentials!.uris,
945 : }
946 : ];
947 : }
948 :
949 : /// Make a P2P call to room
950 : ///
951 : /// Pretty important to set the userId, or all the users in the room get a call.
952 : /// Including your own other devices, so just set it to directChatMatrixId
953 : ///
954 : /// Setting the deviceId would make all other devices for that userId ignore the call
955 : /// Ideally only group calls would need setting both userId and deviceId to allow
956 : /// having 2 devices from the same user in a group call
957 : ///
958 : /// For p2p call, you want to have all the devices of the specified `userId` ring
959 2 : Future<CallSession> inviteToCall(
960 : Room room,
961 : CallType type, {
962 : String? userId,
963 : String? deviceId,
964 : }) async {
965 2 : final roomId = room.id;
966 2 : final callId = genCallID();
967 2 : if (currentGroupCID == null) {
968 4 : incomingCallRoomId[roomId] = callId;
969 : }
970 2 : final opts = CallOptions(
971 : callId: callId,
972 : type: type,
973 : dir: CallDirection.kOutgoing,
974 : room: room,
975 : voip: this,
976 2 : localPartyId: localPartyId,
977 2 : iceServers: await getIceServers(),
978 : );
979 2 : final newCall = createNewCall(opts);
980 :
981 2 : newCall.remoteUserId = userId;
982 2 : newCall.remoteDeviceId = deviceId;
983 :
984 4 : await delegate.registerListeners(newCall);
985 :
986 4 : currentCID = VoipId(roomId: roomId, callId: callId);
987 6 : await newCall.initOutboundCall(type).then((_) {
988 6 : unawaited(delegate.handleNewCall(newCall));
989 : });
990 : return newCall;
991 : }
992 :
993 6 : CallSession createNewCall(CallOptions opts) {
994 6 : final call = CallSession(opts);
995 36 : calls[VoipId(roomId: opts.room.id, callId: opts.callId)] = call;
996 : return call;
997 : }
998 :
999 : /// Create a new group call in an existing room.
1000 : ///
1001 : /// [groupCallId] The room id to call
1002 : ///
1003 : /// [application] normal group call, thrirdroom, etc
1004 : ///
1005 : /// [scope] room, between specifc users, etc.
1006 0 : Future<GroupCallSession> _newGroupCall(
1007 : String groupCallId,
1008 : Room room,
1009 : CallBackend backend,
1010 : String? application,
1011 : String? scope,
1012 : ) async {
1013 0 : if (getGroupCallById(room.id, groupCallId) != null) {
1014 0 : Logs().v('[VOIP] [$groupCallId] already exists.');
1015 0 : return getGroupCallById(room.id, groupCallId)!;
1016 : }
1017 :
1018 0 : final groupCall = GroupCallSession(
1019 : groupCallId: groupCallId,
1020 0 : client: client,
1021 : room: room,
1022 : voip: this,
1023 : backend: backend,
1024 : application: application,
1025 : scope: scope,
1026 : );
1027 :
1028 0 : setGroupCallById(groupCall);
1029 :
1030 : return groupCall;
1031 : }
1032 :
1033 : /// Create a new group call in an existing room.
1034 : ///
1035 : /// [groupCallId] The room id to call
1036 : ///
1037 : /// [application] normal group call, thrirdroom, etc
1038 : ///
1039 : /// [scope] room, between specifc users, etc.
1040 : ///
1041 : /// [preShareKey] for livekit calls it creates and shares a key with other
1042 : /// participants in the call without entering, useful on onboarding screens.
1043 : /// does not do anything in mesh calls
1044 :
1045 0 : Future<GroupCallSession> fetchOrCreateGroupCall(
1046 : String groupCallId,
1047 : Room room,
1048 : CallBackend backend,
1049 : String? application,
1050 : String? scope, {
1051 : bool preShareKey = true,
1052 : }) async {
1053 : // somehow user were mising their powerlevels events and got stuck
1054 : // with the exception below, this part just makes sure importantStateEvents
1055 : // does not cause it.
1056 0 : await room.postLoad();
1057 :
1058 0 : if (!room.groupCallsEnabledForEveryone) {
1059 0 : await room.enableGroupCalls();
1060 : }
1061 :
1062 0 : if (!room.canJoinGroupCall) {
1063 0 : throw MatrixSDKVoipException(
1064 : '''
1065 0 : User ${client.userID}:${client.deviceID} is not allowed to join famedly calls in room ${room.id},
1066 0 : canJoinGroupCall: ${room.canJoinGroupCall},
1067 0 : groupCallsEnabledForEveryone: ${room.groupCallsEnabledForEveryone},
1068 0 : needed: ${room.powerForChangingStateEvent(EventTypes.GroupCallMember)},
1069 0 : own: ${room.ownPowerLevel}}
1070 0 : plMap: ${room.getState(EventTypes.RoomPowerLevels)?.content}
1071 0 : ''',
1072 : );
1073 : }
1074 :
1075 0 : GroupCallSession? groupCall = getGroupCallById(room.id, groupCallId);
1076 :
1077 0 : groupCall ??= await _newGroupCall(
1078 : groupCallId,
1079 : room,
1080 : backend,
1081 : application,
1082 : scope,
1083 : );
1084 :
1085 : if (preShareKey) {
1086 0 : await groupCall.backend.preShareKey(groupCall);
1087 : }
1088 :
1089 : return groupCall;
1090 : }
1091 :
1092 2 : GroupCallSession? getGroupCallById(String roomId, String groupCallId) {
1093 6 : return groupCalls[VoipId(roomId: roomId, callId: groupCallId)];
1094 : }
1095 :
1096 6 : void setGroupCallById(GroupCallSession groupCallSession) {
1097 18 : groupCalls[VoipId(
1098 12 : roomId: groupCallSession.room.id,
1099 6 : callId: groupCallSession.groupCallId,
1100 : )] = groupCallSession;
1101 : }
1102 :
1103 : /// Create a new group call from a room state event.
1104 6 : Future<void> createGroupCallFromRoomStateEvent(
1105 : CallMembership membership, {
1106 : bool emitHandleNewGroupCall = true,
1107 : }) async {
1108 6 : if (membership.isExpired) return;
1109 :
1110 18 : final room = client.getRoomById(membership.roomId);
1111 :
1112 : if (room == null) {
1113 0 : Logs().w('Couldn\'t find room ${membership.roomId} for GroupCallSession');
1114 : return;
1115 : }
1116 :
1117 12 : if (membership.application != 'm.call' && membership.scope != 'm.room') {
1118 0 : Logs().w('Received invalid group call application or scope.');
1119 : return;
1120 : }
1121 :
1122 6 : final groupCall = GroupCallSession(
1123 6 : client: client,
1124 : voip: this,
1125 : room: room,
1126 6 : backend: membership.backend,
1127 6 : groupCallId: membership.callId,
1128 6 : application: membership.application,
1129 6 : scope: membership.scope,
1130 : );
1131 :
1132 12 : if (groupCalls.containsKey(
1133 18 : VoipId(roomId: membership.roomId, callId: membership.callId),
1134 : )) {
1135 : return;
1136 : }
1137 :
1138 6 : setGroupCallById(groupCall);
1139 :
1140 12 : onIncomingGroupCall.add(groupCall);
1141 : if (emitHandleNewGroupCall) {
1142 12 : await delegate.handleNewGroupCall(groupCall);
1143 : }
1144 : }
1145 :
1146 0 : @Deprecated('Call `hasActiveGroupCall` on the room directly instead')
1147 0 : bool hasActiveCall(Room room) => room.hasActiveGroupCall(this);
1148 : }
|
