Line data Source code
1 : import 'dart:async';
2 :
3 : import 'package:collection/collection.dart';
4 : import 'package:webrtc_interface/webrtc_interface.dart';
5 :
6 : import 'package:matrix/matrix.dart';
7 : import 'package:matrix/src/utils/cached_stream_controller.dart';
8 : import 'package:matrix/src/voip/models/call_options.dart';
9 : import 'package:matrix/src/voip/utils/stream_helper.dart';
10 : import 'package:matrix/src/voip/utils/user_media_constraints.dart';
11 :
12 : class MeshBackend extends CallBackend {
13 6 : MeshBackend({
14 : super.type = 'mesh',
15 : });
16 :
17 : final List<CallSession> _callSessions = [];
18 :
19 : /// participant:volume
20 : final Map<CallParticipant, double> _audioLevelsMap = {};
21 :
22 : /// The stream is used to prepare for incoming peer calls like registering listeners
23 : StreamSubscription<CallSession>? _callSetupSubscription;
24 :
25 : /// The stream is used to signal the start of an incoming peer call
26 : StreamSubscription<CallSession>? _callStartSubscription;
27 :
28 : Timer? _activeSpeakerLoopTimeout;
29 :
30 : final CachedStreamController<WrappedMediaStream> onStreamAdd =
31 : CachedStreamController();
32 :
33 : final CachedStreamController<WrappedMediaStream> onStreamRemoved =
34 : CachedStreamController();
35 :
36 : final CachedStreamController<GroupCallSession> onGroupCallFeedsChanged =
37 : CachedStreamController();
38 :
39 6 : @override
40 : Map<String, Object?> toJson() {
41 6 : return {
42 6 : 'type': type,
43 : };
44 : }
45 :
46 : CallParticipant? _activeSpeaker;
47 : WrappedMediaStream? _localUserMediaStream;
48 : WrappedMediaStream? _localScreenshareStream;
49 : final List<WrappedMediaStream> _userMediaStreams = [];
50 : final List<WrappedMediaStream> _screenshareStreams = [];
51 :
52 4 : List<WrappedMediaStream> _getLocalStreams() {
53 4 : final feeds = <WrappedMediaStream>[];
54 :
55 4 : if (localUserMediaStream != null) {
56 8 : feeds.add(localUserMediaStream!);
57 : }
58 :
59 4 : if (localScreenshareStream != null) {
60 4 : feeds.add(localScreenshareStream!);
61 : }
62 :
63 : return feeds;
64 : }
65 :
66 4 : Future<MediaStream> _getUserMedia(
67 : GroupCallSession groupCall,
68 : CallType type,
69 : ) async {
70 4 : final mediaConstraints = {
71 : 'audio': UserMediaConstraints.micMediaConstraints,
72 4 : 'video': type == CallType.kVideo
73 : ? UserMediaConstraints.camMediaConstraints
74 : : false,
75 : };
76 :
77 : try {
78 12 : return await groupCall.voip.delegate.mediaDevices
79 4 : .getUserMedia(mediaConstraints);
80 : } catch (e) {
81 0 : groupCall.setState(GroupCallState.localCallFeedUninitialized);
82 : rethrow;
83 : }
84 : }
85 :
86 2 : Future<MediaStream> _getDisplayMedia(GroupCallSession groupCall) async {
87 2 : final mediaConstraints = {
88 : 'audio': false,
89 : 'video': true,
90 : };
91 : try {
92 6 : return await groupCall.voip.delegate.mediaDevices
93 2 : .getDisplayMedia(mediaConstraints);
94 : } catch (e, s) {
95 0 : throw MatrixSDKVoipException('_getDisplayMedia failed', stackTrace: s);
96 : }
97 : }
98 :
99 4 : CallSession? _getCallForParticipant(
100 : GroupCallSession groupCall,
101 : CallParticipant participant,
102 : ) {
103 8 : return _callSessions.singleWhereOrNull(
104 2 : (call) =>
105 6 : call.groupCallId == groupCall.groupCallId &&
106 2 : CallParticipant(
107 2 : groupCall.voip,
108 2 : userId: call.remoteUserId!,
109 2 : deviceId: call.remoteDeviceId,
110 2 : ) ==
111 : participant,
112 : );
113 : }
114 :
115 : /// Register listeners for a peer call to use for the group calls, that is
116 : /// needed before even call is added to `_callSessions`.
117 : /// We do this here for onStreamAdd and onStreamRemoved to make sure we don't
118 : /// miss any events that happen before the call is completely started.
119 4 : void _registerListenersBeforeCallAdd(CallSession call) {
120 16 : call.onStreamAdd.stream.listen((stream) {
121 4 : if (!stream.isLocal()) {
122 4 : onStreamAdd.add(stream);
123 : }
124 : });
125 :
126 12 : call.onStreamRemoved.stream.listen((stream) {
127 0 : if (!stream.isLocal()) {
128 0 : onStreamRemoved.add(stream);
129 : }
130 : });
131 : }
132 :
133 4 : Future<void> _addCall(GroupCallSession groupCall, CallSession call) async {
134 8 : _callSessions.add(call);
135 4 : _initCall(groupCall, call);
136 : // ignore: deprecated_member_use_from_same_package
137 8 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
138 12 : groupCall.matrixRTCEventStream.add(CallAddedEvent(call));
139 : }
140 :
141 : /// init a peer call from group calls.
142 4 : void _initCall(GroupCallSession groupCall, CallSession call) {
143 4 : if (call.remoteUserId == null) {
144 0 : throw MatrixSDKVoipException(
145 : 'Cannot init call without proper invitee user and device Id',
146 : );
147 : }
148 :
149 12 : call.onCallStateChanged.stream.listen(
150 2 : ((event) async {
151 2 : await _onCallStateChanged(call, event);
152 : }),
153 : );
154 :
155 14 : call.onCallReplaced.stream.listen((CallSession newCall) async {
156 2 : await _replaceCall(groupCall, call, newCall);
157 : });
158 :
159 14 : call.onCallStreamsChanged.stream.listen((call) async {
160 2 : await call.tryRemoveStopedStreams();
161 2 : await _onStreamsChanged(groupCall, call);
162 : });
163 :
164 14 : call.onCallHangupNotifierForGroupCalls.stream.listen((event) async {
165 2 : await _onCallHangup(groupCall, call);
166 : });
167 : }
168 :
169 2 : Future<void> _replaceCall(
170 : GroupCallSession groupCall,
171 : CallSession existingCall,
172 : CallSession replacementCall,
173 : ) async {
174 2 : final existingCallIndex = _callSessions
175 10 : .indexWhere((element) => element.callId == existingCall.callId);
176 :
177 4 : if (existingCallIndex == -1) {
178 0 : throw MatrixSDKVoipException('Couldn\'t find call to replace');
179 : }
180 :
181 4 : _callSessions.removeAt(existingCallIndex);
182 4 : _callSessions.add(replacementCall);
183 :
184 2 : await _disposeCall(groupCall, existingCall, CallErrorCode.replaced);
185 2 : _registerListenersBeforeCallAdd(replacementCall);
186 2 : _initCall(groupCall, replacementCall);
187 :
188 : // ignore: deprecated_member_use_from_same_package
189 4 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
190 2 : groupCall.matrixRTCEventStream
191 4 : .add(CallReplacedEvent(existingCall, replacementCall));
192 : }
193 :
194 : /// Removes a peer call from group calls.
195 2 : Future<void> _removeCall(
196 : GroupCallSession groupCall,
197 : CallSession call,
198 : CallErrorCode hangupReason,
199 : ) async {
200 2 : await _disposeCall(groupCall, call, hangupReason);
201 :
202 12 : _callSessions.removeWhere((element) => call.callId == element.callId);
203 :
204 : // ignore: deprecated_member_use_from_same_package
205 4 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
206 6 : groupCall.matrixRTCEventStream.add(CallRemovedEvent(call));
207 : }
208 :
209 2 : Future<void> _disposeCall(
210 : GroupCallSession groupCall,
211 : CallSession call,
212 : CallErrorCode hangupReason,
213 : ) async {
214 2 : if (call.remoteUserId == null) {
215 0 : throw MatrixSDKVoipException(
216 : 'Cannot init call without proper invitee user and device Id',
217 : );
218 : }
219 :
220 4 : if (call.hangupReason == CallErrorCode.replaced) {
221 : return;
222 : }
223 :
224 4 : if (call.state != CallState.kEnded) {
225 : // no need to emit individual handleCallEnded on group calls
226 : // also prevents a loop of hangup and onCallHangupNotifierForGroupCalls
227 2 : await call.hangup(reason: hangupReason, shouldEmit: false);
228 : }
229 :
230 2 : final usermediaStream = _getUserMediaStreamByParticipantId(
231 2 : CallParticipant(
232 2 : groupCall.voip,
233 2 : userId: call.remoteUserId!,
234 2 : deviceId: call.remoteDeviceId,
235 2 : ).id,
236 : );
237 :
238 : if (usermediaStream != null) {
239 0 : await _removeUserMediaStream(groupCall, usermediaStream);
240 : }
241 :
242 2 : final screenshareStream = _getScreenshareStreamByParticipantId(
243 2 : CallParticipant(
244 2 : groupCall.voip,
245 2 : userId: call.remoteUserId!,
246 2 : deviceId: call.remoteDeviceId,
247 2 : ).id,
248 : );
249 :
250 : if (screenshareStream != null) {
251 0 : await _removeScreenshareStream(groupCall, screenshareStream);
252 : }
253 : }
254 :
255 2 : Future<void> _onStreamsChanged(
256 : GroupCallSession groupCall,
257 : CallSession call,
258 : ) async {
259 2 : if (call.remoteUserId == null) {
260 0 : throw MatrixSDKVoipException(
261 : 'Cannot init call without proper invitee user and device Id',
262 : );
263 : }
264 :
265 2 : final currentUserMediaStream = _getUserMediaStreamByParticipantId(
266 2 : CallParticipant(
267 2 : groupCall.voip,
268 2 : userId: call.remoteUserId!,
269 2 : deviceId: call.remoteDeviceId,
270 2 : ).id,
271 : );
272 :
273 2 : final remoteUsermediaStream = call.remoteUserMediaStream;
274 2 : final remoteStreamChanged = remoteUsermediaStream != currentUserMediaStream;
275 :
276 : if (remoteStreamChanged) {
277 : if (currentUserMediaStream == null && remoteUsermediaStream != null) {
278 2 : await _addUserMediaStream(groupCall, remoteUsermediaStream);
279 : } else if (currentUserMediaStream != null &&
280 : remoteUsermediaStream != null) {
281 0 : await _replaceUserMediaStream(
282 : groupCall,
283 : currentUserMediaStream,
284 : remoteUsermediaStream,
285 : );
286 : } else if (currentUserMediaStream != null &&
287 : remoteUsermediaStream == null) {
288 0 : await _removeUserMediaStream(groupCall, currentUserMediaStream);
289 : }
290 : }
291 :
292 2 : final currentScreenshareStream = _getScreenshareStreamByParticipantId(
293 2 : CallParticipant(
294 2 : groupCall.voip,
295 2 : userId: call.remoteUserId!,
296 2 : deviceId: call.remoteDeviceId,
297 2 : ).id,
298 : );
299 2 : final remoteScreensharingStream = call.remoteScreenSharingStream;
300 : final remoteScreenshareStreamChanged =
301 2 : remoteScreensharingStream != currentScreenshareStream;
302 :
303 : if (remoteScreenshareStreamChanged) {
304 : if (currentScreenshareStream == null &&
305 : remoteScreensharingStream != null) {
306 0 : _addScreenshareStream(groupCall, remoteScreensharingStream);
307 : } else if (currentScreenshareStream != null &&
308 : remoteScreensharingStream != null) {
309 0 : await _replaceScreenshareStream(
310 : groupCall,
311 : currentScreenshareStream,
312 : remoteScreensharingStream,
313 : );
314 : } else if (currentScreenshareStream != null &&
315 : remoteScreensharingStream == null) {
316 0 : await _removeScreenshareStream(groupCall, currentScreenshareStream);
317 : }
318 : }
319 :
320 4 : onGroupCallFeedsChanged.add(groupCall);
321 : }
322 :
323 2 : WrappedMediaStream? _getUserMediaStreamByParticipantId(String participantId) {
324 2 : final stream = _userMediaStreams
325 10 : .where((stream) => stream.participant.id == participantId);
326 2 : if (stream.isNotEmpty) {
327 2 : return stream.first;
328 : }
329 : return null;
330 : }
331 :
332 4 : void _onActiveSpeakerLoop(GroupCallSession groupCall) async {
333 : CallParticipant? nextActiveSpeaker;
334 : // idc about screen sharing atm.
335 : final userMediaStreamsCopyList =
336 8 : List<WrappedMediaStream>.from(_userMediaStreams);
337 8 : for (final stream in userMediaStreamsCopyList) {
338 12 : if (stream.participant.isLocal && stream.pc == null) {
339 : continue;
340 : }
341 :
342 4 : final List<StatsReport> statsReport = await stream.pc!.getStats();
343 : statsReport
344 8 : .removeWhere((element) => !element.values.containsKey('audioLevel'));
345 :
346 : // https://www.w3.org/TR/webrtc-stats/#summary
347 : final otherPartyAudioLevel = statsReport
348 2 : .singleWhereOrNull(
349 2 : (element) =>
350 4 : element.type == 'inbound-rtp' &&
351 6 : element.values['kind'] == 'audio',
352 : )
353 4 : ?.values['audioLevel'];
354 : if (otherPartyAudioLevel != null) {
355 6 : _audioLevelsMap[stream.participant] = otherPartyAudioLevel;
356 : }
357 :
358 : // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source
359 : // firefox does not seem to have this though. Works on chrome and android
360 : final ownAudioLevel = statsReport
361 2 : .singleWhereOrNull(
362 2 : (element) =>
363 4 : element.type == 'media-source' &&
364 6 : element.values['kind'] == 'audio',
365 : )
366 4 : ?.values['audioLevel'];
367 2 : if (groupCall.localParticipant != null &&
368 : ownAudioLevel != null &&
369 8 : _audioLevelsMap[groupCall.localParticipant] != ownAudioLevel) {
370 6 : _audioLevelsMap[groupCall.localParticipant!] = ownAudioLevel;
371 : }
372 : }
373 :
374 : double maxAudioLevel = double.negativeInfinity;
375 : // TODO: we probably want a threshold here?
376 10 : _audioLevelsMap.forEach((key, value) {
377 2 : if (value > maxAudioLevel) {
378 : nextActiveSpeaker = key;
379 : maxAudioLevel = value;
380 : }
381 : });
382 :
383 4 : if (nextActiveSpeaker != null && _activeSpeaker != nextActiveSpeaker) {
384 2 : _activeSpeaker = nextActiveSpeaker;
385 : // ignore: deprecated_member_use_from_same_package
386 4 : groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged);
387 2 : groupCall.matrixRTCEventStream
388 6 : .add(GroupCallActiveSpeakerChanged(_activeSpeaker!));
389 : }
390 6 : _activeSpeakerLoopTimeout?.cancel();
391 8 : _activeSpeakerLoopTimeout = Timer(
392 : CallConstants.activeSpeakerInterval,
393 4 : () => _onActiveSpeakerLoop(groupCall),
394 : );
395 : }
396 :
397 2 : WrappedMediaStream? _getScreenshareStreamByParticipantId(
398 : String participantId,
399 : ) {
400 2 : final stream = _screenshareStreams
401 2 : .where((stream) => stream.participant.id == participantId);
402 2 : if (stream.isNotEmpty) {
403 0 : return stream.first;
404 : }
405 : return null;
406 : }
407 :
408 2 : void _addScreenshareStream(
409 : GroupCallSession groupCall,
410 : WrappedMediaStream stream,
411 : ) {
412 4 : _screenshareStreams.add(stream);
413 4 : onStreamAdd.add(stream);
414 : // ignore: deprecated_member_use_from_same_package
415 2 : groupCall.onGroupCallEvent
416 : // ignore: deprecated_member_use_from_same_package
417 2 : .add(GroupCallStateChange.screenshareStreamsChanged);
418 2 : groupCall.matrixRTCEventStream
419 4 : .add(GroupCallStreamAdded(GroupCallStreamType.screenshare));
420 : }
421 :
422 0 : Future<void> _replaceScreenshareStream(
423 : GroupCallSession groupCall,
424 : WrappedMediaStream existingStream,
425 : WrappedMediaStream replacementStream,
426 : ) async {
427 0 : final streamIndex = _screenshareStreams.indexWhere(
428 0 : (stream) => stream.participant.id == existingStream.participant.id,
429 : );
430 :
431 0 : if (streamIndex == -1) {
432 0 : throw MatrixSDKVoipException(
433 : 'Couldn\'t find screenshare stream to replace',
434 : );
435 : }
436 :
437 0 : _screenshareStreams.replaceRange(streamIndex, 1, [replacementStream]);
438 :
439 0 : await existingStream.dispose();
440 : // ignore: deprecated_member_use_from_same_package
441 0 : groupCall.onGroupCallEvent
442 : // ignore: deprecated_member_use_from_same_package
443 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
444 0 : groupCall.matrixRTCEventStream
445 0 : .add(GroupCallStreamReplaced(GroupCallStreamType.screenshare));
446 : }
447 :
448 2 : Future<void> _removeScreenshareStream(
449 : GroupCallSession groupCall,
450 : WrappedMediaStream stream,
451 : ) async {
452 2 : final streamIndex = _screenshareStreams
453 14 : .indexWhere((stream) => stream.participant.id == stream.participant.id);
454 :
455 4 : if (streamIndex == -1) {
456 0 : throw MatrixSDKVoipException(
457 : 'Couldn\'t find screenshare stream to remove',
458 : );
459 : }
460 :
461 4 : _screenshareStreams.removeWhere(
462 12 : (element) => element.participant.id == stream.participant.id,
463 : );
464 :
465 4 : onStreamRemoved.add(stream);
466 :
467 2 : if (stream.isLocal()) {
468 4 : await stopMediaStream(stream.stream);
469 : }
470 :
471 : // ignore: deprecated_member_use_from_same_package
472 2 : groupCall.onGroupCallEvent
473 : // ignore: deprecated_member_use_from_same_package
474 2 : .add(GroupCallStateChange.screenshareStreamsChanged);
475 2 : groupCall.matrixRTCEventStream
476 4 : .add(GroupCallStreamRemoved(GroupCallStreamType.screenshare));
477 : }
478 :
479 2 : Future<void> _onCallStateChanged(CallSession call, CallState state) async {
480 4 : final audioMuted = localUserMediaStream?.isAudioMuted() ?? true;
481 2 : if (call.localUserMediaStream != null &&
482 4 : call.isMicrophoneMuted != audioMuted) {
483 0 : await call.setMicrophoneMuted(audioMuted);
484 : }
485 :
486 4 : final videoMuted = localUserMediaStream?.isVideoMuted() ?? true;
487 :
488 2 : if (call.localUserMediaStream != null &&
489 4 : call.isLocalVideoMuted != videoMuted) {
490 0 : await call.setLocalVideoMuted(videoMuted);
491 : }
492 : }
493 :
494 2 : Future<void> _onCallHangup(
495 : GroupCallSession groupCall,
496 : CallSession call,
497 : ) async {
498 4 : if (call.hangupReason == CallErrorCode.replaced) {
499 : return;
500 : }
501 2 : await _onStreamsChanged(groupCall, call);
502 4 : await _removeCall(groupCall, call, call.hangupReason!);
503 : }
504 :
505 4 : Future<void> _addUserMediaStream(
506 : GroupCallSession groupCall,
507 : WrappedMediaStream stream,
508 : ) async {
509 8 : _userMediaStreams.add(stream);
510 8 : onStreamAdd.add(stream);
511 : // ignore: deprecated_member_use_from_same_package
512 4 : groupCall.onGroupCallEvent
513 : // ignore: deprecated_member_use_from_same_package
514 4 : .add(GroupCallStateChange.userMediaStreamsChanged);
515 4 : groupCall.matrixRTCEventStream
516 8 : .add(GroupCallStreamAdded(GroupCallStreamType.userMedia));
517 : }
518 :
519 0 : Future<void> _replaceUserMediaStream(
520 : GroupCallSession groupCall,
521 : WrappedMediaStream existingStream,
522 : WrappedMediaStream replacementStream,
523 : ) async {
524 0 : final streamIndex = _userMediaStreams.indexWhere(
525 0 : (stream) => stream.participant.id == existingStream.participant.id,
526 : );
527 :
528 0 : if (streamIndex == -1) {
529 0 : throw MatrixSDKVoipException(
530 : 'Couldn\'t find user media stream to replace',
531 : );
532 : }
533 :
534 0 : _userMediaStreams.replaceRange(streamIndex, 1, [replacementStream]);
535 :
536 0 : await existingStream.dispose();
537 : // ignore: deprecated_member_use_from_same_package
538 0 : groupCall.onGroupCallEvent
539 : // ignore: deprecated_member_use_from_same_package
540 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
541 0 : groupCall.matrixRTCEventStream
542 0 : .add(GroupCallStreamReplaced(GroupCallStreamType.userMedia));
543 : }
544 :
545 0 : Future<void> _removeUserMediaStream(
546 : GroupCallSession groupCall,
547 : WrappedMediaStream stream,
548 : ) async {
549 0 : final streamIndex = _userMediaStreams.indexWhere(
550 0 : (element) => element.participant.id == stream.participant.id,
551 : );
552 :
553 0 : if (streamIndex == -1) {
554 0 : throw MatrixSDKVoipException(
555 : 'Couldn\'t find user media stream to remove',
556 : );
557 : }
558 :
559 0 : _userMediaStreams.removeWhere(
560 0 : (element) => element.participant.id == stream.participant.id,
561 : );
562 0 : _audioLevelsMap.remove(stream.participant);
563 0 : onStreamRemoved.add(stream);
564 :
565 0 : if (stream.isLocal()) {
566 0 : await stopMediaStream(stream.stream);
567 : }
568 :
569 : // ignore: deprecated_member_use_from_same_package
570 0 : groupCall.onGroupCallEvent
571 : // ignore: deprecated_member_use_from_same_package
572 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
573 0 : groupCall.matrixRTCEventStream
574 0 : .add(GroupCallStreamRemoved(GroupCallStreamType.userMedia));
575 :
576 0 : if (_activeSpeaker == stream.participant && _userMediaStreams.isNotEmpty) {
577 0 : _activeSpeaker = _userMediaStreams[0].participant;
578 : // ignore: deprecated_member_use_from_same_package
579 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged);
580 0 : groupCall.matrixRTCEventStream
581 0 : .add(GroupCallActiveSpeakerChanged(_activeSpeaker!));
582 : }
583 : }
584 :
585 0 : @override
586 : bool get e2eeEnabled => false;
587 :
588 0 : @override
589 0 : CallParticipant? get activeSpeaker => _activeSpeaker;
590 :
591 4 : @override
592 4 : WrappedMediaStream? get localUserMediaStream => _localUserMediaStream;
593 :
594 4 : @override
595 4 : WrappedMediaStream? get localScreenshareStream => _localScreenshareStream;
596 :
597 0 : @override
598 : List<WrappedMediaStream> get userMediaStreams =>
599 0 : List.unmodifiable(_userMediaStreams);
600 :
601 0 : @override
602 : List<WrappedMediaStream> get screenShareStreams =>
603 0 : List.unmodifiable(_screenshareStreams);
604 :
605 0 : @override
606 : Future<void> updateMediaDeviceForCalls() async {
607 0 : for (final call in _callSessions) {
608 0 : await call.updateMediaDeviceForCall();
609 : }
610 : }
611 :
612 : /// Initializes the local user media stream.
613 : /// The media stream must be prepared before the group call enters.
614 : /// if you allow the user to configure their camera and such ahead of time,
615 : /// you can pass that `stream` on to this function.
616 : /// This allows you to configure the camera before joining the call without
617 : /// having to reopen the stream and possibly losing settings.
618 4 : @override
619 : Future<WrappedMediaStream?> initLocalStream(
620 : GroupCallSession groupCall, {
621 : WrappedMediaStream? stream,
622 : }) async {
623 8 : if (groupCall.state != GroupCallState.localCallFeedUninitialized) {
624 0 : throw MatrixSDKVoipException(
625 0 : 'Cannot initialize local call feed in the ${groupCall.state} state.',
626 : );
627 : }
628 :
629 4 : groupCall.setState(GroupCallState.initializingLocalCallFeed);
630 :
631 : WrappedMediaStream localWrappedMediaStream;
632 :
633 : if (stream == null) {
634 : MediaStream stream;
635 :
636 : try {
637 4 : stream = await _getUserMedia(groupCall, CallType.kVideo);
638 : } catch (error) {
639 0 : groupCall.setState(GroupCallState.localCallFeedUninitialized);
640 : rethrow;
641 : }
642 :
643 4 : localWrappedMediaStream = WrappedMediaStream(
644 : stream: stream,
645 4 : participant: groupCall.localParticipant!,
646 4 : room: groupCall.room,
647 4 : client: groupCall.client,
648 : purpose: SDPStreamMetadataPurpose.Usermedia,
649 8 : audioMuted: stream.getAudioTracks().isEmpty,
650 8 : videoMuted: stream.getVideoTracks().isEmpty,
651 : isGroupCall: true,
652 4 : voip: groupCall.voip,
653 : );
654 : } else {
655 : localWrappedMediaStream = stream;
656 : }
657 :
658 4 : _localUserMediaStream = localWrappedMediaStream;
659 4 : await _addUserMediaStream(groupCall, localWrappedMediaStream);
660 :
661 4 : groupCall.setState(GroupCallState.localCallFeedInitialized);
662 :
663 4 : _activeSpeaker = null;
664 :
665 : return localWrappedMediaStream;
666 : }
667 :
668 2 : @override
669 : Future<void> setDeviceMuted(
670 : GroupCallSession groupCall,
671 : bool muted,
672 : MediaInputKind kind,
673 : ) async {
674 6 : if (!await hasMediaDevice(groupCall.voip.delegate, kind)) {
675 : return;
676 : }
677 :
678 2 : if (localUserMediaStream != null) {
679 : switch (kind) {
680 2 : case MediaInputKind.audioinput:
681 4 : localUserMediaStream!.setAudioMuted(muted);
682 2 : setTracksEnabled(
683 6 : localUserMediaStream!.stream!.getAudioTracks(),
684 : !muted,
685 : );
686 2 : for (final call in _callSessions) {
687 0 : await call.setMicrophoneMuted(muted);
688 : }
689 : break;
690 2 : case MediaInputKind.videoinput:
691 4 : localUserMediaStream!.setVideoMuted(muted);
692 2 : setTracksEnabled(
693 6 : localUserMediaStream!.stream!.getVideoTracks(),
694 : !muted,
695 : );
696 2 : for (final call in _callSessions) {
697 0 : await call.setLocalVideoMuted(muted);
698 : }
699 : break;
700 : }
701 : }
702 :
703 : // ignore: deprecated_member_use_from_same_package
704 4 : groupCall.onGroupCallEvent.add(GroupCallStateChange.localMuteStateChanged);
705 6 : groupCall.matrixRTCEventStream.add(GroupCallLocalMutedChanged(muted, kind));
706 : return;
707 : }
708 :
709 4 : void _onIncomingCallInMeshSetup(
710 : GroupCallSession groupCall,
711 : CallSession newCall,
712 : ) {
713 : // The incoming calls may be for another room, which we will ignore.
714 20 : if (newCall.room.id != groupCall.room.id) return;
715 :
716 8 : if (newCall.state != CallState.kRinging) {
717 8 : Logs().v(
718 : '[_onIncomingCallInMeshSetup] Incoming call no longer in ringing state. Ignoring.',
719 : );
720 : return;
721 : }
722 :
723 0 : if (newCall.groupCallId == null ||
724 0 : newCall.groupCallId != groupCall.groupCallId) {
725 0 : Logs().v(
726 0 : '[_onIncomingCallInMeshSetup] Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call',
727 : );
728 : return;
729 : }
730 :
731 0 : final existingCall = _getCallForParticipant(
732 : groupCall,
733 0 : CallParticipant(
734 0 : groupCall.voip,
735 0 : userId: newCall.remoteUserId!,
736 0 : deviceId: newCall.remoteDeviceId,
737 : ),
738 : );
739 :
740 : // if it's an existing call, `_registerListenersForCall` will be called in
741 : // `_replaceCall` that is used in `_onIncomingCallStart`.
742 : if (existingCall != null) return;
743 :
744 0 : Logs().v(
745 0 : '[_onIncomingCallInMeshSetup] GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}',
746 : );
747 :
748 0 : _registerListenersBeforeCallAdd(newCall);
749 : }
750 :
751 4 : Future<void> _onIncomingCallInMeshStart(
752 : GroupCallSession groupCall,
753 : CallSession newCall,
754 : ) async {
755 : // The incoming calls may be for another room, which we will ignore.
756 20 : if (newCall.room.id != groupCall.room.id) {
757 : return;
758 : }
759 :
760 8 : if (newCall.state != CallState.kRinging) {
761 8 : Logs().v(
762 : '[_onIncomingCallInMeshStart] Incoming call no longer in ringing state. Ignoring.',
763 : );
764 : return;
765 : }
766 :
767 0 : if (newCall.groupCallId == null ||
768 0 : newCall.groupCallId != groupCall.groupCallId) {
769 0 : Logs().v(
770 0 : '[_onIncomingCallInMeshStart] Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call',
771 : );
772 0 : await newCall.reject();
773 : return;
774 : }
775 :
776 0 : final existingCall = _getCallForParticipant(
777 : groupCall,
778 0 : CallParticipant(
779 0 : groupCall.voip,
780 0 : userId: newCall.remoteUserId!,
781 0 : deviceId: newCall.remoteDeviceId,
782 : ),
783 : );
784 :
785 0 : if (existingCall != null && existingCall.callId == newCall.callId) {
786 : return;
787 : }
788 :
789 0 : Logs().v(
790 0 : '[_onIncomingCallInMeshStart] GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}',
791 : );
792 :
793 : // Check if the user calling has an existing call and use this call instead.
794 : if (existingCall != null) {
795 0 : await _replaceCall(groupCall, existingCall, newCall);
796 : } else {
797 0 : await _addCall(groupCall, newCall);
798 : }
799 :
800 0 : await newCall.answerWithStreams(_getLocalStreams());
801 : }
802 :
803 2 : @override
804 : Future<void> setScreensharingEnabled(
805 : GroupCallSession groupCall,
806 : bool enabled,
807 : String desktopCapturerSourceId,
808 : ) async {
809 4 : if (enabled == (localScreenshareStream != null)) {
810 : return;
811 : }
812 :
813 : if (enabled) {
814 : try {
815 4 : Logs().v('Asking for screensharing permissions...');
816 2 : final stream = await _getDisplayMedia(groupCall);
817 2 : for (final track in stream.getTracks()) {
818 : // screen sharing should only have 1 video track anyway, so this only
819 : // fires once
820 0 : track.onEnded = () async {
821 0 : await setScreensharingEnabled(groupCall, false, '');
822 : };
823 : }
824 4 : Logs().v(
825 : 'Screensharing permissions granted. Setting screensharing enabled on all calls',
826 : );
827 4 : _localScreenshareStream = WrappedMediaStream(
828 : stream: stream,
829 2 : participant: groupCall.localParticipant!,
830 2 : room: groupCall.room,
831 2 : client: groupCall.client,
832 : purpose: SDPStreamMetadataPurpose.Screenshare,
833 4 : audioMuted: stream.getAudioTracks().isEmpty,
834 4 : videoMuted: stream.getVideoTracks().isEmpty,
835 : isGroupCall: true,
836 2 : voip: groupCall.voip,
837 : );
838 :
839 4 : _addScreenshareStream(groupCall, localScreenshareStream!);
840 :
841 : // ignore: deprecated_member_use_from_same_package
842 2 : groupCall.onGroupCallEvent
843 : // ignore: deprecated_member_use_from_same_package
844 2 : .add(GroupCallStateChange.localScreenshareStateChanged);
845 2 : groupCall.matrixRTCEventStream
846 4 : .add(GroupCallLocalScreenshareStateChanged(true));
847 2 : for (final call in _callSessions) {
848 0 : await call.addLocalStream(
849 0 : await localScreenshareStream!.stream!.clone(),
850 0 : localScreenshareStream!.purpose,
851 : );
852 : }
853 :
854 2 : await groupCall.sendMemberStateEvent();
855 :
856 : return;
857 : } catch (e, s) {
858 0 : Logs().e('[VOIP] Enabling screensharing error', e, s);
859 : // ignore: deprecated_member_use_from_same_package
860 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.error);
861 0 : groupCall.matrixRTCEventStream
862 0 : .add(GroupCallStateError(e.toString(), s));
863 : return;
864 : }
865 : } else {
866 2 : for (final call in _callSessions) {
867 0 : await call.removeLocalStream(call.localScreenSharingStream!);
868 : }
869 6 : await stopMediaStream(localScreenshareStream?.stream);
870 4 : await _removeScreenshareStream(groupCall, localScreenshareStream!);
871 2 : _localScreenshareStream = null;
872 :
873 2 : await groupCall.sendMemberStateEvent();
874 :
875 : // ignore: deprecated_member_use_from_same_package
876 2 : groupCall.onGroupCallEvent
877 : // ignore: deprecated_member_use_from_same_package
878 2 : .add(GroupCallStateChange.localMuteStateChanged);
879 2 : groupCall.matrixRTCEventStream
880 4 : .add(GroupCallLocalScreenshareStateChanged(false));
881 : return;
882 : }
883 : }
884 :
885 2 : @override
886 : Future<void> dispose(GroupCallSession groupCall) async {
887 2 : if (localUserMediaStream != null) {
888 0 : await _removeUserMediaStream(groupCall, localUserMediaStream!);
889 0 : _localUserMediaStream = null;
890 : }
891 :
892 2 : if (localScreenshareStream != null) {
893 0 : await stopMediaStream(localScreenshareStream!.stream);
894 0 : await _removeScreenshareStream(groupCall, localScreenshareStream!);
895 0 : _localScreenshareStream = null;
896 : }
897 :
898 : // removeCall removes it from `_callSessions` later.
899 4 : final callsCopy = _callSessions.toList();
900 :
901 2 : for (final call in callsCopy) {
902 0 : await _removeCall(groupCall, call, CallErrorCode.userHangup);
903 : }
904 :
905 2 : _activeSpeaker = null;
906 2 : _activeSpeakerLoopTimeout?.cancel();
907 2 : await _callSetupSubscription?.cancel();
908 2 : await _callStartSubscription?.cancel();
909 : }
910 :
911 0 : @override
912 : bool get isLocalVideoMuted {
913 0 : if (localUserMediaStream != null) {
914 0 : return localUserMediaStream!.isVideoMuted();
915 : }
916 :
917 : return true;
918 : }
919 :
920 0 : @override
921 : bool get isMicrophoneMuted {
922 0 : if (localUserMediaStream != null) {
923 0 : return localUserMediaStream!.isAudioMuted();
924 : }
925 :
926 : return true;
927 : }
928 :
929 4 : @override
930 : Future<void> setupP2PCallsWithExistingMembers(
931 : GroupCallSession groupCall,
932 : ) async {
933 8 : for (final call in _callSessions) {
934 4 : _onIncomingCallInMeshSetup(groupCall, call);
935 4 : await _onIncomingCallInMeshStart(groupCall, call);
936 : }
937 :
938 20 : _callSetupSubscription = groupCall.voip.onIncomingCallSetup.stream.listen(
939 0 : (newCall) => _onIncomingCallInMeshSetup(groupCall, newCall),
940 : );
941 :
942 20 : _callStartSubscription = groupCall.voip.onIncomingCallStart.stream.listen(
943 0 : (newCall) => _onIncomingCallInMeshStart(groupCall, newCall),
944 : );
945 :
946 4 : _onActiveSpeakerLoop(groupCall);
947 : }
948 :
949 4 : @override
950 : Future<void> setupP2PCallWithNewMember(
951 : GroupCallSession groupCall,
952 : CallParticipant rp,
953 : CallMembership mem,
954 : ) async {
955 4 : final existingCall = _getCallForParticipant(groupCall, rp);
956 : if (existingCall != null) {
957 0 : if (existingCall.remoteSessionId != mem.membershipId) {
958 0 : await existingCall.hangup(reason: CallErrorCode.unknownError);
959 : } else {
960 0 : Logs().e(
961 0 : '[VOIP] onMemberStateChanged Not updating _participants list, already have a ongoing call with ${rp.id}',
962 : );
963 : return;
964 : }
965 : }
966 :
967 : // Only initiate a call with a participant who has a id that is lexicographically
968 : // less than your own. Otherwise, that user will call you.
969 20 : if (groupCall.localParticipant!.id.compareTo(rp.id) > 0) {
970 16 : Logs().i('[VOIP] Waiting for ${rp.id} to send call invite.');
971 : return;
972 : }
973 :
974 4 : final opts = CallOptions(
975 4 : callId: genCallID(),
976 4 : room: groupCall.room,
977 4 : voip: groupCall.voip,
978 : dir: CallDirection.kOutgoing,
979 8 : localPartyId: groupCall.voip.currentSessionId,
980 4 : groupCallId: groupCall.groupCallId,
981 : type: CallType.kVideo,
982 8 : iceServers: await groupCall.voip.getIceServers(),
983 : );
984 8 : final newCall = groupCall.voip.createNewCall(opts);
985 :
986 : /// both invitee userId and deviceId are set here because there can be
987 : /// multiple devices from same user in a call, so we specifiy who the
988 : /// invite is for
989 : ///
990 : /// MOVE TO CREATENEWCALL?
991 8 : newCall.remoteUserId = mem.userId;
992 8 : newCall.remoteDeviceId = mem.deviceId;
993 : // party id set to when answered
994 8 : newCall.remoteSessionId = mem.membershipId;
995 :
996 4 : _registerListenersBeforeCallAdd(newCall);
997 :
998 4 : await newCall.placeCallWithStreams(
999 4 : _getLocalStreams(),
1000 6 : requestScreenSharing: mem.feeds?.any(
1001 0 : (element) =>
1002 0 : element['purpose'] == SDPStreamMetadataPurpose.Screenshare,
1003 : ) ??
1004 : false,
1005 : );
1006 :
1007 4 : await _addCall(groupCall, newCall);
1008 : }
1009 :
1010 4 : @override
1011 : List<Map<String, String>>? getCurrentFeeds() {
1012 4 : return _getLocalStreams()
1013 4 : .map(
1014 8 : (feed) => ({
1015 4 : 'purpose': feed.purpose,
1016 : }),
1017 : )
1018 4 : .toList();
1019 : }
1020 :
1021 0 : @override
1022 : bool operator ==(Object other) =>
1023 0 : identical(this, other) || (other is MeshBackend && type == other.type);
1024 0 : @override
1025 0 : int get hashCode => type.hashCode;
1026 :
1027 : /// get everything is livekit specific mesh calls shouldn't be affected by these
1028 0 : @override
1029 : Future<void> onCallEncryption(
1030 : GroupCallSession groupCall,
1031 : String userId,
1032 : String deviceId,
1033 : Map<String, dynamic> content,
1034 : ) async {
1035 : return;
1036 : }
1037 :
1038 0 : @override
1039 : Future<void> onCallEncryptionKeyRequest(
1040 : GroupCallSession groupCall,
1041 : String userId,
1042 : String deviceId,
1043 : Map<String, dynamic> content,
1044 : ) async {
1045 : return;
1046 : }
1047 :
1048 2 : @override
1049 : Future<void> onLeftParticipant(
1050 : GroupCallSession groupCall,
1051 : List<CallParticipant> anyLeft,
1052 : ) async {
1053 : return;
1054 : }
1055 :
1056 2 : @override
1057 : Future<void> onNewParticipant(
1058 : GroupCallSession groupCall,
1059 : List<CallParticipant> anyJoined,
1060 : ) async {
1061 : return;
1062 : }
1063 :
1064 0 : @override
1065 : Future<void> requestEncrytionKey(
1066 : GroupCallSession groupCall,
1067 : List<CallParticipant> remoteParticipants,
1068 : ) async {
1069 : return;
1070 : }
1071 :
1072 0 : @override
1073 : Future<void> preShareKey(GroupCallSession groupCall) async {
1074 : return;
1075 : }
1076 : }
|