Skip to content

Commit 616d47c

Browse files
youennfyouennf
committed
Safari local video element pauses after bluetooth audioinput is disconnected
https://bugs.webkit.org/show_bug.cgi?id=231787 rdar://problem/84529041 Reviewed by Eric Carlson. We receive a remote pause command when BT is disconnected. We also get remote commands from keyboard and the current heuristic does not work well with video conference websites that have multiple media elements playing at the same time. We introduce a new heuristic in that case, where instead of pausing/playing media elements, we mute/unmute capture and audio rendering. This allow users for instance to restart capture/audio using Safari UI. We update WebPlaybackControlsManager setPlaying to always send an IPC message since calling playing may unmute WebProcess. Covered by API test. * Source/WebCore/dom/Document.cpp: (WebCore::Document::updateIsPlayingMedia): * Source/WebCore/dom/Document.h: (WebCore::Document::activeMediaElementsWithMediaStreamCount const): * Source/WebCore/html/MediaElementSession.cpp: (WebCore::isDocumentPlayingSeveralMediaStreams): (WebCore::processRemoteControlCommandIfPlayingMediaStreams): (WebCore::MediaElementSession::didReceiveRemoteControlCommand): (WebCore::MediaElementSession::nowPlayingInfo const): * Source/WebCore/platform/mac/WebPlaybackControlsManager.mm: (-[WebPlaybackControlsManager setPlaying:]): * Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj: * Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm: * Tools/TestWebKitAPI/Tests/WebKitCocoa/webrtc-remote.html: Added. Canonical link: https://commits.webkit.org/259478@main
1 parent 5357882 commit 616d47c

File tree

7 files changed

+203
-7
lines changed

7 files changed

+203
-7
lines changed

‎Source/WebCore/dom/Document.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4592,6 +4592,12 @@ void Document::updateIsPlayingMedia()
45924592
state.add(MediaStreamTrack::captureState(*this));
45934593
if (m_activeSpeechRecognition)
45944594
state.add(MediaProducerMediaState::HasActiveAudioCaptureDevice);
4595+
4596+
m_activeMediaElementsWithMediaStreamCount = 0;
4597+
forEachMediaElement([&](auto& element) {
4598+
if (element.hasMediaStreamSrcObject() && element.isPlaying())
4599+
++m_activeMediaElementsWithMediaStreamCount;
4600+
});
45954601
#endif
45964602

45974603
if (m_userHasInteractedWithMediaElement)

‎Source/WebCore/dom/Document.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1525,6 +1525,7 @@ class Document
15251525
bool hasHadCaptureMediaStreamTrack() const { return m_hasHadCaptureMediaStreamTrack; }
15261526
void stopMediaCapture(MediaProducerMediaCaptureKind);
15271527
void mediaStreamCaptureStateChanged();
1528+
size_t activeMediaElementsWithMediaStreamCount() const { return m_activeMediaElementsWithMediaStreamCount; }
15281529
#endif
15291530

15301531
// FIXME: Find a better place for this functionality.
@@ -2265,6 +2266,7 @@ class Document
22652266
#if ENABLE(MEDIA_STREAM)
22662267
String m_idHashSalt;
22672268
bool m_hasHadCaptureMediaStreamTrack { false };
2269+
size_t m_activeMediaElementsWithMediaStreamCount { 0 };
22682270
#endif
22692271

22702272
#if ASSERT_ENABLED

‎Source/WebCore/html/MediaElementSession.cpp‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1118,10 +1118,56 @@ bool MediaElementSession::allowsPlaybackControlsForAutoplayingAudio() const
11181118
}
11191119

11201120
#if ENABLE(MEDIA_SESSION)
1121+
#if ENABLE(MEDIA_STREAM)
1122+
static bool isDocumentPlayingSeveralMediaStreams(Document& document)
1123+
{
1124+
// We restrict to capturing document for now, until we have a good way to state to the UIProcess application that audio rendering is muted from here.
1125+
return document.activeMediaElementsWithMediaStreamCount() > 1 && MediaProducer::isCapturing(document.mediaState());
1126+
}
1127+
1128+
static bool processRemoteControlCommandIfPlayingMediaStreams(Document& document, PlatformMediaSession::RemoteControlCommandType commandType)
1129+
{
1130+
auto* page = document.page();
1131+
if (!page)
1132+
return false;
1133+
1134+
if (!isDocumentPlayingSeveralMediaStreams(document))
1135+
return false;
1136+
1137+
WebCore::MediaProducerMutedStateFlags mutedState;
1138+
mutedState.add(WebCore::MediaProducerMutedState::AudioIsMuted);
1139+
mutedState.add(WebCore::MediaProducer::AudioAndVideoCaptureIsMuted);
1140+
mutedState.add(WebCore::MediaProducerMutedState::ScreenCaptureIsMuted);
1141+
1142+
switch (commandType) {
1143+
case PlatformMediaSession::PlayCommand:
1144+
page->setMuted({ });
1145+
return true;
1146+
case PlatformMediaSession::StopCommand:
1147+
case PlatformMediaSession::PauseCommand:
1148+
page->setMuted(mutedState);
1149+
return true;
1150+
case PlatformMediaSession::TogglePlayPauseCommand:
1151+
if (page->mutedState().containsAny(mutedState))
1152+
page->setMuted({ });
1153+
else
1154+
page->setMuted(mutedState);
1155+
return true;
1156+
default:
1157+
break;
1158+
}
1159+
return false;
1160+
}
1161+
#endif
1162+
11211163
void MediaElementSession::didReceiveRemoteControlCommand(RemoteControlCommandType commandType, const RemoteCommandArgument& argument)
11221164
{
11231165
auto* session = mediaSession();
11241166
if (!session || !session->hasActiveActionHandlers()) {
1167+
#if ENABLE(MEDIA_STREAM)
1168+
if (processRemoteControlCommandIfPlayingMediaStreams(m_element.document(), commandType))
1169+
return;
1170+
#endif
11251171
PlatformMediaSession::didReceiveRemoteControlCommand(commandType, argument);
11261172
return;
11271173
}
@@ -1189,6 +1235,11 @@ std::optional<NowPlayingInfo> MediaElementSession::nowPlayingInfo() const
11891235
auto* page = m_element.document().page();
11901236
bool allowsNowPlayingControlsVisibility = page && !page->isVisibleAndActive();
11911237
bool isPlaying = state() == PlatformMediaSession::Playing;
1238+
#if ENABLE(MEDIA_SESSION) && ENABLE(MEDIA_STREAM)
1239+
if (isPlaying && isDocumentPlayingSeveralMediaStreams(m_element.document()) && page)
1240+
isPlaying = !page->mutedState().contains(MediaProducerMutedState::AudioIsMuted);
1241+
#endif
1242+
11921243
bool supportsSeeking = m_element.supportsSeeking();
11931244
double rate = 1.0;
11941245
double duration = supportsSeeking ? m_element.duration() : MediaPlayer::invalidTime();

‎Source/WebCore/platform/mac/WebPlaybackControlsManager.mm‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -341,13 +341,8 @@ - (void)setPlaying:(BOOL)playing
341341
if (!_playbackSessionInterfaceMac)
342342
return;
343343

344-
if (auto* model = _playbackSessionInterfaceMac->playbackSessionModel()) {
345-
BOOL isCurrentlyPlaying = model->isPlaying();
346-
if (!isCurrentlyPlaying && _playing)
347-
model->sendRemoteCommand(WebCore::PlatformMediaSession::RemoteControlCommandType::PlayCommand, { });
348-
else if (isCurrentlyPlaying && !_playing)
349-
model->sendRemoteCommand(WebCore::PlatformMediaSession::RemoteControlCommandType::PauseCommand, { });
350-
}
344+
if (auto* model = _playbackSessionInterfaceMac->playbackSessionModel())
345+
model->sendRemoteCommand(_playing ? WebCore::PlatformMediaSession::RemoteControlCommandType::PlayCommand : WebCore::PlatformMediaSession::RemoteControlCommandType::PauseCommand, { });
351346
}
352347

353348
- (BOOL)isPlaying

‎Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
076E507F1F4513D6006E9F5A /* Logging.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 076E507E1F45031E006E9F5A /* Logging.cpp */; };
6363
077A5AF3230638A600A7105C /* AccessibilityTestPlugin.mm in Sources */ = {isa = PBXBuildFile; fileRef = 0746645822FF630500E3451A /* AccessibilityTestPlugin.mm */; };
6464
0794742D25CB33FD00C597EB /* media-remote.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = 0794742C25CB33B000C597EB /* media-remote.html */; };
65+
0794742D25CB33FD00C597EC /* webrtc-remote.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = 0794742C25CB33B000C597EC /* webrtc-remote.html */; };
6566
0799C34B1EBA3301003B7532 /* disableGetUserMedia.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = 0799C34A1EBA32F4003B7532 /* disableGetUserMedia.html */; };
6667
07C046CA1E4262A8007201E7 /* CARingBuffer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 07C046C91E42573E007201E7 /* CARingBuffer.cpp */; };
6768
07CE1CF31F06A7E000BF89F5 /* GetUserMediaNavigation.mm in Sources */ = {isa = PBXBuildFile; fileRef = 07CE1CF21F06A7E000BF89F5 /* GetUserMediaNavigation.mm */; };
@@ -1887,6 +1888,7 @@
18871888
F45C640128178AD70090DFAB /* webp-image.html in Copy Resources */,
18881889
51714EB41CF8C78C004723C4 /* WebProcessKillIDBCleanup-1.html in Copy Resources */,
18891890
51714EB51CF8C78C004723C4 /* WebProcessKillIDBCleanup-2.html in Copy Resources */,
1891+
0794742D25CB33FD00C597EC /* webrtc-remote.html in Copy Resources */,
18901892
536770361CC81B6100D425B1 /* WebScriptObjectDescription.html in Copy Resources */,
18911893
5120C83E1E67678F0025B250 /* WebsiteDataStoreCustomPaths.html in Copy Resources */,
18921894
93F79A5A28E64A06003E7CEB /* websql-database-tracker.db in Copy Resources */,
@@ -1943,6 +1945,7 @@
19431945
076E507E1F45031E006E9F5A /* Logging.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Logging.cpp; sourceTree = "<group>"; };
19441946
0794740C25CA0BDE00C597EB /* MediaSession.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MediaSession.mm; sourceTree = "<group>"; };
19451947
0794742C25CB33B000C597EB /* media-remote.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "media-remote.html"; sourceTree = "<group>"; };
1948+
0794742C25CB33B000C597EC /* webrtc-remote.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "webrtc-remote.html"; sourceTree = "<group>"; };
19461949
0799C34A1EBA32F4003B7532 /* disableGetUserMedia.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = disableGetUserMedia.html; sourceTree = "<group>"; };
19471950
07C046C91E42573E007201E7 /* CARingBuffer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CARingBuffer.cpp; sourceTree = "<group>"; };
19481951
07CC7DFD2266330800E39181 /* MediaBufferingPolicy.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MediaBufferingPolicy.mm; sourceTree = "<group>"; };
@@ -4801,6 +4804,7 @@
48014804
F45C63FE28178A530090DFAB /* webp-image.html */,
48024805
51714EB21CF8C761004723C4 /* WebProcessKillIDBCleanup-1.html */,
48034806
51714EB31CF8C761004723C4 /* WebProcessKillIDBCleanup-2.html */,
4807+
0794742C25CB33B000C597EC /* webrtc-remote.html */,
48044808
5120C83B1E674E350025B250 /* WebsiteDataStoreCustomPaths.html */,
48054809
93F79A5928E649EC003E7CEB /* websql-database-tracker.db */,
48064810
93F79A5128E649EB003E7CEB /* websql-database.db */,

‎Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#import <WebKit/_WKFeature.h>
4646
#import <WebKit/_WKProcessPoolConfiguration.h>
4747
#import <WebKit/_WKWebsiteDataStoreConfiguration.h>
48+
#import <pal/spi/mac/MediaRemoteSPI.h>
4849
#import <wtf/text/StringBuilder.h>
4950
#import <wtf/text/WTFString.h>
5051

@@ -1269,6 +1270,83 @@ function capture()
12691270
}
12701271
#endif
12711272

1273+
#if WK_HAVE_C_SPI
1274+
TEST(WebKit2, WebRTCAndRemoteCommands)
1275+
{
1276+
auto configuration = adoptNS([[WKWebViewConfiguration alloc] init]);
1277+
auto context = adoptWK(TestWebKitAPI::Util::createContextForInjectedBundleTest("InternalsInjectedBundleTest"));
1278+
configuration.get().processPool = (WKProcessPool *)context.get();
1279+
configuration.get().processPool._configuration.shouldCaptureAudioInUIProcess = NO;
1280+
1281+
initializeMediaCaptureConfiguration(configuration.get());
1282+
1283+
auto messageHandler = adoptNS([[GUMMessageHandler alloc] init]);
1284+
[[configuration.get() userContentController] addScriptMessageHandler:messageHandler.get() name:@"gum"];
1285+
1286+
auto webView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 320, 500) configuration:configuration.get()]);
1287+
1288+
auto delegate = adoptNS([[UserMediaCaptureUIDelegate alloc] init]);
1289+
[webView setUIDelegate:delegate.get()];
1290+
[webView _setMediaCaptureReportingDelayForTesting:0];
1291+
1292+
auto observer = adoptNS([[MediaCaptureObserver alloc] init]);
1293+
[webView addObserver:observer.get() forKeyPath:@"microphoneCaptureState" options:NSKeyValueObservingOptionNew context:nil];
1294+
[webView addObserver:observer.get() forKeyPath:@"cameraCaptureState" options:NSKeyValueObservingOptionNew context:nil];
1295+
1296+
cameraCaptureStateChange = false;
1297+
microphoneCaptureStateChange = false;
1298+
1299+
done = false;
1300+
[webView loadTestPageNamed:@"webrtc-remote"];
1301+
1302+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
1303+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1304+
1305+
done = false;
1306+
[webView stringByEvaluatingJavaScript:@"startTest()"];
1307+
TestWebKitAPI::Util::run(&done);
1308+
done = false;
1309+
1310+
cameraCaptureStateChange = false;
1311+
microphoneCaptureStateChange = false;
1312+
[webView stringByEvaluatingJavaScript:@"sendCommand('pause')"];
1313+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateMuted));
1314+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateMuted));
1315+
1316+
[webView stringByEvaluatingJavaScript:@"sendCommand('play')"];
1317+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
1318+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1319+
1320+
[webView stringByEvaluatingJavaScript:@"sendCommand('toggleplaypause')"];
1321+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateMuted));
1322+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateMuted));
1323+
1324+
[webView stringByEvaluatingJavaScript:@"sendCommand('toggleplaypause')"];
1325+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
1326+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1327+
1328+
done = false;
1329+
// register handlers will catch commands, so capture should not muted.
1330+
[webView stringByEvaluatingJavaScript:@"registerHandlers()"];
1331+
TestWebKitAPI::Util::run(&done);
1332+
done = false;
1333+
1334+
[webView stringByEvaluatingJavaScript:@"sendCommand('pause')"];
1335+
TestWebKitAPI::Util::run(&done);
1336+
done = false;
1337+
1338+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
1339+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1340+
1341+
[webView stringByEvaluatingJavaScript:@"sendCommand('play')"];
1342+
TestWebKitAPI::Util::run(&done);
1343+
done = false;
1344+
1345+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
1346+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1347+
}
1348+
#endif // WK_HAVE_C_SPI
1349+
12721350
} // namespace TestWebKitAPI
12731351

12741352
#endif // ENABLE(MEDIA_STREAM)

‎Tools/TestWebKitAPI/Tests/WebKitCocoa/webrtc-remote.html‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
</head>
5+
<body onload="startCapture()">
6+
<video controls autoplay playsinline muted id="local"></video>
7+
<br>
8+
<video controls autoplay playsinline id="remote"></video>
9+
<script>
10+
let stream;
11+
async function startCapture()
12+
{
13+
stream = await navigator.mediaDevices.getUserMedia({audio:true, video: true});
14+
local.srcObject = stream;
15+
// We emulate the remote stream by cloning the local stream.
16+
remote.srcObject = stream.clone();
17+
}
18+
19+
async function startPlaying()
20+
{
21+
await local.play();
22+
await remote.play();
23+
if (window.webkit)
24+
window.webkit.messageHandlers.gum.postMessage("PASS");
25+
}
26+
27+
function startTest()
28+
{
29+
startPlaying();
30+
}
31+
32+
let currentCommand;
33+
function sendCommand(command)
34+
{
35+
if (!window.internals) {
36+
if (window.webkit)
37+
window.webkit.messageHandlers.gum.postMessage("FAIL");
38+
return;
39+
}
40+
currentCommand = command;
41+
if (window.webkit)
42+
window.internals.postRemoteControlCommand(command);
43+
}
44+
45+
function registerHandlers()
46+
{
47+
navigator.mediaSession.setActionHandler("play", () => {
48+
if (window.webkit)
49+
window.webkit.messageHandlers.gum.postMessage(currentCommand == "play" ? "PASS" : "FAIL, got play but expected " + currentCommand);
50+
});
51+
navigator.mediaSession.setActionHandler("pause", () => {
52+
if (window.webkit)
53+
window.webkit.messageHandlers.gum.postMessage(currentCommand == "pause" ? "PASS" : "FAIL, got pause but expected " + currentCommand);
54+
});
55+
if (window.webkit)
56+
window.webkit.messageHandlers.gum.postMessage("PASS");
57+
}
58+
</script>
59+
</body>
60+
</html>

0 commit comments

Comments
 (0)