Skip to content

Commit df85955

Browse files
youennfyouennf
committed
setCameraActive should not unmute microphone if UA previously muted both camera and microphone
rdar://136221456 https://bugs.webkit.org/show_bug.cgi?id=279889 Reviewed by Eric Carlson. When trying to unmute capture via setCameraActive/setMicrophoneActive a capture that was muted by the UA, we would unmute both microphone and camera at the same time. This is probably not great for the user. For that reason, we now store in WebPageProxy whether the web page would like to have capture muted (for each capture type). This new state, called m_mutedCaptureKindsDesiredByWebApp, can be changed in two ways: - this state can be set to muted whenever UA decides so (WKWebView user). - this state can be set to muted/unmuted via setCameraActive/setMicrophoneActive if successful. The muted state sent to WebPage in WebPageProxy::setMuted is updated according m_mutedCaptureKindsDesiredByWebApp. This allows to not restart microphone capture if the web app only asked to restart camera capture. Note that m_mutedCaptureKindsDesiredByWebApp is only used to further mute capture (and not unmute capture). Covered by added and updated API tests. * Source/WebCore/page/MediaProducer.h: * Source/WebKit/UIProcess/API/C/WKPage.cpp: (WKPageSetMuted): * Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm: (-[WKWebView setMicrophoneCaptureState:completionHandler:]): (-[WKWebView setCameraCaptureState:completionHandler:]): (-[WKWebView _setPageMuted:]): (-[WKWebView _setDisplayCaptureState:completionHandler:]): (-[WKWebView _setSystemAudioCaptureState:completionHandler:]): * Source/WebKit/UIProcess/WebPageProxy.cpp: (WebKit::WebPageProxy::didCommitLoadForFrame): (WebKit::applyWebAppDesiredMutedKinds): (WebKit::updateMutedCaptureKindsDesiredByWebApp): (WebKit::WebPageProxy::setMuted): (WebKit::WebPageProxy::resetState): (WebKit::WebPageProxy::willStartCapture): (WebKit::WebPageProxy::validateCaptureStateUpdate): * Source/WebKit/UIProcess/WebPageProxy.h: * Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm: (TestWebKitAPI::(WebKit2, ToggleCameraCaptureWhenRestarting)): (TestWebKitAPI::(WebKit2, ToggleMicrophoneCaptureWhenRestarting)): (TestWebKitAPI::(WebKit2, ToggleCaptureWhenRestarting)): Deleted. Canonical link: https://commits.webkit.org/284069@main
1 parent 40f9d7c commit df85955

File tree

6 files changed

+167
-24
lines changed

6 files changed

+167
-24
lines changed

‎Source/WebCore/page/MediaProducer.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,11 @@ enum class MediaProducerMediaState : uint32_t {
7272
using MediaProducerMediaStateFlags = OptionSet<MediaProducerMediaState>;
7373

7474
enum class MediaProducerMediaCaptureKind : uint8_t {
75-
Microphone,
76-
Camera,
77-
Display,
78-
SystemAudio,
79-
EveryKind,
75+
Microphone = 1 << 0,
76+
Camera = 1 << 1,
77+
Display = 1 << 2,
78+
SystemAudio = 1 << 3,
79+
EveryKind = 1 << 4,
8080
};
8181

8282
enum class MediaProducerMutedState : uint8_t {

‎Source/WebKit/UIProcess/API/C/WKPage.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2827,7 +2827,7 @@ void WKPageSetMuted(WKPageRef pageRef, WKMediaMutedState mutedState)
28272827
if (mutedState & kWKMediaMicrophoneCaptureUnmuted)
28282828
coreState.remove(WebCore::MediaProducerMutedState::AudioCaptureIsMuted);
28292829

2830-
toImpl(pageRef)->setMuted(coreState);
2830+
toImpl(pageRef)->setMuted(coreState, WebKit::WebPageProxy::FromApplication::Yes);
28312831
}
28322832

28332833
void WKPageSetMediaCaptureEnabled(WKPageRef pageRef, bool enabled)

‎Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1122,7 +1122,7 @@ - (void)setMicrophoneCaptureState:(WKMediaCaptureState)state completionHandler:(
11221122
mutedState.remove(WebCore::MediaProducerMutedState::AudioCaptureIsMuted);
11231123
else if (state == WKMediaCaptureStateMuted)
11241124
mutedState.add(WebCore::MediaProducerMutedState::AudioCaptureIsMuted);
1125-
_page->setMuted(mutedState, [completionHandler = makeBlockPtr(completionHandler)] {
1125+
_page->setMuted(mutedState, WebKit::WebPageProxy::FromApplication::Yes, [completionHandler = makeBlockPtr(completionHandler)] {
11261126
completionHandler();
11271127
});
11281128
}
@@ -1144,7 +1144,7 @@ - (void)setCameraCaptureState:(WKMediaCaptureState)state completionHandler:(void
11441144
mutedState.remove(WebCore::MediaProducerMutedState::VideoCaptureIsMuted);
11451145
else if (state == WKMediaCaptureStateMuted)
11461146
mutedState.add(WebCore::MediaProducerMutedState::VideoCaptureIsMuted);
1147-
_page->setMuted(mutedState, [completionHandler = makeBlockPtr(completionHandler)] {
1147+
_page->setMuted(mutedState, WebKit::WebPageProxy::FromApplication::Yes, [completionHandler = makeBlockPtr(completionHandler)] {
11481148
completionHandler();
11491149
});
11501150
}
@@ -4532,7 +4532,7 @@ - (void)_setPageMuted:(_WKMediaMutedState)mutedState
45324532
if (mutedState & _WKMediaScreenCaptureMuted)
45334533
coreState.add(WebCore::MediaProducerMutedState::ScreenCaptureIsMuted);
45344534

4535-
_page->setMuted(coreState);
4535+
_page->setMuted(coreState, WebKit::WebPageProxy::FromApplication::Yes);
45364536
}
45374537

45384538
- (void)_removeDataDetectedLinks:(dispatch_block_t)completion
@@ -4625,7 +4625,7 @@ - (void)_setDisplayCaptureState:(WKDisplayCaptureState)state completionHandler:(
46254625
mutedState.remove(displayMutedFlags);
46264626
else if (state == WKDisplayCaptureStateMuted)
46274627
mutedState.add(displayMutedFlags);
4628-
_page->setMuted(mutedState, [completionHandler = makeBlockPtr(completionHandler)] {
4628+
_page->setMuted(mutedState, WebKit::WebPageProxy::FromApplication::Yes, [completionHandler = makeBlockPtr(completionHandler)] {
46294629
completionHandler();
46304630
});
46314631
}
@@ -4647,7 +4647,7 @@ - (void)_setSystemAudioCaptureState:(WKSystemAudioCaptureState)state completionH
46474647
mutedState.remove(WebCore::MediaProducerMutedState::WindowCaptureIsMuted);
46484648
else if (state == WKSystemAudioCaptureStateMuted)
46494649
mutedState.add(WebCore::MediaProducerMutedState::WindowCaptureIsMuted);
4650-
_page->setMuted(mutedState, [completionHandler = makeBlockPtr(completionHandler)] {
4650+
_page->setMuted(mutedState, WebKit::WebPageProxy::FromApplication::Yes, [completionHandler = makeBlockPtr(completionHandler)] {
46514651
completionHandler();
46524652
});
46534653
}

‎Source/WebKit/UIProcess/WebPageProxy.cpp‎

Lines changed: 83 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6781,6 +6781,10 @@ void WebPageProxy::didCommitLoadForFrame(IPC::Connection& connection, FrameIdent
67816781
#if ENABLE(MEDIA_STREAM)
67826782
if (m_userMediaPermissionRequestManager)
67836783
m_userMediaPermissionRequestManager->didCommitLoadForFrame(frameID);
6784+
if (frame->isMainFrame()) {
6785+
m_shouldListenToVoiceActivity = false;
6786+
m_mutedCaptureKindsDesiredByWebApp = { };
6787+
}
67846788
#endif
67856789

67866790
#if ENABLE(EXTENSION_CAPABILITIES)
@@ -8460,8 +8464,59 @@ void WebPageProxy::setMediaVolume(float volume)
84608464
send(Messages::WebPage::SetMediaVolume(volume));
84618465
}
84628466

8463-
void WebPageProxy::setMuted(WebCore::MediaProducerMutedStateFlags state, CompletionHandler<void()>&& completionHandler)
8467+
#if ENABLE(MEDIA_STREAM)
8468+
static WebCore::MediaProducerMutedStateFlags applyWebAppDesiredMutedKinds(WebCore::MediaProducerMutedStateFlags state, OptionSet<WebCore::MediaProducerMediaCaptureKind> desiredMutedKinds)
8469+
{
8470+
if (desiredMutedKinds.contains(WebCore::MediaProducerMediaCaptureKind::EveryKind))
8471+
state.add(MediaProducer::MediaStreamCaptureIsMuted);
8472+
else {
8473+
if (desiredMutedKinds.contains(WebCore::MediaProducerMediaCaptureKind::Microphone))
8474+
state.add(WebCore::MediaProducer::MutedState::AudioCaptureIsMuted);
8475+
if (desiredMutedKinds.contains(WebCore::MediaProducerMediaCaptureKind::Camera))
8476+
state.add(WebCore::MediaProducer::MutedState::VideoCaptureIsMuted);
8477+
if (desiredMutedKinds.contains(WebCore::MediaProducerMediaCaptureKind::Display)) {
8478+
state.add(WebCore::MediaProducer::MutedState::ScreenCaptureIsMuted);
8479+
state.add(WebCore::MediaProducer::MutedState::WindowCaptureIsMuted);
8480+
}
8481+
if (desiredMutedKinds.contains(WebCore::MediaProducerMediaCaptureKind::SystemAudio))
8482+
state.add(WebCore::MediaProducer::MutedState::SystemAudioCaptureIsMuted);
8483+
}
8484+
8485+
return state;
8486+
}
8487+
8488+
static void updateMutedCaptureKindsDesiredByWebApp(OptionSet<WebCore::MediaProducerMediaCaptureKind>& mutedCaptureKindsDesiredByWebApp, WebCore::MediaProducerMutedStateFlags newState)
8489+
{
8490+
if (newState.contains(WebCore::MediaProducerMutedState::AudioCaptureIsMuted))
8491+
mutedCaptureKindsDesiredByWebApp.add(WebCore::MediaProducerMediaCaptureKind::Microphone);
8492+
else
8493+
mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::Microphone);
8494+
8495+
if (newState.contains(WebCore::MediaProducerMutedState::VideoCaptureIsMuted))
8496+
mutedCaptureKindsDesiredByWebApp.add(WebCore::MediaProducerMediaCaptureKind::Camera);
8497+
else
8498+
mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::Camera);
8499+
8500+
if (newState.contains(WebCore::MediaProducerMutedState::ScreenCaptureIsMuted)
8501+
|| newState.contains(WebCore::MediaProducerMutedState::WindowCaptureIsMuted))
8502+
mutedCaptureKindsDesiredByWebApp.add(WebCore::MediaProducerMediaCaptureKind::Display);
8503+
else
8504+
mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::Display);
8505+
8506+
if (newState.contains(WebCore::MediaProducerMutedState::SystemAudioCaptureIsMuted))
8507+
mutedCaptureKindsDesiredByWebApp.add(WebCore::MediaProducerMediaCaptureKind::SystemAudio);
8508+
else
8509+
mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::SystemAudio);
8510+
}
8511+
#endif
8512+
8513+
void WebPageProxy::setMuted(WebCore::MediaProducerMutedStateFlags state, FromApplication fromApplication, CompletionHandler<void()>&& completionHandler)
84648514
{
8515+
#if ENABLE(MEDIA_STREAM)
8516+
if (fromApplication == FromApplication::Yes)
8517+
updateMutedCaptureKindsDesiredByWebApp(m_mutedCaptureKindsDesiredByWebApp, state);
8518+
#endif
8519+
84658520
if (!isAllowedToChangeMuteState())
84668521
state.add(WebCore::MediaProducer::MediaStreamCaptureIsMuted);
84678522

@@ -8478,17 +8533,17 @@ void WebPageProxy::setMuted(WebCore::MediaProducerMutedStateFlags state, Complet
84788533

84798534
protectedLegacyMainFrameProcess()->pageMutedStateChanged(m_webPageID, state);
84808535

8481-
WEBPAGEPROXY_RELEASE_LOG(Media, "setMuted: %d", state.toRaw());
8536+
#if ENABLE(MEDIA_STREAM)
8537+
auto newState = applyWebAppDesiredMutedKinds(state, m_mutedCaptureKindsDesiredByWebApp);
8538+
#else
8539+
auto newState = state;
8540+
#endif
8541+
WEBPAGEPROXY_RELEASE_LOG(Media, "setMuted, app state = %d, final state = %d", state.toRaw(), newState.toRaw());
84828542

8483-
sendWithAsyncReply(Messages::WebPage::SetMuted(state), WTFMove(completionHandler));
8543+
sendWithAsyncReply(Messages::WebPage::SetMuted(newState), WTFMove(completionHandler));
84848544
activityStateDidChange({ ActivityState::IsAudible, ActivityState::IsCapturingMedia });
84858545
}
84868546

8487-
void WebPageProxy::setMuted(WebCore::MediaProducerMutedStateFlags state)
8488-
{
8489-
setMuted(state, [] { });
8490-
}
8491-
84928547
void WebPageProxy::setMediaCaptureEnabled(bool enabled)
84938548
{
84948549
m_mediaCaptureEnabled = enabled;
@@ -10529,6 +10584,7 @@ void WebPageProxy::resetState(ResetStateReason resetStateReason)
1052910584
#if ENABLE(MEDIA_STREAM)
1053010585
m_userMediaPermissionRequestManager = nullptr;
1053110586
m_shouldListenToVoiceActivity = false;
10587+
m_mutedCaptureKindsDesiredByWebApp = { };
1053210588
#endif
1053310589

1053410590
#if ENABLE(POINTER_LOCK)
@@ -11369,6 +11425,21 @@ void WebPageProxy::willStartCapture(UserMediaPermissionRequestProxy& request, Co
1136911425
if (auto beforeStartingCaptureCallback = request.beforeStartingCaptureCallback())
1137011426
beforeStartingCaptureCallback();
1137111427

11428+
switch (request.requestType()) {
11429+
case WebCore::MediaStreamRequest::Type::UserMedia:
11430+
if (request.userRequest().audioConstraints.isValid)
11431+
m_mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::Microphone);
11432+
if (request.userRequest().videoConstraints.isValid)
11433+
m_mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::Camera);
11434+
break;
11435+
case WebCore::MediaStreamRequest::Type::DisplayMediaWithAudio:
11436+
m_mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::SystemAudio);
11437+
FALLTHROUGH;
11438+
case WebCore::MediaStreamRequest::Type::DisplayMedia:
11439+
m_mutedCaptureKindsDesiredByWebApp.remove(WebCore::MediaProducerMediaCaptureKind::Display);
11440+
break;
11441+
}
11442+
1137211443
activateMediaStreamCaptureInPage();
1137311444

1137411445
#if ENABLE(GPU_PROCESS)
@@ -11462,6 +11533,7 @@ void WebPageProxy::validateCaptureStateUpdate(WebCore::UserMediaRequestIdentifie
1146211533
}
1146311534

1146411535
if (!isActive) {
11536+
m_mutedCaptureKindsDesiredByWebApp.add(kind);
1146511537
completionHandler({ });
1146611538
return;
1146711539
}
@@ -11518,6 +11590,8 @@ void WebPageProxy::validateCaptureStateUpdate(WebCore::UserMediaRequestIdentifie
1151811590
case WebCore::MediaProducerMediaCaptureKind::EveryKind:
1151911591
ASSERT_NOT_REACHED();
1152011592
}
11593+
11594+
m_mutedCaptureKindsDesiredByWebApp.remove(kind);
1152111595
completionHandler({ });
1152211596
}
1152311597

@@ -11535,7 +11609,7 @@ void WebPageProxy::voiceActivityDetected()
1153511609
{
1153611610
send(Messages::WebPage::VoiceActivityDetected { });
1153711611
}
11538-
#endif
11612+
#endif // ENABLE(MEDIA_STREAM)
1153911613

1154011614
void WebPageProxy::syncIfMockDevicesEnabledChanged()
1154111615
{

‎Source/WebKit/UIProcess/WebPageProxy.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1691,8 +1691,9 @@ class WebPageProxy final : public API::ObjectImpl<API::Object::Type::Page>, publ
16911691
uint64_t renderTreeSize() const { return m_renderTreeSize; }
16921692

16931693
void setMediaVolume(float);
1694-
void setMuted(WebCore::MediaProducerMutedStateFlags);
1695-
void setMuted(WebCore::MediaProducerMutedStateFlags, CompletionHandler<void()>&&);
1694+
1695+
enum class FromApplication : bool { No, Yes };
1696+
void setMuted(WebCore::MediaProducerMutedStateFlags, FromApplication = FromApplication::No, CompletionHandler<void()>&& = [] { });
16961697
bool isAudioMuted() const;
16971698
void setMayStartMediaWhenInWindow(bool);
16981699
bool mayStartMediaWhenInWindow() const { return m_mayStartMediaWhenInWindow; }
@@ -3307,6 +3308,7 @@ class WebPageProxy final : public API::ObjectImpl<API::Object::Type::Page>, publ
33073308
#if ENABLE(MEDIA_STREAM)
33083309
std::unique_ptr<UserMediaPermissionRequestManagerProxy> m_userMediaPermissionRequestManager;
33093310
bool m_shouldListenToVoiceActivity { false };
3311+
OptionSet<WebCore::MediaProducerMediaCaptureKind> m_mutedCaptureKindsDesiredByWebApp;
33103312
#endif
33113313

33123314
#if ENABLE(ENCRYPTED_MEDIA)

‎Tools/TestWebKitAPI/Tests/WebKit/GetUserMedia.mm‎

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1636,7 +1636,7 @@ function callGetUserMedia(audio, video, successMessage, failureMessage)
16361636
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
16371637
}
16381638

1639-
TEST(WebKit2, ToggleCaptureWhenRestarting)
1639+
TEST(WebKit2, ToggleCameraCaptureWhenRestarting)
16401640
{
16411641
[TestProtocol registerWithScheme:@"https"];
16421642

@@ -1695,7 +1695,74 @@ function callGetUserMedia(audio, video, successMessage, failureMessage)
16951695

16961696
// Validate handlers/events order.
16971697
done = false;
1698-
[webView stringByEvaluatingJavaScript:@"validateActionState('deactivating camera, muting camera, deactivating microphone, muting microphone, setCameraActive successful, unmuting camera, activating microphone, unmuting microphone, end')"];
1698+
[webView stringByEvaluatingJavaScript:@"validateActionState('deactivating camera, muting camera, deactivating microphone, muting microphone, setCameraActive successful, unmuting camera, end')"];
1699+
TestWebKitAPI::Util::run(&done);
1700+
}
1701+
1702+
TEST(WebKit2, ToggleMicrophoneCaptureWhenRestarting)
1703+
{
1704+
[TestProtocol registerWithScheme:@"https"];
1705+
1706+
auto configuration = adoptNS([[WKWebViewConfiguration alloc] init]);
1707+
WKPreferencesSetBoolValueForKeyForTesting((__bridge WKPreferencesRef)[configuration preferences], true, WKStringCreateWithUTF8CString("MediaSessionCaptureToggleAPIEnabled"));
1708+
1709+
auto context = adoptWK(TestWebKitAPI::Util::createContextForInjectedBundleTest("InternalsInjectedBundleTest"));
1710+
configuration.get().processPool = (WKProcessPool *)context.get();
1711+
configuration.get().processPool._configuration.shouldCaptureAudioInUIProcess = NO;
1712+
1713+
initializeMediaCaptureConfiguration(configuration.get());
1714+
1715+
auto messageHandler = adoptNS([[GUMMessageHandler alloc] init]);
1716+
[[configuration.get() userContentController] addScriptMessageHandler:messageHandler.get() name:@"gum"];
1717+
1718+
auto webView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 320, 500) configuration:configuration.get()]);
1719+
1720+
auto delegate = adoptNS([[UserMediaCaptureUIDelegate alloc] init]);
1721+
[webView setUIDelegate:delegate.get()];
1722+
[webView _setMediaCaptureReportingDelayForTesting:0];
1723+
1724+
auto observer = adoptNS([[MediaCaptureObserver alloc] init]);
1725+
[webView addObserver:observer.get() forKeyPath:@"microphoneCaptureState" options:NSKeyValueObservingOptionNew context:nil];
1726+
[webView addObserver:observer.get() forKeyPath:@"cameraCaptureState" options:NSKeyValueObservingOptionNew context:nil];
1727+
1728+
done = false;
1729+
[webView loadRequest:[NSURLRequest requestWithURL:[NSURL URLWithString:@"https://bundle-file/media-session-capture.html"]]];
1730+
TestWebKitAPI::Util::run(&done);
1731+
1732+
cameraCaptureStateChange = false;
1733+
microphoneCaptureStateChange = false;
1734+
done = false;
1735+
[webView stringByEvaluatingJavaScript:@"startCapture()"];
1736+
TestWebKitAPI::Util::run(&done);
1737+
1738+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateActive));
1739+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1740+
1741+
// Mute capture.
1742+
cameraCaptureStateChange = false;
1743+
microphoneCaptureStateChange = false;
1744+
1745+
[webView setCameraCaptureState:WKMediaCaptureStateMuted completionHandler:nil];
1746+
[webView setMicrophoneCaptureState:WKMediaCaptureStateMuted completionHandler:nil];
1747+
1748+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateMuted));
1749+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateMuted));
1750+
1751+
// Unmute via MediaSession.
1752+
cameraCaptureStateChange = false;
1753+
microphoneCaptureStateChange = false;
1754+
done = false;
1755+
[webView stringByEvaluatingJavaScript:@"setMicrophoneActive(true)"];
1756+
TestWebKitAPI::Util::run(&done);
1757+
1758+
EXPECT_TRUE(waitUntilMicrophoneState(webView.get(), WKMediaCaptureStateActive));
1759+
1760+
sleep(0.5_s);
1761+
EXPECT_TRUE(waitUntilCameraState(webView.get(), WKMediaCaptureStateMuted));
1762+
1763+
// Validate handlers/events order.
1764+
done = false;
1765+
[webView stringByEvaluatingJavaScript:@"validateActionState('deactivating camera, muting camera, deactivating microphone, muting microphone, setMicrophoneActive successful, unmuting microphone, end')"];
16991766
TestWebKitAPI::Util::run(&done);
17001767
}
17011768
#endif // WK_HAVE_C_SPI

0 commit comments

Comments
 (0)