thek0tyara/nheko
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support desktop screen sharing on X11

Browse source
This commit is contained in:
trilene 2021-02-18 15:55:29 -05:00
parent 0d971f543b
commit 8df10eeeca
15 changed files with 376 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

134
src/WebRTCSession.cpp
View file

@ -10,6 +10,7 @@
#include <thread>
#include <utility>
#include "CallDevices.h"
#include "ChatPage.h"
#include "Logging.h"
#include "UserSettingsPage.h"
@ -29,14 +30,20 @@ extern "C"
// https://github.com/vector-im/riot-web/issues/10173
#define STUN_SERVER "stun://turn.matrix.org:3478"
Q_DECLARE_METATYPE(webrtc::CallType)
Q_DECLARE_METATYPE(webrtc::State)
using webrtc::CallType;
using webrtc::State;
WebRTCSession::WebRTCSession()
: QObject()
, devices_(CallDevices::instance())
{
qRegisterMetaType<webrtc::CallType>();
qmlRegisterUncreatableMetaObject(
webrtc::staticMetaObject, "im.nheko", 1, 0, "CallType", "Can't instantiate enum");
qRegisterMetaType<webrtc::State>();
qmlRegisterUncreatableMetaObject(
webrtc::staticMetaObject, "im.nheko", 1, 0, "WebRTCState", "Can't instantiate enum");
@ -455,7 +462,8 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
nhlog::ui()->info("WebRTC: incoming video resolution: {}x{}",
videoCallSize.first,
videoCallSize.second);
addCameraView(pipe, videoCallSize);
if (session->callType() == CallType::VIDEO)
addCameraView(pipe, videoCallSize);
} else {
g_free(mediaType);
nhlog::ui()->error("WebRTC: unknown pad type: {}", GST_PAD_NAME(newpad));
@ -467,7 +475,7 @@ linkNewPad(GstElement *decodebin, GstPad *newpad, GstElement *pipe)
if (GST_PAD_LINK_FAILED(gst_pad_link(newpad, queuepad)))
nhlog::ui()->error("WebRTC: unable to link new pad");
else {
if (!session->isVideo() ||
if (session->callType() == CallType::VOICE ||
(haveAudioStream_ &&
(haveVideoStream_ || session->isRemoteVideoRecvOnly()))) {
emit session->stateChanged(State::CONNECTED);
@ -523,14 +531,17 @@ getMediaAttributes(const GstSDPMessage *sdp,
const char *mediaType,
const char *encoding,
int &payloadType,
bool &recvOnly)
bool &recvOnly,
bool &sendOnly)
{
payloadType = -1;
recvOnly = false;
sendOnly = false;
for (guint mlineIndex = 0; mlineIndex < gst_sdp_message_medias_len(sdp); ++mlineIndex) {
const GstSDPMedia *media = gst_sdp_message_get_media(sdp, mlineIndex);
if (!std::strcmp(gst_sdp_media_get_media(media), mediaType)) {
recvOnly = gst_sdp_media_get_attribute_val(media, "recvonly") != nullptr;
sendOnly = gst_sdp_media_get_attribute_val(media, "sendonly") != nullptr;
const gchar *rtpval = nullptr;
for (guint n = 0; n == 0 || rtpval; ++n) {
rtpval = gst_sdp_media_get_attribute_val_n(media, "rtpmap", n);
@ -603,11 +614,12 @@ WebRTCSession::havePlugins(bool isVideo, std::string *errorMessage)
}
bool
WebRTCSession::createOffer(bool isVideo)
WebRTCSession::createOffer(CallType callType)
{
isOffering_ = true;
isVideo_ = isVideo;
callType_ = callType;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
@ -630,8 +642,10 @@ WebRTCSession::acceptOffer(const std::string &sdp)
if (state_ != State::DISCONNECTED)
return false;
callType_ = webrtc::CallType::VOICE;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
haveAudioStream_ = false;
haveVideoStream_ = false;
@ -645,7 +659,8 @@ WebRTCSession::acceptOffer(const std::string &sdp)
int opusPayloadType;
bool recvOnly;
if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly)) {
bool sendOnly;
if (getMediaAttributes(offer->sdp, "audio", "opus", opusPayloadType, recvOnly, sendOnly)) {
if (opusPayloadType == -1) {
nhlog::ui()->error("WebRTC: remote audio offer - no opus encoding");
gst_webrtc_session_description_free(offer);
@ -658,13 +673,18 @@ WebRTCSession::acceptOffer(const std::string &sdp)
}
int vp8PayloadType;
isVideo_ =
getMediaAttributes(offer->sdp, "video", "vp8", vp8PayloadType, isRemoteVideoRecvOnly_);
if (isVideo_ && vp8PayloadType == -1) {
bool isVideo = getMediaAttributes(offer->sdp,
"video",
"vp8",
vp8PayloadType,
isRemoteVideoRecvOnly_,
isRemoteVideoSendOnly_);
if (isVideo && vp8PayloadType == -1) {
nhlog::ui()->error("WebRTC: remote video offer - no vp8 encoding");
gst_webrtc_session_description_free(offer);
return false;
}
callType_ = isVideo ? CallType::VIDEO : CallType::VOICE;
if (!startPipeline(opusPayloadType, vp8PayloadType)) {
gst_webrtc_session_description_free(offer);
@ -695,10 +715,14 @@ WebRTCSession::acceptAnswer(const std::string &sdp)
return false;
}
if (isVideo_) {
if (callType_ != CallType::VOICE) {
int unused;
if (!getMediaAttributes(
answer->sdp, "video", "vp8", unused, isRemoteVideoRecvOnly_))
if (!getMediaAttributes(answer->sdp,
"video",
"vp8",
unused,
isRemoteVideoRecvOnly_,
isRemoteVideoSendOnly_))
isRemoteVideoRecvOnly_ = true;
}
@ -855,39 +879,59 @@ WebRTCSession::createPipeline(int opusPayloadType, int vp8PayloadType)
return false;
}
return isVideo_ ? addVideoPipeline(vp8PayloadType) : true;
return callType_ == CallType::VOICE || isRemoteVideoSendOnly_
? true
: addVideoPipeline(vp8PayloadType);
}
bool
WebRTCSession::addVideoPipeline(int vp8PayloadType)
{
// allow incoming video calls despite localUser having no webcam
if (!devices_.haveCamera())
if (callType_ == CallType::VIDEO && !devices_.haveCamera())
return !isOffering_;
std::pair<int, int> resolution;
std::pair<int, int> frameRate;
GstDevice *device = devices_.videoDevice(resolution, frameRate);
if (!device)
return false;
GstElement *source = nullptr;
GstCaps *caps = nullptr;
if (callType_ == CallType::VIDEO) {
std::pair<int, int> resolution;
std::pair<int, int> frameRate;
GstDevice *device = devices_.videoDevice(resolution, frameRate);
if (!device)
return false;
source = gst_device_create_element(device, nullptr);
caps = gst_caps_new_simple("video/x-raw",
"width",
G_TYPE_INT,
resolution.first,
"height",
G_TYPE_INT,
resolution.second,
"framerate",
GST_TYPE_FRACTION,
frameRate.first,
frameRate.second,
nullptr);
} else {
source = gst_element_factory_make("ximagesrc", nullptr);
if (!source) {
nhlog::ui()->error("WebRTC: failed to create ximagesrc");
return false;
}
g_object_set(source, "use-damage", 0, nullptr);
g_object_set(source, "xid", 0, nullptr);
int frameRate = ChatPage::instance()->userSettings()->screenShareFrameRate();
caps = gst_caps_new_simple(
"video/x-raw", "framerate", GST_TYPE_FRACTION, frameRate, 1, nullptr);
nhlog::ui()->debug("WebRTC: screen share frame rate: {} fps", frameRate);
}
GstElement *source = gst_device_create_element(device, nullptr);
GstElement *videoconvert = gst_element_factory_make("videoconvert", nullptr);
GstElement *capsfilter = gst_element_factory_make("capsfilter", "camerafilter");
GstCaps *caps = gst_caps_new_simple("video/x-raw",
"width",
G_TYPE_INT,
resolution.first,
"height",
G_TYPE_INT,
resolution.second,
"framerate",
GST_TYPE_FRACTION,
frameRate.first,
frameRate.second,
nullptr);
g_object_set(capsfilter, "caps", caps, nullptr);
gst_caps_unref(caps);
GstElement *tee = gst_element_factory_make("tee", "videosrctee");
GstElement *queue = gst_element_factory_make("queue", nullptr);
GstElement *vp8enc = gst_element_factory_make("vp8enc", nullptr);
@ -938,14 +982,25 @@ WebRTCSession::addVideoPipeline(int vp8PayloadType)
gst_object_unref(webrtcbin);
return false;
}
if (callType_ == CallType::SCREEN &&
!ChatPage::instance()->userSettings()->screenShareRemoteVideo()) {
GArray *transceivers;
g_signal_emit_by_name(webrtcbin, "get-transceivers", &transceivers);
GstWebRTCRTPTransceiver *transceiver =
g_array_index(transceivers, GstWebRTCRTPTransceiver *, 1);
transceiver->direction = GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
g_array_unref(transceivers);
}
gst_object_unref(webrtcbin);
return true;
}
bool
WebRTCSession::haveLocalVideo() const
WebRTCSession::haveLocalCamera() const
{
if (isVideo_ && state_ >= State::INITIATED) {
if (callType_ == CallType::VIDEO && state_ >= State::INITIATED) {
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipe_), "videosrctee");
if (tee) {
gst_object_unref(tee);
@ -1008,9 +1063,10 @@ WebRTCSession::end()
}
webrtc_ = nullptr;
isVideo_ = false;
callType_ = CallType::VOICE;
isOffering_ = false;
isRemoteVideoRecvOnly_ = false;
isRemoteVideoSendOnly_ = false;
videoItem_ = nullptr;
insetSinkPad_ = nullptr;
if (state_ != State::DISCONNECTED)
@ -1026,16 +1082,12 @@ WebRTCSession::havePlugins(bool, std::string *)
}
bool
WebRTCSession::haveLocalVideo() const
WebRTCSession::haveLocalCamera() const
{
return false;
}
bool
WebRTCSession::createOffer(bool)
{
return false;
}
bool WebRTCSession::createOffer(webrtc::CallType) { return false; }
bool
WebRTCSession::acceptOffer(const std::string &)