Skip to content

Commit

Permalink
Add WebRTC Simulcast Support
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sean-Der
Sean-Der committed Jul 9, 2023
1 parent f05c05f commit 5ab24b7
Show file tree
Hide file tree
Showing 7 changed files with 178 additions and 16 deletions.
1 change: 1 addition & 0 deletions UI/data/locale/en-US.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,7 @@ Basic.Settings.Output.RetryDelay="Retry Delay"
Basic.Settings.Output.MaxRetries="Maximum Retries"
Basic.Settings.Output.Advanced="Enable Custom Encoder Settings (Advanced)"
Basic.Settings.Output.EncoderPreset="Encoder Preset"
Basic.Settings.Output.Simulcast="Enable Simulcast"
Basic.Settings.Output.EncoderPreset.ultrafast="%1 (low CPU usage, lowest quality)"
Basic.Settings.Output.EncoderPreset.veryfast="%1 (default) (medium CPU usage, standard quality)"
Basic.Settings.Output.EncoderPreset.fast="%1 (high CPU usage, high quality)"
Expand Down
18 changes: 14 additions & 4 deletions UI/forms/OBSBasicSettings.ui
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1766,7 +1766,7 @@
<bool>true</bool>
</property>
</widget>
</item>
</item>
<item row="5" column="0">
<widget class="QLabel" name="label_23">
<property name="text">
Expand All @@ -1776,11 +1776,21 @@
<cstring>simpleOutCustom</cstring>
</property>
</widget>
</item>
</item>
<item row="5" column="1">
<widget class="QLineEdit" name="simpleOutCustom"/>
</item>
<item row="6" column="0">
<item row="6" column="1">
<widget class="QCheckBox" name="simpleOutSimulcast">
<property name="text">
<string>Basic.Settings.Output.Simulcast</string>
</property>
<property name="checked">
<bool>false</bool>
</property>
</widget>
</item>
<item row="7" column="0">
<widget class="QLabel" name="simpleOutStrAEncoderLabel">
<property name="text">
<string>Basic.Settings.Output.Encoder.Audio</string>
Expand All @@ -1790,7 +1800,7 @@
</property>
</widget>
</item>
<item row="6" column="1">
<item row="7" column="1">
<widget class="QComboBox" name="simpleOutStrAEncoder"/>
</item>
</layout>
Expand Down
60 changes: 59 additions & 1 deletion UI/window-basic-main-outputs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,7 @@ void BasicOutputHandler::DestroyVirtualCameraScene()
struct SimpleOutput : BasicOutputHandler {
OBSEncoder audioStreaming;
OBSEncoder videoStreaming;
std::vector<OBSEncoder> videoStreamingSimulcast;
OBSEncoder audioRecording;
OBSEncoder audioArchive;
OBSEncoder videoRecording;
Expand Down Expand Up @@ -542,6 +543,26 @@ void SimpleOutput::LoadStreamingPreset_Lossy(const char *encoderId)
if (!videoStreaming)
throw "Failed to create video streaming encoder (simple output)";
obs_encoder_release(videoStreaming);

if (config_get_bool(main->Config(), "SimpleOutput", "UseSimulcast")) {
std::string encoder_name = "simple_video_stream_simulcast_0";
for (auto i = 0; i < 2; i++) {
encoder_name[encoder_name.size() - 1] =
to_string(i).at(0);
auto simulcast_encoder = obs_video_encoder_create(
encoderId, encoder_name.c_str(), nullptr,
nullptr);

if (simulcast_encoder) {
videoStreamingSimulcast.push_back(
simulcast_encoder);
obs_encoder_release(simulcast_encoder);
} else {
blog(LOG_WARNING,
"Failed to create video streaming simulcast encoders (simple output)");
}
}
}
}

/* mistakes have been made to lead us to this. */
Expand Down Expand Up @@ -838,9 +859,39 @@ void SimpleOutput::Update()
default:
obs_encoder_set_preferred_video_format(videoStreaming,
VIDEO_FORMAT_NV12);
for (auto enc : videoStreamingSimulcast)
obs_encoder_set_preferred_video_format(
enc, VIDEO_FORMAT_NV12);
}

obs_encoder_update(videoStreaming, videoSettings);
for (auto enc : videoStreamingSimulcast)
obs_encoder_update(enc, videoSettings);

if (videoStreamingSimulcast.size() > 0) {
uint32_t width = video_output_get_width(video) / 1.5;
width -= width % 2;

uint32_t height = video_output_get_height(video) / 1.5;
height -= height % 2;

obs_data_set_int(videoSettings, "bitrate", videoBitrate / 2);
obs_encoder_set_scaled_size(videoStreamingSimulcast[0], width,
height);
}

if (videoStreamingSimulcast.size() > 1) {
uint32_t width = video_output_get_width(video) / 2;
width -= width % 2;

uint32_t height = video_output_get_height(video) / 2;
height -= height % 2;

obs_data_set_int(videoSettings, "bitrate", videoBitrate / 4);
obs_encoder_set_scaled_size(videoStreamingSimulcast[1], width,
height);
}

obs_encoder_update(audioStreaming, audioSettings);
obs_encoder_update(audioArchive, audioSettings);
}
Expand Down Expand Up @@ -1039,6 +1090,9 @@ inline void SimpleOutput::SetupOutputs()
{
SimpleOutput::Update();
obs_encoder_set_video(videoStreaming, obs_get_video());
for (auto enc : videoStreamingSimulcast)
obs_encoder_set_video(enc, obs_get_video());

obs_encoder_set_audio(audioStreaming, obs_get_audio());
obs_encoder_set_audio(audioArchive, obs_get_audio());
int tracks =
Expand Down Expand Up @@ -1135,7 +1189,11 @@ bool SimpleOutput::SetupStreaming(obs_service_t *service)
outputType = type;
}

obs_output_set_video_encoder(streamOutput, videoStreaming);
obs_output_set_video_encoder2(streamOutput, videoStreaming, 0);
for (size_t i = 0; i < videoStreamingSimulcast.size(); i++)
obs_output_set_video_encoder2(
streamOutput, videoStreamingSimulcast[i], i + 1);

obs_output_set_audio_encoder(streamOutput, audioStreaming, 0);
obs_output_set_service(streamOutput, service);
return true;
Expand Down
6 changes: 6 additions & 0 deletions UI/window-basic-settings-stream.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,6 +505,12 @@ void OBSBasicSettings::on_service_currentIndexChanged(int idx)
if (idx == 0)
lastCustomServer = ui->customServer->text();
}

if (IsWHIP()) {
ui->simpleOutSimulcast->show();
} else {
ui->simpleOutSimulcast->hide();
}
}

void OBSBasicSettings::on_customServer_textChanged(const QString &)
Expand Down
13 changes: 13 additions & 0 deletions UI/window-basic-settings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,7 @@ OBSBasicSettings::OBSBasicSettings(QWidget *parent)
HookWidget(ui->simpleOutAdvanced, CHECK_CHANGED, OUTPUTS_CHANGED);
HookWidget(ui->simpleOutPreset, COMBO_CHANGED, OUTPUTS_CHANGED);
HookWidget(ui->simpleOutCustom, EDIT_CHANGED, OUTPUTS_CHANGED);
HookWidget(ui->simpleOutSimulcast, CHECK_CHANGED, OUTPUTS_CHANGED);
HookWidget(ui->simpleOutRecQuality, COMBO_CHANGED, OUTPUTS_CHANGED);
HookWidget(ui->simpleOutRecEncoder, COMBO_CHANGED, OUTPUTS_CHANGED);
HookWidget(ui->simpleOutRecAEncoder, COMBO_CHANGED, OUTPUTS_CHANGED);
Expand Down Expand Up @@ -1927,6 +1928,8 @@ void OBSBasicSettings::LoadSimpleOutputSettings()
config_get_uint(main->Config(), "SimpleOutput", "ABitrate");
bool advanced =
config_get_bool(main->Config(), "SimpleOutput", "UseAdvanced");
bool simulcast =
config_get_bool(main->Config(), "SimpleOutput", "UseSimulcast");
const char *preset =
config_get_string(main->Config(), "SimpleOutput", "Preset");
const char *qsvPreset =
Expand Down Expand Up @@ -1996,6 +1999,15 @@ void OBSBasicSettings::LoadSimpleOutputSettings()
ui->simpleOutAdvanced->setChecked(advanced);
ui->simpleOutCustom->setText(custom);

OBSService service = GetStream1Service();
if (strcmp(obs_service_get_id(service), "whip_custom") == 0) {
ui->simpleOutSimulcast->show();
} else {
ui->simpleOutSimulcast->hide();
}

ui->simpleOutSimulcast->setChecked(simulcast);

idx = ui->simpleOutRecQuality->findData(QString(recQual));
if (idx == -1)
idx = 0;
Expand Down Expand Up @@ -3845,6 +3857,7 @@ void OBSBasicSettings::SaveOutputSettings()
SaveCheckBox(ui->simpleOutAdvanced, "SimpleOutput", "UseAdvanced");
SaveComboData(ui->simpleOutPreset, "SimpleOutput", presetType);
SaveEdit(ui->simpleOutCustom, "SimpleOutput", "x264Settings");
SaveCheckBox(ui->simpleOutSimulcast, "SimpleOutput", "UseSimulcast");
SaveComboData(ui->simpleOutRecQuality, "SimpleOutput", "RecQuality");
SaveComboData(ui->simpleOutRecEncoder, "SimpleOutput", "RecEncoder");
SaveComboData(ui->simpleOutRecAEncoder, "SimpleOutput",
Expand Down
85 changes: 75 additions & 10 deletions plugins/obs-webrtc/whip-output.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,21 @@ const uint32_t audio_ssrc = 5002;
const char *audio_mid = "0";
const uint8_t audio_payload_type = 111;

const uint32_t video_ssrc = 5000;
const uint32_t video_l_ssrc = 5000;
const uint32_t video_m_ssrc = 5004;
const uint32_t video_h_ssrc = 5006;

const char *video_mid = "1";
const uint8_t video_payload_type = 96;

const std::string rtpHeaderExtUriMid = "urn:ietf:params:rtp-hdrext:sdes:mid";
const std::string rtpHeaderExtUriRid =
"urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id";

const std::string highRid = "h";
const std::string medRid = "m";
const std::string lowRid = "l";

WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output)
: output(output),
endpoint_url(),
Expand All @@ -27,8 +38,7 @@ WHIPOutput::WHIPOutput(obs_data_t *, obs_output_t *output)
total_bytes_sent(0),
connect_time_ms(0),
start_time_ns(0),
last_audio_timestamp(0),
last_video_timestamp(0)
last_audio_timestamp(0)
{
}

Expand All @@ -45,6 +55,27 @@ bool WHIPOutput::Start()
{
std::lock_guard<std::mutex> l(start_stop_mutex);

for (size_t idx = 0; idx < 5; idx++) {
auto encoder = obs_output_get_video_encoder2(output, idx);
if (encoder == nullptr) {
break;
}

auto v = std::make_shared<videoLayerState>();
if (idx == 0) {
v->ssrc = video_l_ssrc;
v->rid = highRid;
} else if (idx == 1) {
v->ssrc = video_m_ssrc;
v->rid = medRid;
} else if (idx == 2) {
v->ssrc = video_h_ssrc;
v->rid = lowRid;
}

videoLayerStates[obs_encoder_get_width(encoder)] = v;
}

if (!obs_output_can_begin_data_capture(output, 0))
return false;
if (!obs_output_initialize_encoders(output, 0))
Expand Down Expand Up @@ -80,11 +111,28 @@ void WHIPOutput::Data(struct encoder_packet *packet)
audio_sr_reporter);
last_audio_timestamp = packet->dts_usec;
} else if (packet->type == OBS_ENCODER_VIDEO) {
auto height = obs_encoder_get_width(packet->encoder);
int64_t duration = packet->dts_usec - last_video_timestamp;
auto rtp_config = video_sr_reporter->rtpConfig;
auto videoLayerState =
videoLayerStates[obs_encoder_get_width(packet->encoder)];
if (videoLayerState == nullptr) {
Stop(false);
obs_output_signal_stop(output, OBS_OUTPUT_ENCODE_ERROR);
return;
}

rtp_config->sequenceNumber = videoLayerState->sequenceNumber;
rtp_config->ssrc = videoLayerState->ssrc;
rtp_config->rid = videoLayerState->rid;
rtp_config->timestamp = videoLayerState->rtpTimestamp;
int64_t duration =
packet->dts_usec - videoLayerState->lastVideoTimestamp;

Send(packet->data, packet->size, duration, video_track,
video_sr_reporter);
last_video_timestamp = packet->dts_usec;

videoLayerState->sequenceNumber = rtp_config->sequenceNumber;
videoLayerState->lastVideoTimestamp = packet->dts_usec;
videoLayerState->rtpTimestamp = rtp_config->timestamp;
}
}

Expand Down Expand Up @@ -122,13 +170,29 @@ void WHIPOutput::ConfigureVideoTrack(std::string media_stream_id,
rtc::Description::Video video_description(
video_mid, rtc::Description::Direction::SendOnly);
video_description.addH264Codec(video_payload_type);
video_description.addSSRC(video_ssrc, cname, media_stream_id,
video_description.addSSRC(video_l_ssrc, cname, media_stream_id,
media_stream_track_id);

video_description.addExtMap(
rtc::Description::Entry::ExtMap(1, rtpHeaderExtUriMid));
video_description.addExtMap(
rtc::Description::Entry::ExtMap(2, rtpHeaderExtUriRid));

if (videoLayerStates.size() >= 2) {
for (const auto &v : videoLayerStates) {
video_description.addRid(v.second->rid);
}
}

video_track = peer_connection->addTrack(video_description);

auto rtp_config = std::make_shared<rtc::RtpPacketizationConfig>(
video_ssrc, cname, video_payload_type,
video_l_ssrc, cname, video_payload_type,
rtc::H264RtpPacketizer::defaultClockRate);
rtp_config->midId = 1;
rtp_config->ridId = 2;
rtp_config->mid = video_mid;

auto packetizer = std::make_shared<rtc::H264RtpPacketizer>(
rtc::H264RtpPacketizer::Separator::StartSequence, rtp_config);
video_sr_reporter = std::make_shared<rtc::RtcpSrReporter>(rtp_config);
Expand Down Expand Up @@ -402,7 +466,7 @@ void WHIPOutput::StopThread(bool signal)
connect_time_ms = 0;
start_time_ns = 0;
last_audio_timestamp = 0;
last_video_timestamp = 0;
videoLayerStates.clear();
}

void WHIPOutput::Send(void *data, uintptr_t size, uint64_t duration,
Expand Down Expand Up @@ -449,7 +513,8 @@ void register_whip_output()
struct obs_output_info info = {};

info.id = "whip_output";
info.flags = OBS_OUTPUT_AV | OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE;
info.flags = OBS_OUTPUT_AV | OBS_OUTPUT_ENCODED | OBS_OUTPUT_SERVICE |
OBS_OUTPUT_MULTI_TRACK_AV;
info.get_name = [](void *) -> const char * {
return obs_module_text("Output.Name");
};
Expand Down
11 changes: 10 additions & 1 deletion plugins/obs-webrtc/whip-output.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@
blog(level, "[obs-webrtc] [whip_output: '%s'] " format, \
obs_output_get_name(output), ##__VA_ARGS__)

struct videoLayerState {
uint16_t sequenceNumber;
uint32_t rtpTimestamp;
int64_t lastVideoTimestamp;
uint32_t ssrc;
std::string rid;
};

class WHIPOutput {
public:
WHIPOutput(obs_data_t *settings, obs_output_t *output);
Expand Down Expand Up @@ -62,11 +70,12 @@ class WHIPOutput {
std::shared_ptr<rtc::RtcpSrReporter> audio_sr_reporter;
std::shared_ptr<rtc::RtcpSrReporter> video_sr_reporter;

std::map<uint32_t, std::shared_ptr<videoLayerState>> videoLayerStates;

std::atomic<size_t> total_bytes_sent;
std::atomic<int> connect_time_ms;
int64_t start_time_ns;
int64_t last_audio_timestamp;
int64_t last_video_timestamp;
};

void register_whip_output();
Expand Down

0 comments on commit 5ab24b7

Please sign in to comment.