Skip to content

Commit 281a8db

Browse files
cloudwebrtchiroshihoriegraszka22
committed
Simulcast support for iOS/Android.
Simulcast support for iOS SDK (#4) Support for simulcast in Android SDK (#3) include simulcast headers for mac also (#10) Fix simulcast using hardware encoder on Android (#48) Co-authored-by: Hiroshi Horie <[email protected]> Co-authored-by: Angelika Serwa <[email protected]>
1 parent 7454824 commit 281a8db

File tree

12 files changed

+337
-12
lines changed

12 files changed

+337
-12
lines changed

‎sdk/BUILD.gn‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -723,6 +723,7 @@ if (is_ios || is_mac) {
723723
]
724724

725725
deps = [
726+
":simulcast",
726727
":base_objc",
727728
":native_video",
728729
":videocodec_objc",
@@ -822,6 +823,22 @@ if (is_ios || is_mac) {
822823
]
823824
}
824825

826+
rtc_library("simulcast") {
827+
sources = [
828+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
829+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
830+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
831+
"objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
832+
]
833+
834+
deps = [
835+
":base_objc",
836+
":wrapped_native_codec_objc",
837+
"../media:rtc_media_base",
838+
"../media:rtc_simulcast_encoder_adapter",
839+
]
840+
}
841+
825842
rtc_library("mediaconstraints_objc") {
826843
configs += [ "..:no_global_constructors" ]
827844
sources = [
@@ -1368,6 +1385,9 @@ if (is_ios || is_mac) {
13681385
"objc/api/video_codec/RTCVideoEncoderAV1.h",
13691386
"objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
13701387
"objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
1388+
# Added for Simulcast support
1389+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
1390+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
13711391
]
13721392

13731393
if (!build_with_chromium) {
@@ -1518,6 +1538,9 @@ if (is_ios || is_mac) {
15181538
"objc/components/video_codec/RTCVideoEncoderH264.h",
15191539
"objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
15201540
"objc/helpers/RTCDispatcher.h",
1541+
# Added for Simulcast support
1542+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
1543+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
15211544
]
15221545
if (!build_with_chromium) {
15231546
sources += [

‎sdk/android/BUILD.gn‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,8 @@ if (is_android) {
530530
sources = [
531531
"api/org/webrtc/SoftwareVideoDecoderFactory.java",
532532
"api/org/webrtc/SoftwareVideoEncoderFactory.java",
533+
"api/org/webrtc/SimulcastVideoEncoder.java",
534+
"api/org/webrtc/SimulcastVideoEncoderFactory.java",
533535
]
534536

535537
deps = [
@@ -908,6 +910,21 @@ if (current_os == "linux" || is_android) {
908910
]
909911
}
910912

913+
rtc_library("simulcast_jni") {
914+
visibility = [ "*" ]
915+
allow_poison = [ "software_video_codecs" ]
916+
sources = [
917+
"src/jni/simulcast_video_encoder.cc",
918+
"src/jni/simulcast_video_encoder.h"
919+
]
920+
deps = [
921+
":base_jni",
922+
":video_jni",
923+
":native_api_codecs",
924+
"../../media:rtc_simulcast_encoder_adapter"
925+
]
926+
}
927+
911928
rtc_library("swcodecs_jni") {
912929
visibility = [ "*" ]
913930
allow_poison = [ "software_video_codecs" ]
@@ -921,6 +938,7 @@ if (current_os == "linux" || is_android) {
921938
":libvpx_vp8_jni",
922939
":libvpx_vp9_jni",
923940
":native_api_jni",
941+
":simulcast_jni",
924942
":video_jni",
925943
"../../api/environment",
926944
"../../api/video_codecs:builtin_video_decoder_factory",

‎sdk/android/api/org/webrtc/SimulcastVideoEncoder.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package org.webrtc;
2+
3+
public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder {
4+
5+
static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info);
6+
7+
VideoEncoderFactory primary;
8+
VideoEncoderFactory fallback;
9+
VideoCodecInfo info;
10+
11+
public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) {
12+
this.primary = primary;
13+
this.fallback = fallback;
14+
this.info = info;
15+
}
16+
17+
@Override
18+
public long createNativeVideoEncoder() {
19+
return nativeCreateEncoder(primary, fallback, info);
20+
}
21+
22+
@Override
23+
public boolean isHardwareEncoder() {
24+
return false;
25+
}
26+
27+
}
28+

‎sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
3+
*
4+
* Use of this source code is governed by a BSD-style license
5+
* that can be found in the LICENSE file in the root of the source
6+
* tree. An additional intellectual property rights grant can be found
7+
* in the file PATENTS. All contributing project authors may
8+
* be found in the AUTHORS file in the root of the source tree.
9+
*/
10+
11+
package org.webrtc;
12+
13+
import androidx.annotation.Nullable;
14+
import java.util.ArrayList;
15+
import java.util.HashMap;
16+
import java.util.List;
17+
import java.util.Arrays;
18+
19+
public class SimulcastVideoEncoderFactory implements VideoEncoderFactory {
20+
21+
VideoEncoderFactory primary;
22+
VideoEncoderFactory fallback;
23+
24+
public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) {
25+
this.primary = primary;
26+
this.fallback = fallback;
27+
}
28+
29+
@Nullable
30+
@Override
31+
public VideoEncoder createEncoder(VideoCodecInfo info) {
32+
return new SimulcastVideoEncoder(primary, fallback, info);
33+
}
34+
35+
@Override
36+
public VideoCodecInfo[] getSupportedCodecs() {
37+
List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
38+
codecs.addAll(Arrays.asList(primary.getSupportedCodecs()));
39+
codecs.addAll(Arrays.asList(fallback.getSupportedCodecs()));
40+
return codecs.toArray(new VideoCodecInfo[codecs.size()]);
41+
}
42+
43+
}

‎sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ class HardwareVideoEncoder implements VideoEncoder {
4545
private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
4646
private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
4747

48-
// Size of the input frames should be multiple of 16 for the H/W encoder.
49-
private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16;
48+
// Size of the input frames should be multiple of 2 for the H/W encoder.
49+
private static final int REQUIRED_RESOLUTION_ALIGNMENT = 2;
5050

5151
/**
5252
* Keeps track of the number of output buffers that have been passed down the pipeline and not yet
@@ -210,6 +210,11 @@ public VideoCodecStatus initEncode(Settings settings, Callback callback) {
210210
this.callback = callback;
211211
automaticResizeOn = settings.automaticResizeOn;
212212

213+
if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0
214+
|| settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
215+
Logging.e(TAG, "MediaCodec requires 2x2 alignment.");
216+
return VideoCodecStatus.ERR_SIZE;
217+
}
213218
this.width = settings.width;
214219
this.height = settings.height;
215220
useSurfaceMode = canUseSurface();
@@ -533,6 +538,12 @@ private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseS
533538
if (status != VideoCodecStatus.OK) {
534539
return status;
535540
}
541+
542+
if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0
543+
|| newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
544+
Logging.e(TAG, "MediaCodec requires 2x2 alignment.");
545+
return VideoCodecStatus.ERR_SIZE;
546+
}
536547
width = newWidth;
537548
height = newHeight;
538549
useSurfaceMode = newUseSurfaceMode;

‎sdk/android/src/jni/simulcast_video_encoder.cc‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#include <jni.h>
2+
3+
#include "sdk/android/src/jni/jni_helpers.h"
4+
#include "sdk/android/src/jni/video_encoder_factory_wrapper.h"
5+
#include "sdk/android/src/jni/video_codec_info.h"
6+
#include "sdk/android/native_api/codecs/wrapper.h"
7+
#include "media/engine/simulcast_encoder_adapter.h"
8+
#include "rtc_base/logging.h"
9+
10+
using namespace webrtc;
11+
using namespace webrtc::jni;
12+
13+
#ifdef __cplusplus
14+
extern "C" {
15+
#endif
16+
17+
// (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info)
18+
JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) {
19+
RTC_LOG(LS_INFO) << "Create simulcast video encoder";
20+
JavaParamRef<jobject> info_ref(info);
21+
SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref);
22+
23+
// TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい
24+
// https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795
25+
return NativeToJavaPointer(std::make_unique<SimulcastEncoderAdapter>(
26+
JavaToNativeVideoEncoderFactory(env, primary).release(),
27+
JavaToNativeVideoEncoderFactory(env, fallback).release(),
28+
format).release());
29+
}
30+
31+
32+
#ifdef __cplusplus
33+
}
34+
#endif

‎sdk/android/src/jni/simulcast_video_encoder.h‎

Lines changed: 22 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#import "RTCMacros.h"
2+
#import "RTCVideoEncoder.h"
3+
#import "RTCVideoEncoderFactory.h"
4+
#import "RTCVideoCodecInfo.h"
5+
6+
RTC_OBJC_EXPORT
7+
@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject
8+
9+
+ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
10+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
11+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
12+
13+
@end

‎sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoEncoderSimulcast.h"
5+
#import "RTCNativeVideoEncoder.h"
6+
#import "RTCNativeVideoEncoderBuilder+Native.h"
7+
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
8+
#include "api/transport/field_trial_based_config.h"
9+
10+
#include "native/api/video_encoder_factory.h"
11+
#include "media/engine/simulcast_encoder_adapter.h"
12+
13+
@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcastBuilder)
14+
: RTC_OBJC_TYPE(RTCNativeVideoEncoder) <RTC_OBJC_TYPE (RTCNativeVideoEncoderBuilder)> {
15+
16+
id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> _primary;
17+
id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> _fallback;
18+
RTC_OBJC_TYPE(RTCVideoCodecInfo) *_videoCodecInfo;
19+
}
20+
21+
- (id)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
22+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
23+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
24+
25+
@end
26+
27+
@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcastBuilder)
28+
29+
- (std::unique_ptr<webrtc::VideoEncoder>)build:(const webrtc::Environment&)env {
30+
auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(_primary);
31+
auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(_fallback);
32+
auto nativeFormat = [_videoCodecInfo nativeSdpVideoFormat];
33+
return std::make_unique<webrtc::SimulcastEncoderAdapter>(
34+
env,
35+
nativePrimary.release(),
36+
nativeFallback.release(),
37+
std::move(nativeFormat));
38+
}
39+
40+
- (id)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
41+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
42+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
43+
if (self = [super init]) {
44+
self->_primary = primary;
45+
self->_fallback = fallback;
46+
self->_videoCodecInfo = videoCodecInfo;
47+
}
48+
return self;
49+
}
50+
51+
@end
52+
53+
@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast)
54+
55+
+ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
56+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
57+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
58+
return [[RTC_OBJC_TYPE(RTCVideoEncoderSimulcastBuilder) alloc]
59+
initWithPrimary:primary
60+
fallback:fallback
61+
videoCodecInfo:videoCodecInfo];
62+
}
63+
64+
@end

‎sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoEncoderFactory.h"
5+
6+
NS_ASSUME_NONNULL_BEGIN
7+
8+
RTC_OBJC_EXPORT
9+
@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
10+
11+
- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
12+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback;
13+
14+
@end
15+
16+
NS_ASSUME_NONNULL_END

0 commit comments

Comments
 (0)