Skip to content

Commit 8c9aa75

Browse files
cloudwebrtccloudwebrtc
authored
feat: add external audio processor for android. (#103)
* feat: add external audio processor for android. * update. * update. * update. * add null ptr check. * rename files. * fix typo. * some comment. * update.
1 parent d5afc4b commit 8c9aa75

7 files changed

+523
-0
lines changed

‎sdk/android/BUILD.gn‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ if (is_android) {
261261
"api/org/webrtc/CryptoOptions.java",
262262
"api/org/webrtc/DataChannel.java",
263263
"api/org/webrtc/DtmfSender.java",
264+
"api/org/webrtc/ExternalAudioProcessingFactory.java",
264265
"api/org/webrtc/FecControllerFactoryFactoryInterface.java",
265266
"api/org/webrtc/FrameCryptor.java",
266267
"api/org/webrtc/FrameCryptorAlgorithm.java",
@@ -726,6 +727,11 @@ if (current_os == "linux" || is_android) {
726727
"src/jni/pc/data_channel.cc",
727728
"src/jni/pc/data_channel.h",
728729
"src/jni/pc/dtmf_sender.cc",
730+
"src/jni/pc/external_audio_processing_factory.cc",
731+
"src/jni/pc/external_audio_processing_factory.h",
732+
"src/jni/pc/external_audio_processing_interface.h",
733+
"src/jni/pc/external_audio_processor.cc",
734+
"src/jni/pc/external_audio_processor.h",
729735
"src/jni/pc/frame_cryptor.cc",
730736
"src/jni/pc/frame_cryptor.h",
731737
"src/jni/pc/frame_cryptor_key_provider.cc",
@@ -1414,6 +1420,7 @@ if (current_os == "linux" || is_android) {
14141420
"api/org/webrtc/CryptoOptions.java",
14151421
"api/org/webrtc/DataChannel.java",
14161422
"api/org/webrtc/DtmfSender.java",
1423+
"api/org/webrtc/ExternalAudioProcessingFactory.java",
14171424
"api/org/webrtc/FrameCryptor.java",
14181425
"api/org/webrtc/FrameCryptorFactory.java",
14191426
"api/org/webrtc/FrameCryptorKeyProvider.java",

‎sdk/android/api/org/webrtc/ExternalAudioProcessingFactory.java‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/*
2+
* Copyright 2022 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.webrtc;
18+
19+
import java.nio.ByteBuffer;
20+
21+
import androidx.annotation.Nullable;
22+
import org.webrtc.AudioProcessingFactory;
23+
24+
25+
public class ExternalAudioProcessingFactory implements AudioProcessingFactory {
26+
27+
/**
28+
* Interface for external audio processing.
29+
*/
30+
public static interface AudioProcessing {
31+
/**
32+
* Called when the processor should be initialized with a new sample rate and
33+
* number of channels.
34+
*/
35+
@CalledByNative("AudioProcessing")
36+
void initialize(int sampleRateHz, int numChannels);
37+
/** Called when the processor should be reset with a new sample rate. */
38+
@CalledByNative("AudioProcessing")
39+
void reset(int newRate);
40+
/**
41+
* Processes the given capture or render signal. NOTE: `buffer.data` will be
42+
* freed once this function returns so callers who want to use the data
43+
* asynchronously must make sure to copy it first.
44+
*/
45+
@CalledByNative("AudioProcessing")
46+
void process(int numBands, int numFrames, ByteBuffer buffer);
47+
}
48+
49+
private long apmPtr;
50+
private long capturePostProcessingPtr;
51+
private long renderPreProcessingPtr;
52+
53+
public ExternalAudioProcessingFactory() {
54+
apmPtr = nativeGetDefaultApm();
55+
capturePostProcessingPtr = 0;
56+
renderPreProcessingPtr = 0;
57+
}
58+
59+
@Override
60+
public long createNative() {
61+
if(apmPtr == 0) {
62+
apmPtr = nativeGetDefaultApm();
63+
}
64+
return apmPtr;
65+
}
66+
67+
/**
68+
* Sets the capture post processing module.
69+
* This module is applied to the audio signal after capture and before sending
70+
* to the audio encoder.
71+
*/
72+
public void setCapturePostProcessing(@Nullable AudioProcessing processing) {
73+
checkExternalAudioProcessorExists();
74+
long newPtr = nativeSetCapturePostProcessing(processing);
75+
if (capturePostProcessingPtr != 0) {
76+
JniCommon.nativeReleaseRef(capturePostProcessingPtr);
77+
capturePostProcessingPtr = 0;
78+
}
79+
capturePostProcessingPtr = newPtr;
80+
}
81+
82+
/**
83+
* Sets the render pre processing module.
84+
* This module is applied to the audio signal after receiving from the audio
85+
* decoder and before rendering.
86+
*/
87+
public void setRenderPreProcessing(@Nullable AudioProcessing processing) {
88+
checkExternalAudioProcessorExists();
89+
long newPtr = nativeSetRenderPreProcessing(processing);
90+
if (renderPreProcessingPtr != 0) {
91+
JniCommon.nativeReleaseRef(renderPreProcessingPtr);
92+
renderPreProcessingPtr = 0;
93+
}
94+
renderPreProcessingPtr = newPtr;
95+
}
96+
97+
/**
98+
* Sets the bypass flag for the capture post processing module.
99+
* If true, the registered audio processing will be bypassed.
100+
*/
101+
public void setBypassFlagForCapturePost( boolean bypass) {
102+
checkExternalAudioProcessorExists();
103+
nativeSetBypassFlagForCapturePost(bypass);
104+
}
105+
106+
/**
107+
* Sets the bypass flag for the render pre processing module.
108+
* If true, the registered audio processing will be bypassed.
109+
*/
110+
public void setBypassFlagForRenderPre( boolean bypass) {
111+
checkExternalAudioProcessorExists();
112+
nativeSetBypassFlagForRenderPre(bypass);
113+
}
114+
115+
/**
116+
* Destroys the ExternalAudioProcessor.
117+
*/
118+
public void destroy() {
119+
checkExternalAudioProcessorExists();
120+
if (renderPreProcessingPtr != 0) {
121+
JniCommon.nativeReleaseRef(renderPreProcessingPtr);
122+
renderPreProcessingPtr = 0;
123+
}
124+
if (capturePostProcessingPtr != 0) {
125+
JniCommon.nativeReleaseRef(capturePostProcessingPtr);
126+
capturePostProcessingPtr = 0;
127+
}
128+
nativeDestroy();
129+
apmPtr = 0;
130+
}
131+
132+
private void checkExternalAudioProcessorExists() {
133+
if (apmPtr == 0) {
134+
throw new IllegalStateException("ExternalAudioProcessor has been disposed.");
135+
}
136+
}
137+
138+
private static native long nativeGetDefaultApm();
139+
private static native long nativeSetCapturePostProcessing(AudioProcessing processing);
140+
private static native long nativeSetRenderPreProcessing(AudioProcessing processing);
141+
private static native void nativeSetBypassFlagForCapturePost(boolean bypass);
142+
private static native void nativeSetBypassFlagForRenderPre(boolean bypass);
143+
private static native void nativeDestroy();
144+
}

‎sdk/android/src/jni/pc/external_audio_processing_factory.cc‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
/*
2+
* Copyright 2022 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "sdk/android/src/jni/pc/external_audio_processing_factory.h"
18+
19+
#include <jni.h>
20+
#include <syslog.h>
21+
22+
#include "api/make_ref_counted.h"
23+
#include "rtc_base/ref_counted_object.h"
24+
#include "sdk/android/generated_peerconnection_jni/ExternalAudioProcessingFactory_jni.h"
25+
#include "sdk/android/native_api/jni/java_types.h"
26+
#include "sdk/android/native_api/jni/scoped_java_ref.h"
27+
#include "sdk/android/src/jni/jni_helpers.h"
28+
#include "sdk/android/src/jni/pc/external_audio_processor.h"
29+
30+
namespace webrtc {
31+
namespace jni {
32+
33+
ExternalAudioProcessingJni::ExternalAudioProcessingJni(
34+
JNIEnv* jni,
35+
const JavaRef<jobject>& j_processing)
36+
: j_processing_global_(jni, j_processing) {}
37+
ExternalAudioProcessingJni::~ExternalAudioProcessingJni() {}
38+
void ExternalAudioProcessingJni::Initialize(int sample_rate_hz,
39+
int num_channels) {
40+
JNIEnv* env = AttachCurrentThreadIfNeeded();
41+
Java_AudioProcessing_initialize(env, j_processing_global_, sample_rate_hz,
42+
num_channels);
43+
}
44+
45+
void ExternalAudioProcessingJni::Reset(int new_rate) {
46+
JNIEnv* env = AttachCurrentThreadIfNeeded();
47+
Java_AudioProcessing_reset(env, j_processing_global_, new_rate);
48+
}
49+
50+
void ExternalAudioProcessingJni::Process(int num_bands, int num_frames, int buffer_size, float* buffer) {
51+
JNIEnv* env = AttachCurrentThreadIfNeeded();
52+
ScopedJavaLocalRef<jobject> audio_buffer =
53+
NewDirectByteBuffer(env, (void*)buffer, buffer_size * sizeof(float));
54+
Java_AudioProcessing_process(env, j_processing_global_, num_bands, num_frames, audio_buffer);
55+
}
56+
57+
ExternalAudioProcessingFactory::ExternalAudioProcessingFactory() {
58+
capture_post_processor_ = new ExternalAudioProcessor();
59+
std::unique_ptr<webrtc::CustomProcessing> capture_post_processor(
60+
capture_post_processor_);
61+
62+
render_pre_processor_ = new ExternalAudioProcessor();
63+
std::unique_ptr<webrtc::CustomProcessing> render_pre_processor(
64+
render_pre_processor_);
65+
66+
apm_ = webrtc::AudioProcessingBuilder()
67+
.SetCapturePostProcessing(std::move(capture_post_processor))
68+
.SetRenderPreProcessing(std::move(render_pre_processor))
69+
.Create();
70+
71+
webrtc::AudioProcessing::Config config;
72+
apm_->ApplyConfig(config);
73+
}
74+
75+
static ExternalAudioProcessingFactory* default_processor_ptr;
76+
77+
static jlong JNI_ExternalAudioProcessingFactory_GetDefaultApm(JNIEnv* env) {
78+
if (!default_processor_ptr) {
79+
auto default_processor = rtc::make_ref_counted<ExternalAudioProcessingFactory>();
80+
default_processor_ptr = default_processor.release();
81+
}
82+
return webrtc::jni::jlongFromPointer(default_processor_ptr->apm().get());
83+
}
84+
85+
static jlong JNI_ExternalAudioProcessingFactory_SetCapturePostProcessing(
86+
JNIEnv* env,
87+
const JavaParamRef<jobject>& j_processing) {
88+
if (!default_processor_ptr) {
89+
return 0;
90+
}
91+
auto processing =
92+
rtc::make_ref_counted<ExternalAudioProcessingJni>(env, j_processing);
93+
processing->AddRef();
94+
default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing(
95+
processing.get());
96+
return jlongFromPointer(processing.get());
97+
}
98+
99+
static jlong JNI_ExternalAudioProcessingFactory_SetRenderPreProcessing(
100+
JNIEnv* env,
101+
const JavaParamRef<jobject>& j_processing) {
102+
if (!default_processor_ptr) {
103+
return 0;
104+
}
105+
auto processing =
106+
rtc::make_ref_counted<ExternalAudioProcessingJni>(env, j_processing);
107+
processing->AddRef();
108+
default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing(
109+
processing.get());
110+
return jlongFromPointer(processing.get());
111+
}
112+
113+
static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForCapturePost(
114+
JNIEnv* env,
115+
jboolean bypass) {
116+
if (!default_processor_ptr) {
117+
return;
118+
}
119+
default_processor_ptr->capture_post_processor()->SetBypassFlag(bypass);
120+
}
121+
122+
static void JNI_ExternalAudioProcessingFactory_SetBypassFlagForRenderPre(
123+
JNIEnv* env,
124+
jboolean bypass) {
125+
if (!default_processor_ptr) {
126+
return;
127+
}
128+
default_processor_ptr->render_pre_processor()->SetBypassFlag(bypass);
129+
}
130+
131+
static void JNI_ExternalAudioProcessingFactory_Destroy(JNIEnv* env) {
132+
if (!default_processor_ptr) {
133+
return;
134+
}
135+
default_processor_ptr->render_pre_processor()->SetExternalAudioProcessing(
136+
nullptr);
137+
default_processor_ptr->capture_post_processor()->SetExternalAudioProcessing(
138+
nullptr);
139+
delete default_processor_ptr;
140+
}
141+
142+
} // namespace jni
143+
} // namespace webrtc

‎sdk/android/src/jni/pc/external_audio_processing_factory.h‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright 2022 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include <jni.h>
18+
19+
#define WEBRTC_APM_DEBUG_DUMP 0
20+
21+
#include "rtc_base/ref_counted_object.h"
22+
#include "sdk/android/native_api/jni/scoped_java_ref.h"
23+
#include "sdk/android/src/jni/pc/external_audio_processor.h"
24+
#include "sdk/android/src/jni/pc/external_audio_processing_interface.h"
25+
26+
namespace webrtc {
27+
namespace jni {
28+
29+
class ExternalAudioProcessingJni
30+
: public webrtc::ExternalAudioProcessingInterface,
31+
public rtc::RefCountInterface {
32+
public:
33+
ExternalAudioProcessingJni(JNIEnv* jni, const JavaRef<jobject>& j_processing);
34+
~ExternalAudioProcessingJni();
35+
36+
protected:
37+
virtual void Initialize(int sample_rate_hz, int num_channels) override;
38+
virtual void Reset(int new_rate) override;
39+
virtual void Process(int num_bans, int num_frames, int buffer_size, float* buffer) override;
40+
41+
private:
42+
const ScopedJavaGlobalRef<jobject> j_processing_global_;
43+
const ScopedJavaGlobalRef<jobject> j_processing_;
44+
};
45+
46+
class ExternalAudioProcessingFactory : public rtc::RefCountInterface {
47+
public:
48+
ExternalAudioProcessingFactory();
49+
virtual ~ExternalAudioProcessingFactory() = default;
50+
51+
ExternalAudioProcessor* capture_post_processor() {
52+
return capture_post_processor_;
53+
}
54+
55+
ExternalAudioProcessor* render_pre_processor() {
56+
return render_pre_processor_;
57+
}
58+
59+
rtc::scoped_refptr<webrtc::AudioProcessing> apm() { return apm_; }
60+
61+
private:
62+
rtc::scoped_refptr<webrtc::AudioProcessing> apm_;
63+
ExternalAudioProcessor* capture_post_processor_;
64+
ExternalAudioProcessor* render_pre_processor_;
65+
};
66+
67+
} // namespace jni
68+
} // namespace webrtc

0 commit comments

Comments
 (0)