Add Direction indicator to TransformableFrames

Currently the implementation of FrameTransformers uses distinct,
incompatible types for recevied vs about-to-be-sent frames. This adds a
flag in the interface so we can at least check that we are being given
the correct type. crbug.com/1250638 tracks removing the need for this.

Chrome will be updated after this to check the direction flag and provide
a javascript error if the wrong type of frame is written into the
encoded insertable streams writable stream, rather than crashing.

Bug: chromium:1247260
Change-Id: I9cbb66962ea0718ed47c5e5dba19a8ff9635b0b1
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/232301
Reviewed-by: Harald Alvestrand <[email protected]>
Commit-Queue: Tony Herre <[email protected]>
Cr-Commit-Position: refs/heads/main@{#35100}

Author: Tony Herre

api/frame_transformer_interface.h

@@ -36,6 +36,16 @@ class TransformableFrameInterface {
   virtual uint8_t GetPayloadType() const = 0;
   virtual uint32_t GetSsrc() const = 0;
   virtual uint32_t GetTimestamp() const = 0;
+
+  enum class Direction {
+    kUnknown,
+    kReceiver,
+    kSender,
+  };
+  // TODO(crbug.com/1250638): Remove this distinction between receiver and
+  // sender frames to allow received frames to be directly re-transmitted on
+  // other PeerConnectionss.
+  virtual Direction GetDirection() const { return Direction::kUnknown; }
 };
 
 class TransformableVideoFrameInterface : public TransformableFrameInterface {

audio/channel_receive_frame_transformer_delegate.cc

@@ -18,15 +18,16 @@
 namespace webrtc {
 namespace {
 
-class TransformableAudioFrame : public TransformableAudioFrameInterface {
+class TransformableIncomingAudioFrame
+    : public TransformableAudioFrameInterface {
  public:
-  TransformableAudioFrame(rtc::ArrayView<const uint8_t> payload,
-                          const RTPHeader& header,
-                          uint32_t ssrc)
+  TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload,
+                                  const RTPHeader& header,
+                                  uint32_t ssrc)
       : payload_(payload.data(), payload.size()),
         header_(header),
         ssrc_(ssrc) {}
-  ~TransformableAudioFrame() override = default;
+  ~TransformableIncomingAudioFrame() override = default;
   rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
 
   void SetData(rtc::ArrayView<const uint8_t> data) override {
@@ -37,6 +38,7 @@ class TransformableAudioFrame : public TransformableAudioFrameInterface {
   uint32_t GetSsrc() const override { return ssrc_; }
   uint32_t GetTimestamp() const override { return header_.timestamp; }
   const RTPHeader& GetHeader() const override { return header_; }
+  Direction GetDirection() const override { return Direction::kReceiver; }
 
  private:
   rtc::Buffer payload_;
@@ -72,7 +74,7 @@ void ChannelReceiveFrameTransformerDelegate::Transform(
     uint32_t ssrc) {
   RTC_DCHECK_RUN_ON(&sequence_checker_);
   frame_transformer_->Transform(
-      std::make_unique<TransformableAudioFrame>(packet, header, ssrc));
+      std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc));
 }
 
 void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(
@@ -89,7 +91,10 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
   RTC_DCHECK_RUN_ON(&sequence_checker_);
   if (!receive_frame_callback_)
     return;
-  auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get());
+  RTC_CHECK_EQ(frame->GetDirection(),
+               TransformableFrameInterface::Direction::kReceiver);
+  auto* transformed_frame =
+      static_cast<TransformableIncomingAudioFrame*>(frame.get());
   receive_frame_callback_(transformed_frame->GetData(),
                           transformed_frame->GetHeader());
 }

audio/channel_send_frame_transformer_delegate.cc

@@ -15,24 +15,24 @@
 namespace webrtc {
 namespace {
 
-class TransformableAudioFrame : public TransformableFrameInterface {
+class TransformableOutgoingAudioFrame : public TransformableFrameInterface {
  public:
-  TransformableAudioFrame(AudioFrameType frame_type,
-                          uint8_t payload_type,
-                          uint32_t rtp_timestamp,
-                          uint32_t rtp_start_timestamp,
-                          const uint8_t* payload_data,
-                          size_t payload_size,
-                          int64_t absolute_capture_timestamp_ms,
-                          uint32_t ssrc)
+  TransformableOutgoingAudioFrame(AudioFrameType frame_type,
+                                  uint8_t payload_type,
+                                  uint32_t rtp_timestamp,
+                                  uint32_t rtp_start_timestamp,
+                                  const uint8_t* payload_data,
+                                  size_t payload_size,
+                                  int64_t absolute_capture_timestamp_ms,
+                                  uint32_t ssrc)
       : frame_type_(frame_type),
         payload_type_(payload_type),
         rtp_timestamp_(rtp_timestamp),
         rtp_start_timestamp_(rtp_start_timestamp),
         payload_(payload_data, payload_size),
         absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms),
         ssrc_(ssrc) {}
-  ~TransformableAudioFrame() override = default;
+  ~TransformableOutgoingAudioFrame() override = default;
   rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
   void SetData(rtc::ArrayView<const uint8_t> data) override {
     payload_.SetData(data.data(), data.size());
@@ -48,6 +48,7 @@ class TransformableAudioFrame : public TransformableFrameInterface {
   int64_t GetAbsoluteCaptureTimestampMs() const {
     return absolute_capture_timestamp_ms_;
   }
+  Direction GetDirection() const override { return Direction::kSender; }
 
  private:
   AudioFrameType frame_type_;
@@ -90,9 +91,10 @@ void ChannelSendFrameTransformerDelegate::Transform(
     size_t payload_size,
     int64_t absolute_capture_timestamp_ms,
     uint32_t ssrc) {
-  frame_transformer_->Transform(std::make_unique<TransformableAudioFrame>(
-      frame_type, payload_type, rtp_timestamp, rtp_start_timestamp,
-      payload_data, payload_size, absolute_capture_timestamp_ms, ssrc));
+  frame_transformer_->Transform(
+      std::make_unique<TransformableOutgoingAudioFrame>(
+          frame_type, payload_type, rtp_timestamp, rtp_start_timestamp,
+          payload_data, payload_size, absolute_capture_timestamp_ms, ssrc));
 }
 
 void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
@@ -111,9 +113,12 @@ void ChannelSendFrameTransformerDelegate::SendFrame(
     std::unique_ptr<TransformableFrameInterface> frame) const {
   MutexLock lock(&send_lock_);
   RTC_DCHECK_RUN_ON(encoder_queue_);
+  RTC_CHECK_EQ(frame->GetDirection(),
+               TransformableFrameInterface::Direction::kSender);
   if (!send_frame_callback_)
     return;
-  auto* transformed_frame = static_cast<TransformableAudioFrame*>(frame.get());
+  auto* transformed_frame =
+      static_cast<TransformableOutgoingAudioFrame*>(frame.get());
   send_frame_callback_(transformed_frame->GetFrameType(),
                        transformed_frame->GetPayloadType(),
                        transformed_frame->GetTimestamp() -

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc

@@ -78,6 +78,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface {
     return expected_retransmission_time_ms_;
   }
 
+  Direction GetDirection() const override { return Direction::kSender; }
+
  private:
   rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data_;
   const RTPVideoHeader header_;
@@ -147,6 +149,8 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame(
 void RTPSenderVideoFrameTransformerDelegate::SendVideo(
     std::unique_ptr<TransformableFrameInterface> transformed_frame) const {
   RTC_CHECK(encoder_queue_->IsCurrent());
+  RTC_CHECK_EQ(transformed_frame->GetDirection(),
+               TransformableFrameInterface::Direction::kSender);
   MutexLock lock(&sender_lock_);
   if (!sender_)
     return;

video/rtp_video_stream_receiver_frame_transformer_delegate.cc

@@ -59,6 +59,8 @@ class TransformableVideoReceiverFrame
     return std::move(frame_);
   }
 
+  Direction GetDirection() const override { return Direction::kReceiver; }
+
  private:
   std::unique_ptr<RtpFrameObject> frame_;
   const VideoFrameMetadata metadata_;
@@ -111,6 +113,8 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
 void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
     std::unique_ptr<TransformableFrameInterface> frame) {
   RTC_DCHECK_RUN_ON(&network_sequence_checker_);
+  RTC_CHECK_EQ(frame->GetDirection(),
+               TransformableFrameInterface::Direction::kReceiver);
   if (!receiver_)
     return;
   auto transformed_frame = absl::WrapUnique(