​技术背景

我们在做Windows平台流数据转发的时候，除了常规的RTSP转RTMP推送外，还有个场景就是，好多开发者希望拉取的RTSP流，做二次视频分析，或者加动态水印等，处理后的数据，再二次编码推送到RTMP服务或轻量级RTSP服务。

技术实现

本文就以Windows平台拉取RTSP流，回调yuv数据到上层，处理后的数据，二次投递到RTMP服务和轻量级RTSP服务，然后叠加动态水印，并实现处理后的数据实时录像功能，废话不多说，先上图：

上图拉取了RTSP流，然后左侧窗体显示，添加动态水印后，再在右侧预览，并把数据重新投递到推送端，考虑到编码性能，我们可选硬编码。

先说RTSP拉流，其他接口不表，这里主要是设置下video frame callback：

video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back_);

回调上来的video数据，投递到推送端，当然如果需要二次处理的话，处理后再丢给推送端：

public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
    if (frame == IntPtr.Zero)
        return;

    NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));

    if (publisher_wrapper_ != null) {
                 publisher_wrapper_.post_i420_layer_image(publisher_wrapper_.get_external_video_layer_index(), video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
                    video_frame.plane2_, video_frame.stride2_,
                    video_frame.width_, video_frame.height_);
    }
}

audio的也是：

audio_pcm_frame_call_back_ = new SP_SDKAudioPCMFrameCallBack(SetAudioPCMFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetAudioPCMFrameCallBack(player_handle_, IntPtr.Zero, audio_pcm_frame_call_back_);

回调上来的audio数据：

public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
     UInt32 status, IntPtr data, UInt32 size,
     Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
{
    if (data == IntPtr.Zero || size == 0)
        return;

    if (publisher_wrapper_ != null)
        publisher_wrapper_.post_audio_pcm_data(data, size, 0, sample_rate, channel, per_channel_sample_number);
}

由于处理后的数据，需要重新编码，我们播放端会把原始视频宽高回调上来：

video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle);
NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_);

拿到视频宽高后，我们可以把宽高投递到推送端，便于推送端推送相同分辨率出去，当然，二次编码，也可以设置其他期望的分辨率：

private void PlaybackWindowResized(Int32 width, Int32 height)
{
    width_ = width;
    height_ = height;

    if (publisher_wrapper_ != null)
        publisher_wrapper_.SetResolution(width, height);
}

设置文字水印字体、字号：

private void btn_set_font_Click(object sender, EventArgs e)
{
    FontDialog font_dlg = new FontDialog();
    DialogResult result = font_dlg.ShowDialog();

    if (result == DialogResult.OK)
    {
        // 获取用户所选字体
        Font selectedFont = font_dlg.Font;
        btn_set_font.Text = "" + selectedFont.Name + ", " + selectedFont.Size + "pt";

        selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
    }
}

如果需要添加文字水印：

private async void btn_text_osd_Click(object sender, EventArgs e) {
    Bitmap bitmap = null;

    try
    {
        string format = "yyyy-MM-dd HH:mm:ss.fff";

        StringBuilder sb = new StringBuilder();
        sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
        sb.Append("\r\n");
        sb.Append("施工时间：");
        sb.Append(DateTime.Now.DayOfWeek.ToString());
        sb.Append(" ");
        sb.Append(DateTime.Now.ToString(format));
        sb.Append("\r\n");
        sb.Append("当前位置：上海市");
        string str = sb.ToString();
        bitmap = GenerateBitmap(str);
    }
    catch (Exception )
    {
        return;
    }

    if (null == bitmap)
        return;

    int x = 0;
    int y = 200;
    UpdateLayerRegion(publisher_wrapper_.get_text_layer_index(), x, y, bitmap);
    publisher_wrapper_.enable_layer(publisher_wrapper_.get_text_layer_index(), true);

    await Task.Delay(30);
    publisher_wrapper_.post_argb8888_layer_image(publisher_wrapper_.get_text_layer_index(), bitmap);
}

开始推送RTMP：

private void btn_publish_rtmp_Click(object sender, EventArgs e)
{
    if (!OpenPublisherHandle())
        return;

    SetCommonOptionToPublisherSDK();

    String url = "rtmp://192.168.0.108:1935/hls/stream1";
    //String url = "rtmp://192.168.2.154:1935/live/stream1";

    if (url.Length < 8)
    {
        publisher_wrapper_.try_close_handle();
        MessageBox.Show("请输入推送地址");
        return;
    }

    if (!publisher_wrapper_.StartPublisher(url))
    {
        MessageBox.Show("调用StartPublisher失败..");
        return;
    }

    btn_publish_rtmp.Enabled = false;
    btn_stop_publish_rtmp.Enabled = true;
}

publisher_wrapper接口封装：

public bool StartPublisher(String url)
{
    if (is_empty_handle() || is_rtmp_publishing())
        return false;

    if (!String.IsNullOrEmpty(url))
        NTSmartPublisherSDK.NT_PB_SetURL(handle_, url, IntPtr.Zero);

    if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(handle_, IntPtr.Zero))
    {
        try_close_handle();
        return false;
    }

    shared_lock_.EnterWriteLock();
    try
    {
        handle_reference_count_++;
        is_rtmp_publishing_ = true;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }

    return true;
}

public void StopPublisher()
{
    if (is_empty_handle() || !is_rtmp_publishing())
        return;

    shared_lock_.EnterWriteLock();
    try
    {
        is_rtmp_publishing_ = false;
        handle_reference_count_--;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }

    NTSmartPublisherSDK.NT_PB_StopPublisher(handle_);
    try_close_handle();
}

对应的SetCommonOptionToPublisherSDK()实现：

private void SetCommonOptionToPublisherSDK()
{
    if (publisher_wrapper_.is_empty_handle())
        return;

    if (publisher_wrapper_.handle_reference_count() > 0)
        return;

    publisher_wrapper_.config_layers(true);

    publisher_wrapper_.SetFrameRate((uint)video_fps_);

    int cur_video_codec_id = (int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;

    bool is_h264_encoder = true;

    bool is_hw_encoder = false;

    if (btn_check_video_hardware_encoder_.Checked)
    {
        is_hw_encoder = true;
    }

    Int32 cur_sel_encoder_id = 0;
    Int32 cur_sel_gpu = 0;

    if (is_hw_encoder)
    {
        int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
        if (cur_sel_hw >= 0)
        {
            cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
            cur_sel_gpu = -1;

            int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
            if (cur_sel_hw_dev >= 0)
            {
                cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
            }
        }
        else
        {
            is_hw_encoder = false;
        }
    }

    if (!is_hw_encoder)
    {
        if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
        {
            cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
        }
    }

    publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);

    publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));

    publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));

    publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));

    publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);

    if (is_h264_encoder)
    {
        publisher_wrapper_.SetVideoEncoderProfile(3);
    }

    publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));

    publisher_wrapper_.SetPublisherAudioCodecType(1);   //1: AAC 2: Speex
}

图层配置实现：

public bool config_layers(bool is_add_rgbx_zero_layer)
{
    if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
        return false;

    if (is_empty_handle())
        return false;

    int w = video_width_;
    int h = video_height_;

    if ((w & 0x1) != 0)
        --w;

    if ((h & 0x1) != 0)
        --h;

    if (w < 2 || h < 2)
        return false;

    NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero);

    int type, index = 0;
    if (is_add_rgbx_zero_layer)
    {
        NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig();
        type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
        fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h);
        rgba_layer.red_ = 0;
        rgba_layer.green_ = 0;
        rgba_layer.blue_ = 0;
        rgba_layer.alpha_ = 255;
        if (add_layer_config(rgba_layer, type))
            index++;
    }

    NT_PB_ExternalVideoFrameLayerConfig external_video_layer = new NT_PB_ExternalVideoFrameLayerConfig();
    type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
    fill_layer_base(external_video_layer, out external_video_layer.base_, type, index, true, 0, 0, w, h);
    if (add_layer_config(external_video_layer, type))
        external_video_layer_index_ = index++;


    //叠加的文本层
    NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig();
    type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
    fill_layer_base(text_layer, out text_layer.base_, type, index, false, 0, 0, 64, 64);
    if (add_layer_config(text_layer, type))
        text_layer_index_ = index++;

    return index > 0;
}

推送端二次录像：

private void btn_start_recorder_Click(object sender, EventArgs e)
{
    if (!OpenPublisherHandle())
        return;

   SetCommonOptionToPublisherSDK();

    if (!publisher_wrapper_.StartRecorder())
    {
        MessageBox.Show("调用StartRecorder失败..");
        return;
    }

    btn_start_recorder.Enabled = false;
    btn_stop_recorder.Enabled = true;
}

private void btn_pause_recorder_Click(object sender, EventArgs e)
{
    String btn_pause_rec_text = btn_pause_recorder.Text;

    if ("暂停录像" == btn_pause_rec_text)
    {
        UInt32 ret = publisher_wrapper_.PauseRecorder(true);

        if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
        {
            MessageBox.Show("暂停录像失败, 请重新尝试!");
            return;
        }
        else if (NTBaseCodeDefine.NT_ERC_OK == ret)
        {
            btn_pause_recorder.Text = "恢复录像";
        }
    }
    else
    {
        UInt32 ret = publisher_wrapper_.PauseRecorder(false);
        if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
        {
            MessageBox.Show("恢复录像失败, 请重新尝试!");
            return;
        }
        else if (NTBaseCodeDefine.NT_ERC_OK == ret)
        {
            btn_pause_recorder.Text = "暂停录像";
        }
    }
}

private void btn_stop_recorder_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_recording()) {
        publisher_wrapper_.StopRecorder();

        btn_start_recorder.Enabled = true;
        btn_stop_recorder.Enabled = false;
    }
}

启动轻量级RTSP服务：

private void btn_rtsp_service_Click(object sender, EventArgs e)
{
    if(publisher_wrapper_.IsRTSPSerivceRunning())
    {
        publisher_wrapper_.StopRtspService();
        btn_rtsp_service.Text = "启动RTSP服务";
        btn_rtsp_stream.Enabled = false;
    }
    else
    {
        if(publisher_wrapper_.StartRtspService())
        {
            btn_rtsp_service.Text = "停止RTSP服务";
            btn_rtsp_stream.Enabled = true;
        }
    }
}

发布RTSP流：

private void btn_rtsp_stream_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_rtsp_publishing())
    {
        publisher_wrapper_.StopRtspStream();
        btn_rtsp_stream.Text = "发布RTSP流";
        btn_get_rtsp_session_numbers.Enabled = false;
        btn_rtsp_service.Enabled = true;
    }
    else
    {
        if (!OpenPublisherHandle())
            return;

        SetCommonOptionToPublisherSDK();

        if (!publisher_wrapper_.StartRtspStream())
        {
            MessageBox.Show("调用StartRtspStream失败..");
            return;
        }

        btn_rtsp_stream.Text = "停止RTSP流";
        btn_get_rtsp_session_numbers.Enabled = true;
        btn_rtsp_service.Enabled = false;
    }
}

获取RTSP session会话数：

private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_rtsp_publishing())
    {
       int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();

       MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
    }
}

叠加后的数据，本地预览：

private void btn_preview_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_previewing())
    {
        publisher_wrapper_.StopPreview();

        btn_preview.Text = "开始预览";
    }
    else
    {
       if (!OpenPublisherHandle())
           return;

        SetCommonOptionToPublisherSDK();
               
        if (!publisher_wrapper_.StartPreview())
        {
            MessageBox.Show("调用StartPreview失败..");
            return;
        }

        btn_preview.Text = "停止预览";
    }
}

对应的预览封装实现：

public bool StartPreview()
{
    if (is_empty_handle() || is_previewing())
        return false;

    video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
    NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);

    if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(handle_, 0x800000, IntPtr.Zero))
    {
        try_close_handle();
        return false;
    }

    shared_lock_.EnterWriteLock();
    try
    {
        handle_reference_count_++;
        is_previewing_ = true;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }

    return true;
}

public void StopPreview()
{
    if (is_empty_handle() || !is_previewing())
        return;

    shared_lock_.EnterWriteLock();
    try
    {
        is_previewing_ = false;
        handle_reference_count_--;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }

    NTSmartPublisherSDK.NT_PB_StopPreview(handle_);
    try_close_handle();

    if (render_wnd_ != null)
        render_wnd_.Invalidate();
}

预览数据回调：

//预览数据回调
public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
{
    NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));

    NT_VideoFrame pVideoFrame = new NT_VideoFrame();

    pVideoFrame.width_ = pb_image.width_;
    pVideoFrame.height_ = pb_image.height_;

    pVideoFrame.stride_ = pb_image.stride_[0];

    Int32 argb_size = pb_image.stride_[0] * pb_image.height_;

    pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size);

    CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size);

    if (sync_invoke_ != null)
    {
        System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;

        if (sync_invoke_target != null)
        {

            if (sync_invoke_target.InvokeRequired)
            {
                sync_invoke_target.BeginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame });
            }
            else
            {
                set_video_preview_image_callback_(pVideoFrame);
            }
        }
    }
}

总结

以上就是RTSP流二次编辑（如增加动态水印）或视频分析（视觉算法处理）后，再录像、转推至RTMP或轻量级RTSP服务流程，经过二次处理后的流数据，配合我们的SmartPlayer，依然可以整体毫秒级的延迟体验