Windows平台如何实现RTSP拉流添加动态水印|视频处理后转推RTMP或轻量级RTSP服务

技术背景

我们在做Windows平台流数据转发的时候，除了常规的RTSP转RTMP推送外，还有个场景就是，好多开发者希望拉取的RTSP流，做二次视频分析，或者加动态水印等，处理后的数据，再二次编码推送到RTMP服务或轻量级RTSP服务。

技术实现

本文就以Windows平台拉取RTSP流，回调yuv数据到上层，处理后的数据，二次投递到RTMP服务和轻量级RTSP服务，然后叠加动态水印，并实现处理后的数据实时录像功能，废话不多说，先上图：

上图拉取了RTSP流，然后左侧窗体显示，添加动态水印后，再在右侧预览，并把数据重新投递到推送端，考虑到编码性能，我们可选硬编码。

先说RTSP拉流，其他接口不表，这里主要是设置下video frame callback：

 
video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back_);

回调上来的video数据，投递到推送端，当然如果需要二次处理的话，处理后再丢给推送端：

 
public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
    if (frame == IntPtr.Zero)
        return;
 
    NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
 
    if (publisher_wrapper_ != null) {
                 publisher_wrapper_.post_i420_layer_image(publisher_wrapper_.get_external_video_layer_index(), video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
                    video_frame.plane2_, video_frame.stride2_,
                    video_frame.width_, video_frame.height_);
    }
}

audio的也是：

 
audio_pcm_frame_call_back_ = new SP_SDKAudioPCMFrameCallBack(SetAudioPCMFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetAudioPCMFrameCallBack(player_handle_, IntPtr.Zero, audio_pcm_frame_call_back_);

回调上来的audio数据：

 
public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
     UInt32 status, IntPtr data, UInt32 size,
     Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
{
    if (data == IntPtr.Zero || size == 0)
        return;
 
    if (publisher_wrapper_ != null)
        publisher_wrapper_.post_audio_pcm_data(data, size, 0, sample_rate, channel, per_channel_sample_number);
}

由于处理后的数据，需要重新编码，我们播放端会把原始视频宽高回调上来：

 
video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle);
NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_);

拿到视频宽高后，我们可以把宽高投递到推送端，便于推送端推送相同分辨率出去，当然，二次编码，也可以设置其他期望的分辨率：

 
private void PlaybackWindowResized(Int32 width, Int32 height)
{
    width_ = width;
    height_ = height;
 
    if (publisher_wrapper_ != null)
        publisher_wrapper_.SetResolution(width, height);
}

设置文字水印字体、字号：

 
private void btn_set_font_Click(object sender, EventArgs e)
{
    FontDialog font_dlg = new FontDialog();
    DialogResult result = font_dlg.ShowDialog();
 
    if (result == DialogResult.OK)
    {
        // 获取用户所选字体
        Font selectedFont = font_dlg.Font;
        btn_set_font.Text = "" + selectedFont.Name + ", " + selectedFont.Size + "pt";
 
        selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
    }
}

如果需要添加文字水印：

 
private async void btn_text_osd_Click(object sender, EventArgs e) {
    Bitmap bitmap = null;
 
    try
    {
        string format = "yyyy-MM-dd HH:mm:ss.fff";
 
        StringBuilder sb = new StringBuilder();
        sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
        sb.Append("\r\n");
        sb.Append("施工时间：");
        sb.Append(DateTime.Now.DayOfWeek.ToString());
        sb.Append(" ");
        sb.Append(DateTime.Now.ToString(format));
        sb.Append("\r\n");
        sb.Append("当前位置：上海市");
        string str = sb.ToString();
        bitmap = GenerateBitmap(str);
    }
    catch (Exception )
    {
        return;
    }
 
    if (null == bitmap)
        return;
 
    int x = 0;
    int y = 200;
    UpdateLayerRegion(publisher_wrapper_.get_text_layer_index(), x, y, bitmap);
    publisher_wrapper_.enable_layer(publisher_wrapper_.get_text_layer_index(), true);
 
    await Task.Delay(30);
    publisher_wrapper_.post_argb8888_layer_image(publisher_wrapper_.get_text_layer_index(), bitmap);
}

开始推送RTMP：

 
private void btn_publish_rtmp_Click(object sender, EventArgs e)
{
    if (!OpenPublisherHandle())
        return;
 
    SetCommonOptionToPublisherSDK();
 
    String url = "rtmp://192.168.0.108:1935/hls/stream1";
    //String url = "rtmp://192.168.2.154:1935/live/stream1";
 
    if (url.Length < 8)
    {
        publisher_wrapper_.try_close_handle();
        MessageBox.Show("请输入推送地址");
        return;
    }
 
    if (!publisher_wrapper_.StartPublisher(url))
    {
        MessageBox.Show("调用StartPublisher失败..");
        return;
    }
 
    btn_publish_rtmp.Enabled = false;
    btn_stop_publish_rtmp.Enabled = true;
}

publisher_wrapper接口封装：

 
public bool StartPublisher(String url)
{
    if (is_empty_handle() || is_rtmp_publishing())
        return false;
 
    if (!String.IsNullOrEmpty(url))
        NTSmartPublisherSDK.NT_PB_SetURL(handle_, url, IntPtr.Zero);
 
    if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(handle_, IntPtr.Zero))
    {
        try_close_handle();
        return false;
    }
 
    shared_lock_.EnterWriteLock();
    try
    {
        handle_reference_count_++;
        is_rtmp_publishing_ = true;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }
 
    return true;
}
 
public void StopPublisher()
{
    if (is_empty_handle() || !is_rtmp_publishing())
        return;
 
    shared_lock_.EnterWriteLock();
    try
    {
        is_rtmp_publishing_ = false;
        handle_reference_count_--;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }
 
    NTSmartPublisherSDK.NT_PB_StopPublisher(handle_);
    try_close_handle();
}

对应的SetCommonOptionToPublisherSDK()实现：

 
private void SetCommonOptionToPublisherSDK()
{
    if (publisher_wrapper_.is_empty_handle())
        return;
 
    if (publisher_wrapper_.handle_reference_count() > 0)
        return;
 
    publisher_wrapper_.config_layers(true);
 
    publisher_wrapper_.SetFrameRate((uint)video_fps_);
 
    int cur_video_codec_id = (int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;
 
    bool is_h264_encoder = true;
 
    bool is_hw_encoder = false;
 
    if (btn_check_video_hardware_encoder_.Checked)
    {
        is_hw_encoder = true;
    }
 
    Int32 cur_sel_encoder_id = 0;
    Int32 cur_sel_gpu = 0;
 
    if (is_hw_encoder)
    {
        int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
        if (cur_sel_hw >= 0)
        {
            cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
            cur_sel_gpu = -1;
 
            int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
            if (cur_sel_hw_dev >= 0)
            {
                cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
            }
        }
        else
        {
            is_hw_encoder = false;
        }
    }
 
    if (!is_hw_encoder)
    {
        if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
        {
            cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
        }
    }
 
    publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);
 
    publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));
 
    publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));
 
    publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));
 
    publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);
 
    if (is_h264_encoder)
    {
        publisher_wrapper_.SetVideoEncoderProfile(3);
    }
 
    publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));
 
    publisher_wrapper_.SetPublisherAudioCodecType(1);   //1: AAC 2: Speex
}

图层配置实现：

 
public bool config_layers(bool is_add_rgbx_zero_layer)
{
    if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
        return false;
 
    if (is_empty_handle())
        return false;
 
    int w = video_width_;
    int h = video_height_;
 
    if ((w & 0x1) != 0)
        --w;
 
    if ((h & 0x1) != 0)
        --h;
 
    if (w < 2 || h < 2)
        return false;
 
    NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero);
 
    int type, index = 0;
    if (is_add_rgbx_zero_layer)
    {
        NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig();
        type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
        fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h);
        rgba_layer.red_ = 0;
        rgba_layer.green_ = 0;
        rgba_layer.blue_ = 0;
        rgba_layer.alpha_ = 255;
        if (add_layer_config(rgba_layer, type))
            index++;
    }
 
    NT_PB_ExternalVideoFrameLayerConfig external_video_layer = new NT_PB_ExternalVideoFrameLayerConfig();
    type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
    fill_layer_base(external_video_layer, out external_video_layer.base_, type, index, true, 0, 0, w, h);
    if (add_layer_config(external_video_layer, type))
        external_video_layer_index_ = index++;
 
 
    //叠加的文本层
    NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig();
    type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
    fill_layer_base(text_layer, out text_layer.base_, type, index, false, 0, 0, 64, 64);
    if (add_layer_config(text_layer, type))
        text_layer_index_ = index++;
 
    return index > 0;
}

推送端二次录像：

 
private void btn_start_recorder_Click(object sender, EventArgs e)
{
    if (!OpenPublisherHandle())
        return;
 
   SetCommonOptionToPublisherSDK();
 
    if (!publisher_wrapper_.StartRecorder())
    {
        MessageBox.Show("调用StartRecorder失败..");
        return;
    }
 
    btn_start_recorder.Enabled = false;
    btn_stop_recorder.Enabled = true;
}
 
private void btn_pause_recorder_Click(object sender, EventArgs e)
{
    String btn_pause_rec_text = btn_pause_recorder.Text;
 
    if ("暂停录像" == btn_pause_rec_text)
    {
        UInt32 ret = publisher_wrapper_.PauseRecorder(true);
 
        if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
        {
            MessageBox.Show("暂停录像失败, 请重新尝试!");
            return;
        }
        else if (NTBaseCodeDefine.NT_ERC_OK == ret)
        {
            btn_pause_recorder.Text = "恢复录像";
        }
    }
    else
    {
        UInt32 ret = publisher_wrapper_.PauseRecorder(false);
        if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
        {
            MessageBox.Show("恢复录像失败, 请重新尝试!");
            return;
        }
        else if (NTBaseCodeDefine.NT_ERC_OK == ret)
        {
            btn_pause_recorder.Text = "暂停录像";
        }
    }
}
 
private void btn_stop_recorder_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_recording()) {
        publisher_wrapper_.StopRecorder();
 
        btn_start_recorder.Enabled = true;
        btn_stop_recorder.Enabled = false;
    }
}

启动轻量级RTSP服务：

 
private void btn_rtsp_service_Click(object sender, EventArgs e)
{
    if(publisher_wrapper_.IsRTSPSerivceRunning())
    {
        publisher_wrapper_.StopRtspService();
        btn_rtsp_service.Text = "启动RTSP服务";
        btn_rtsp_stream.Enabled = false;
    }
    else
    {
        if(publisher_wrapper_.StartRtspService())
        {
            btn_rtsp_service.Text = "停止RTSP服务";
            btn_rtsp_stream.Enabled = true;
        }
    }
}

发布RTSP流：

 
private void btn_rtsp_stream_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_rtsp_publishing())
    {
        publisher_wrapper_.StopRtspStream();
        btn_rtsp_stream.Text = "发布RTSP流";
        btn_get_rtsp_session_numbers.Enabled = false;
        btn_rtsp_service.Enabled = true;
    }
    else
    {
        if (!OpenPublisherHandle())
            return;
 
        SetCommonOptionToPublisherSDK();
 
        if (!publisher_wrapper_.StartRtspStream())
        {
            MessageBox.Show("调用StartRtspStream失败..");
            return;
        }
 
        btn_rtsp_stream.Text = "停止RTSP流";
        btn_get_rtsp_session_numbers.Enabled = true;
        btn_rtsp_service.Enabled = false;
    }
}

获取RTSP session会话数：

 
private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_rtsp_publishing())
    {
       int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();
 
       MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
    }
}

叠加后的数据，本地预览：

 
private void btn_preview_Click(object sender, EventArgs e)
{
    if (publisher_wrapper_.is_previewing())
    {
        publisher_wrapper_.StopPreview();
 
        btn_preview.Text = "开始预览";
    }
    else
    {
       if (!OpenPublisherHandle())
           return;
 
        SetCommonOptionToPublisherSDK();
               
        if (!publisher_wrapper_.StartPreview())
        {
            MessageBox.Show("调用StartPreview失败..");
            return;
        }
 
        btn_preview.Text = "停止预览";
    }
}

对应的预览封装实现：

 
public bool StartPreview()
{
    if (is_empty_handle() || is_previewing())
        return false;
 
    video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
    NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);
 
    if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(handle_, 0x800000, IntPtr.Zero))
    {
        try_close_handle();
        return false;
    }
 
    shared_lock_.EnterWriteLock();
    try
    {
        handle_reference_count_++;
        is_previewing_ = true;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }
 
    return true;
}
 
public void StopPreview()
{
    if (is_empty_handle() || !is_previewing())
        return;
 
    shared_lock_.EnterWriteLock();
    try
    {
        is_previewing_ = false;
        handle_reference_count_--;
    }
    finally
    {
        shared_lock_.ExitWriteLock();
    }
 
    NTSmartPublisherSDK.NT_PB_StopPreview(handle_);
    try_close_handle();
 
    if (render_wnd_ != null)
        render_wnd_.Invalidate();
}

预览数据回调：

 
//预览数据回调
public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
{
    NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));
 
    NT_VideoFrame pVideoFrame = new NT_VideoFrame();
 
    pVideoFrame.width_ = pb_image.width_;
    pVideoFrame.height_ = pb_image.height_;
 
    pVideoFrame.stride_ = pb_image.stride_[0];
 
    Int32 argb_size = pb_image.stride_[0] * pb_image.height_;
 
    pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size);
 
    CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size);
 
    if (sync_invoke_ != null)
    {
        System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
 
        if (sync_invoke_target != null)
        {
 
            if (sync_invoke_target.InvokeRequired)
            {
                sync_invoke_target.BeginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame });
            }
            else
            {
                set_video_preview_image_callback_(pVideoFrame);
            }
        }
    }
}

总结

以上就是RTSP流二次编辑（如增加动态水印）或视频分析（视觉算法处理）后，再录像、转推至RTMP或轻量级RTSP服务流程，经过二次处理后的流数据，配合我们的SmartPlayer，依然可以整体毫秒级的延迟体验

	video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
	NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back_);

	public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
	{
	if (frame == IntPtr.Zero)
	return;

	NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));

	if (publisher_wrapper_ != null) {
	publisher_wrapper_.post_i420_layer_image(publisher_wrapper_.get_external_video_layer_index(), video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
	video_frame.plane2_, video_frame.stride2_,
	video_frame.width_, video_frame.height_);
	}
	}

	audio_pcm_frame_call_back_ = new SP_SDKAudioPCMFrameCallBack(SetAudioPCMFrameCallBack);
	NTSmartPlayerSDK.NT_SP_SetAudioPCMFrameCallBack(player_handle_, IntPtr.Zero, audio_pcm_frame_call_back_);

	public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
	UInt32 status, IntPtr data, UInt32 size,
	Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
	{
	if (data == IntPtr.Zero \|\| size == 0)
	return;

	if (publisher_wrapper_ != null)
	publisher_wrapper_.post_audio_pcm_data(data, size, 0, sample_rate, channel, per_channel_sample_number);
	}

	video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle);
	NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_);

	private void PlaybackWindowResized(Int32 width, Int32 height)
	{
	width_ = width;
	height_ = height;

	if (publisher_wrapper_ != null)
	publisher_wrapper_.SetResolution(width, height);
	}

	private void btn_set_font_Click(object sender, EventArgs e)
	{
	FontDialog font_dlg = new FontDialog();
	DialogResult result = font_dlg.ShowDialog();

	if (result == DialogResult.OK)
	{
	// 获取用户所选字体
	Font selectedFont = font_dlg.Font;
	btn_set_font.Text = "" + selectedFont.Name + ", " + selectedFont.Size + "pt";

	selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
	}
	}

	private async void btn_text_osd_Click(object sender, EventArgs e) {
	Bitmap bitmap = null;

	try
	{
	string format = "yyyy-MM-dd HH:mm:ss.fff";

	StringBuilder sb = new StringBuilder();
	sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
	sb.Append("\r\n");
	sb.Append("施工时间：");
	sb.Append(DateTime.Now.DayOfWeek.ToString());
	sb.Append(" ");
	sb.Append(DateTime.Now.ToString(format));
	sb.Append("\r\n");
	sb.Append("当前位置：上海市");
	string str = sb.ToString();
	bitmap = GenerateBitmap(str);
	}
	catch (Exception )
	{
	return;
	}

	if (null == bitmap)
	return;

	int x = 0;
	int y = 200;
	UpdateLayerRegion(publisher_wrapper_.get_text_layer_index(), x, y, bitmap);
	publisher_wrapper_.enable_layer(publisher_wrapper_.get_text_layer_index(), true);

	await Task.Delay(30);
	publisher_wrapper_.post_argb8888_layer_image(publisher_wrapper_.get_text_layer_index(), bitmap);
	}

	private void btn_publish_rtmp_Click(object sender, EventArgs e)
	{
	if (!OpenPublisherHandle())
	return;

	SetCommonOptionToPublisherSDK();

	String url = "rtmp://192.168.0.108:1935/hls/stream1";
	//String url = "rtmp://192.168.2.154:1935/live/stream1";

	if (url.Length < 8)
	{
	publisher_wrapper_.try_close_handle();
	MessageBox.Show("请输入推送地址");
	return;
	}

	if (!publisher_wrapper_.StartPublisher(url))
	{
	MessageBox.Show("调用StartPublisher失败..");
	return;
	}

	btn_publish_rtmp.Enabled = false;
	btn_stop_publish_rtmp.Enabled = true;
	}

	public bool StartPublisher(String url)
	{
	if (is_empty_handle() \|\| is_rtmp_publishing())
	return false;

	if (!String.IsNullOrEmpty(url))
	NTSmartPublisherSDK.NT_PB_SetURL(handle_, url, IntPtr.Zero);

	if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(handle_, IntPtr.Zero))
	{
	try_close_handle();
	return false;
	}

	shared_lock_.EnterWriteLock();
	try
	{
	handle_reference_count_++;
	is_rtmp_publishing_ = true;
	}
	finally
	{
	shared_lock_.ExitWriteLock();
	}

	return true;
	}

	public void StopPublisher()
	{
	if (is_empty_handle() \|\| !is_rtmp_publishing())
	return;

	shared_lock_.EnterWriteLock();
	try
	{
	is_rtmp_publishing_ = false;
	handle_reference_count_--;
	}
	finally
	{
	shared_lock_.ExitWriteLock();
	}

	NTSmartPublisherSDK.NT_PB_StopPublisher(handle_);
	try_close_handle();
	}

	private void SetCommonOptionToPublisherSDK()
	{
	if (publisher_wrapper_.is_empty_handle())
	return;

	if (publisher_wrapper_.handle_reference_count() > 0)
	return;

	publisher_wrapper_.config_layers(true);

	publisher_wrapper_.SetFrameRate((uint)video_fps_);

	int cur_video_codec_id = (int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;

	bool is_h264_encoder = true;

	bool is_hw_encoder = false;

	if (btn_check_video_hardware_encoder_.Checked)
	{
	is_hw_encoder = true;
	}

	Int32 cur_sel_encoder_id = 0;
	Int32 cur_sel_gpu = 0;

	if (is_hw_encoder)
	{
	int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
	if (cur_sel_hw >= 0)
	{
	cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
	cur_sel_gpu = -1;

	int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
	if (cur_sel_hw_dev >= 0)
	{
	cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
	}
	}
	else
	{
	is_hw_encoder = false;
	}
	}

	if (!is_hw_encoder)
	{
	if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
	{
	cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
	}
	}

	publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);

	publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));

	publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));

	publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));

	publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);

	if (is_h264_encoder)
	{
	publisher_wrapper_.SetVideoEncoderProfile(3);
	}

	publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));

	publisher_wrapper_.SetPublisherAudioCodecType(1); //1: AAC 2: Speex
	}

	public bool config_layers(bool is_add_rgbx_zero_layer)
	{
	if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
	return false;

	if (is_empty_handle())
	return false;

	int w = video_width_;
	int h = video_height_;

	if ((w & 0x1) != 0)
	--w;

	if ((h & 0x1) != 0)
	--h;

	if (w < 2 \|\| h < 2)
	return false;

	NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero);

	int type, index = 0;
	if (is_add_rgbx_zero_layer)
	{
	NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
	fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h);
	rgba_layer.red_ = 0;
	rgba_layer.green_ = 0;
	rgba_layer.blue_ = 0;
	rgba_layer.alpha_ = 255;
	if (add_layer_config(rgba_layer, type))
	index++;
	}

	NT_PB_ExternalVideoFrameLayerConfig external_video_layer = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(external_video_layer, out external_video_layer.base_, type, index, true, 0, 0, w, h);
	if (add_layer_config(external_video_layer, type))
	external_video_layer_index_ = index++;


	//叠加的文本层
	NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig();
	type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
	fill_layer_base(text_layer, out text_layer.base_, type, index, false, 0, 0, 64, 64);
	if (add_layer_config(text_layer, type))
	text_layer_index_ = index++;

	return index > 0;
	}

	private void btn_start_recorder_Click(object sender, EventArgs e)
	{
	if (!OpenPublisherHandle())
	return;

	SetCommonOptionToPublisherSDK();

	if (!publisher_wrapper_.StartRecorder())
	{
	MessageBox.Show("调用StartRecorder失败..");
	return;
	}

	btn_start_recorder.Enabled = false;
	btn_stop_recorder.Enabled = true;
	}

	private void btn_pause_recorder_Click(object sender, EventArgs e)
	{
	String btn_pause_rec_text = btn_pause_recorder.Text;

	if ("暂停录像" == btn_pause_rec_text)
	{
	UInt32 ret = publisher_wrapper_.PauseRecorder(true);

	if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
	{
	MessageBox.Show("暂停录像失败, 请重新尝试!");
	return;
	}
	else if (NTBaseCodeDefine.NT_ERC_OK == ret)
	{
	btn_pause_recorder.Text = "恢复录像";
	}
	}
	else
	{
	UInt32 ret = publisher_wrapper_.PauseRecorder(false);
	if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
	{
	MessageBox.Show("恢复录像失败, 请重新尝试!");
	return;
	}
	else if (NTBaseCodeDefine.NT_ERC_OK == ret)
	{
	btn_pause_recorder.Text = "暂停录像";
	}
	}
	}

	private void btn_stop_recorder_Click(object sender, EventArgs e)
	{
	if (publisher_wrapper_.is_recording()) {
	publisher_wrapper_.StopRecorder();

	btn_start_recorder.Enabled = true;
	btn_stop_recorder.Enabled = false;
	}
	}

	private void btn_rtsp_service_Click(object sender, EventArgs e)
	{
	if(publisher_wrapper_.IsRTSPSerivceRunning())
	{
	publisher_wrapper_.StopRtspService();
	btn_rtsp_service.Text = "启动RTSP服务";
	btn_rtsp_stream.Enabled = false;
	}
	else
	{
	if(publisher_wrapper_.StartRtspService())
	{
	btn_rtsp_service.Text = "停止RTSP服务";
	btn_rtsp_stream.Enabled = true;
	}
	}
	}

	private void btn_rtsp_stream_Click(object sender, EventArgs e)
	{
	if (publisher_wrapper_.is_rtsp_publishing())
	{
	publisher_wrapper_.StopRtspStream();
	btn_rtsp_stream.Text = "发布RTSP流";
	btn_get_rtsp_session_numbers.Enabled = false;
	btn_rtsp_service.Enabled = true;
	}
	else
	{
	if (!OpenPublisherHandle())
	return;

	SetCommonOptionToPublisherSDK();

	if (!publisher_wrapper_.StartRtspStream())
	{
	MessageBox.Show("调用StartRtspStream失败..");
	return;
	}

	btn_rtsp_stream.Text = "停止RTSP流";
	btn_get_rtsp_session_numbers.Enabled = true;
	btn_rtsp_service.Enabled = false;
	}
	}

	private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
	{
	if (publisher_wrapper_.is_rtsp_publishing())
	{
	int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();

	MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
	}
	}

	private void btn_preview_Click(object sender, EventArgs e)
	{
	if (publisher_wrapper_.is_previewing())
	{
	publisher_wrapper_.StopPreview();

	btn_preview.Text = "开始预览";
	}
	else
	{
	if (!OpenPublisherHandle())
	return;

	SetCommonOptionToPublisherSDK();

	if (!publisher_wrapper_.StartPreview())
	{
	MessageBox.Show("调用StartPreview失败..");
	return;
	}

	btn_preview.Text = "停止预览";
	}
	}

	public bool StartPreview()
	{
	if (is_empty_handle() \|\| is_previewing())
	return false;

	video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
	NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);

	if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(handle_, 0x800000, IntPtr.Zero))
	{
	try_close_handle();
	return false;
	}

	shared_lock_.EnterWriteLock();
	try
	{
	handle_reference_count_++;
	is_previewing_ = true;
	}
	finally
	{
	shared_lock_.ExitWriteLock();
	}

	return true;
	}

	public void StopPreview()
	{
	if (is_empty_handle() \|\| !is_previewing())
	return;

	shared_lock_.EnterWriteLock();
	try
	{
	is_previewing_ = false;
	handle_reference_count_--;
	}
	finally
	{
	shared_lock_.ExitWriteLock();
	}

	NTSmartPublisherSDK.NT_PB_StopPreview(handle_);
	try_close_handle();

	if (render_wnd_ != null)
	render_wnd_.Invalidate();
	}

	//预览数据回调
	public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
	{
	NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));

	NT_VideoFrame pVideoFrame = new NT_VideoFrame();

	pVideoFrame.width_ = pb_image.width_;
	pVideoFrame.height_ = pb_image.height_;

	pVideoFrame.stride_ = pb_image.stride_[0];

	Int32 argb_size = pb_image.stride_[0] * pb_image.height_;

	pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size);

	CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size);

	if (sync_invoke_ != null)
	{
	System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;

	if (sync_invoke_target != null)
	{

	if (sync_invoke_target.InvokeRequired)
	{
	sync_invoke_target.BeginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame });
	}
	else
	{
	set_video_preview_image_callback_(pVideoFrame);
	}
	}
	}
	}

Windows平台如何实现RTSP拉流添加动态水印|视频处理后转推RTMP或轻量级RTSP服务

​技术背景

技术实现

总结

技术背景