Windows platform RTMP push | lightweight RTSP service to achieve local camera | screen | superimposed data preview

Windows platform RTMP push | lightweight RTSP service to achieve local camera | screen | superimposed data preview

background

When you are doing RTMP push or lightweight RTSP services on the Windows platform, whether it is capturing screens or capturing cameras, or the overlay mode of the screen cameras, there will always be such demands. For the collected data, I hope to see the details locally. The actual effect of the collected data or image is the "Preview" function introduced this time.

Not much nonsense, think about the picture:

How to achieve

Start preview

To start the preview, the general process is to call OpenPublisherHandle() to set the initial data source type, and then call the Open() interface to obtain the push handler and set the event callback.

bool CSmartPublisherDemoDlg::OpenPublisherHandle () { if (publisher_handle_ != NULL ) { return true ; } //video auto video_option = NT_PB_E_VIDEO_OPTION_NO_VIDEO; if (BST_CHECKED == btn_check_desktop_camera_switch_. GetCheck () || BST_CHECKED == btn_check_camera_overlay_to_desktop_. GetCheck () || BST_CHECKED == btn_check_desktop_overlay_to_camera_. GetCheck ()) { //Use overlay mode video_option = NT_PB_E_VIDEO_OPTION_LAYER; } else if (BST_CHECKED == btn_check_desktop_input_. GetCheck () && BST_CHECKED == btn_check_scale_desktop_. GetCheck ()) { //Use overlay mode to achieve zoom video_option = NT_PB_E_VIDEO_OPTION_LAYER; } else if (BST_CHECKED == btn_check_desktop_input_. GetCheck () && BST_CHECKED != btn_check_scale_desktop_. GetCheck ()) { video_option = NT_PB_E_VIDEO_OPTION_SCREEN; } else if (BST_CHECKED == btn_check_window_input_. GetCheck ()) { video_option = NT_PB_E_VIDEO_OPTION_WINDOW; //video_option = NT_PB_E_VIDEO_OPTION_LAYER; } else if (BST_CHECKED == btn_check_camera_input_. GetCheck ()) { video_option = NT_PB_E_VIDEO_OPTION_CAMERA; } //Audio auto audio_option = NT_PB_E_AUDIO_OPTION_NO_AUDIO; audio_option = NT_PB_E_AUDIO_OPTION_NO_AUDIO; if (BST_CHECKED == btn_check_auido_mic_input_. GetCheck () && BST_CHECKED == btn_check_auido_speaker_input_. GetCheck ()) { audio_option = NT_PB_E_AUDIO_OPTION_CAPTURE_MIC_SPEAKER_MIXER; } else if (BST_CHECKED == btn_check_auido_mic_input_. GetCheck ()) { audio_option = NT_PB_E_AUDIO_OPTION_CAPTURE_MIC; } else if (BST_CHECKED == btn_check_auido_speaker_input_. GetCheck ()) { audio_option = NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER; } publisher_handle_count_ = 0 ; if (NT_ERC_OK != publisher_api_. Open (&publisher_handle_, video_option, audio_option, 0 , NULL )) { AfxMessageBox (_T( "Call open failed!" )); return false ; } if (publisher_handle_ != NULL ) { //Set event callback publisher_api_. SetEventCallBack (publisher_handle_, GetSafeHwnd (), &NT_PB_SDKPublisherEventHandle); return true ; } else { return false ; } } Copy code

Currently our SDK supports the following video source types, and real-time preview is limited to the data type before encoding:

/*Define Video source options*/ typedef enum _ NT_PB_E_VIDEO_OPTION { NT_PB_E_VIDEO_OPTION_NO_VIDEO = 0x0 , = NT_PB_E_VIDEO_OPTION_SCREEN 0x1 , //Acquisition screen NT_PB_E_VIDEO_OPTION_CAMERA = 0x2 , //camera capture NT_PB_E_VIDEO_OPTION_LAYER = 0x3 , //video merge, video cameras, such as the desktop overlay NT_PB_E_VIDEO_OPTION_ENCODED_DATA = 0x4 , //already coded video data, currently supports H264 NT_PB_E_VIDEO_OPTION_WINDOW = 0x5 , //Acquisition window } NT_PB_E_VIDEO_OPTION; Copy code

Then call SetCommonOptionToPublisherSDK() to complete the overlay and other parameter settings.

void CSmartPublisherDemoDlg::SetCommonOptionToPublisherSDK () { ASSERT (publisher_handle_ != NULL ); layer_conf_wrappers_. clear (); //Video related settings if (BST_CHECKED == btn_check_desktop_camera_switch_. GetCheck () || BST_CHECKED == btn_check_camera_overlay_to_desktop_. GetCheck () || BST_CHECKED == btn_check_desktop_overlay_to_camera_. GetCheck () || BST_CHECKED == btn_check_desktop_input_. GetCheck () || BST_CHECKED == btn_check_window_input_. GetCheck () || BST_CHECKED == btn_check_camera_input_. GetCheck ()) { if (BST_CHECKED == btn_check_desktop_camera_switch_. GetCheck ()) { auto left = GetIntValueFromEdit (&edit_clip_left_); auto top = GetIntValueFromEdit (&edit_clip_top_); auto w = GetIntValueFromEdit (&edit_clip_width_); auto h = GetIntValueFromEdit (&edit_clip_height_); //If one is 0, use full screen if (w == 0 || h == 0 ) { left = 0 ; top = 0 ; GetScreenSize (w, h); } else { //Ensure 4-byte alignment w = NT_ByteAlign (w, 2 ); h = NT_ByteAlign (h, 4 ); } auto index = 0 ; //The 0th layer is filled with an RGBA rectangle, the purpose is to ensure the frame rate, and the color is filled with all black auto rgba_layer_c0 = std::make_shared<nt_pb_sdk::RGBARectangleLayerConfigWrapper>(index++, true , 0 , 0 , w, h); rgba_layer_c0->conf_.red_ = 0 ; rgba_layer_c0->conf_.green_ = 0 ; rgba_layer_c0->conf_.blue_ = 0 ; rgba_layer_c0->conf_.alpha_ = 255 ; layer_conf_wrappers_. push_back (rgba_layer_c0); //The first layer is the camera if (CB_ERR != cur_sel_camera_index_) { auto & camera = cameras_[cur_sel_camera_index_]; auto camera_layer_c1 = std::make_shared<nt_pb_sdk::CameraLayerConfigWrapperV2>(index++, false , 0 , 0 , w, h); strcpy_s (camera_layer_c1->conf_.device_unique_id_utf8_, camera.id_. c_str ()); camera_layer_c1->conf_.is_flip_horizontal_ = BST_CHECKED == btn_check_flip_horizontal_camera_. GetCheck (); camera_layer_c1->conf_.is_flip_vertical_ = BST_CHECKED == btn_check_flip_vertical_camera_. GetCheck (); //Do not rotate in this superposition mode, otherwise the deformation will be severe, either set an angle and adjust the width and height, but do not dynamically rotate camera_layer_c1->conf_.rotate_degress_ = 0 ; layer_conf_wrappers_. push_back (camera_layer_c1); } //The second layer is the screen auto screen_layer_c2 = std::make_shared<nt_pb_sdk::ScreenLayerConfigWrapper>(index++, true , 0 , 0 , w, h); screen_layer_c2->conf_.reserve_ = nullptr ; screen_layer_c2->conf_.clip_region_.x_ = left; screen_layer_c2->conf_.clip_region_.y_ = top; screen_layer_c2->conf_.clip_region_.width_ = w; screen_layer_c2->conf_.clip_region_.height_ = h; layer_conf_wrappers_. push_back (screen_layer_c2); //Fill the 3rd layer with RGBA rectangle, add translucent effect auto r = GetIntValueFromEdit (&edit_rgba_rect_layer_red_); r = ClipIntValue (r, 0 , 255 ); auto g = GetIntValueFromEdit (&edit_rgba_rect_layer_green_); g = ClipIntValue (g, 0 , 255 ); auto b = GetIntValueFromEdit (&edit_rgba_rect_layer_blue_); b = ClipIntValue (b, 0 , 255 ); auto a = GetIntValueFromEdit (&edit_rgba_rect_layer_alpha_); a = ClipIntValue (a, 0 , 255 ); auto rgba_layer_c3 = std::make_shared<nt_pb_sdk::RGBARectangleLayerConfigWrapper>(index++, a != 0 , 0 , 0 , w, h); rgba_layer_c3->conf_.red_ = r; rgba_layer_c3->conf_.green_ = g; rgba_layer_c3->conf_.blue_ = b; rgba_layer_c3->conf_.alpha_ = a; layer_conf_wrappers_. push_back (rgba_layer_c3); //If there is a picture, add a picture layer if (!image_layer_file_name_utf8_. empty () && image_layer_width_> 0 && image_layer_height_> 0 ) { auto image_layer_c4 = std::make_shared<nt_pb_sdk::ImageLayerConfigWrapper>(index++, true , image_layer_left_, image_layer_top_, image_layer_width_, image_layer_height_ ); strcpy_s (image_layer_c4->conf_.file_name_utf8_, image_layer_file_name_utf8_. c_str ()); //The image layer can add background, if you are interested, you can open it to see the effect/ *image_layer_c4->conf_.is_setting_background_ = 1; image_layer_c4->conf_.bk_red_ = 255; image_layer_c4->conf_.bk_green_ = 255; image_layer_c4->conf_.bk_blue_ = 255;*/ layer_conf_wrappers_. push_back (image_layer_c4); } SetLayersConfig (); publisher_api_. SetFrameRate (publisher_handle_, GetIntValueFromEdit (&edit_frame_rate_)); } else if (BST_CHECKED == btn_check_camera_overlay_to_desktop_. GetCheck ()) { auto left = GetIntValueFromEdit (&edit_clip_left_); auto top = GetIntValueFromEdit (&edit_clip_top_); auto w = GetIntValueFromEdit (&edit_clip_width_); auto h = GetIntValueFromEdit (&edit_clip_height_); //If one is 0, use full screen if (w == 0 || h == 0 ) { left = 0 ; top = 0 ; GetScreenSize (w, h); } else { //Ensure 4-byte alignment w = NT_ByteAlign (w, 2 ); h = NT_ByteAlign (h, 4 ); } auto index = 0 ; /*auto image_layer_c0 = std::make_shared<nt_pb_sdk::ImageLayerConfigWrapper>(index++, true, 0, 0, w, h); strcpy_s(image_layer_c0->conf_.file_name_utf8_, "D:\\myxxyy\\testpng\\ggdwdd.png"); image_layer_c0->conf_.is_setting_background_ = 1; image_layer_c0->conf_.bk_red_ = 255; image_layer_c0->conf_.bk_green_ = 255; image_layer_c0->conf_.bk_blue_ = 255; layer_conf_wrappers_.push_back(image_layer_c0);*/ //The 0th layer is the screen auto screen_layer_c0 = std::make_shared<nt_pb_sdk::ScreenLayerConfigWrapper>(index++, true , 0 , 0 , w, h); screen_layer_c0->conf_.reserve_ = nullptr ; screen_layer_c0->conf_.clip_region_.x_ = left; screen_layer_c0->conf_.clip_region_.y_ = top; screen_layer_c0->conf_.clip_region_.width_ = w; screen_layer_c0->conf_.clip_region_.height_ = h; layer_conf_wrappers_. push_back (screen_layer_c0); //The first layer is the camera if (CB_ERR != cur_sel_camera_index_) { auto & camera = cameras_[cur_sel_camera_index_]; auto c_l = GetIntValueFromEdit (&edit_camera_overlay_left_); auto c_t = GetIntValueFromEdit (&edit_camera_overlay_top_); auto c_w = GetIntValueFromEdit (&edit_camera_overlay_width_); auto c_h = GetIntValueFromEdit (&edit_camera_overlay_height_); if (c_w == 0 ) { c_w = w/2 ; } if (c_h == 0 ) { c_h = h/2 ; } if (c_w> 0 && c_h> 0 ) { auto camera_layer_c1 = std::make_shared<nt_pb_sdk::CameraLayerConfigWrapperV2>(index++, true , c_l, c_t , c_w, c_h); strcpy_s (camera_layer_c1->conf_.device_unique_id_utf8_, camera.id_. c_str ()); camera_layer_c1->conf_.is_flip_horizontal_ = BST_CHECKED == btn_check_flip_horizontal_camera_. GetCheck (); camera_layer_c1->conf_.is_flip_vertical_ = BST_CHECKED == btn_check_flip_vertical_camera_. GetCheck (); camera_layer_c1->conf_.rotate_degress_ = GetCameraRotateDegress (); layer_conf_wrappers_. push_back (camera_layer_c1); } } SetLayersConfig (); publisher_api_. SetFrameRate (publisher_handle_, GetIntValueFromEdit (&edit_frame_rate_)); } else if (BST_CHECKED == btn_check_desktop_overlay_to_camera_. GetCheck ()) { if (CB_ERR != cur_sel_camera_index_ && CB_ERR != cur_sel_camera_resolutions_index_ && CB_ERR != cur_sel_camera_frame_rate_index_) { auto & camera = cameras_[cur_sel_camera_index_]; auto & cap = camera.capabilities_[cur_sel_camera_resolutions_index_]; auto index = 0 ; //The 0th layer is the camera auto camera_layer_c0 = std::make_shared<nt_pb_sdk::CameraLayerConfigWrapperV2>(index++, true , 0 , 0 , cap.width_, cap.height_); strcpy_s (camera_layer_c0->conf_.device_unique_id_utf8_, camera.id_. c_str ()); camera_layer_c0->conf_.is_flip_horizontal_ = BST_CHECKED == btn_check_flip_horizontal_camera_. GetCheck (); camera_layer_c0->conf_.is_flip_vertical_ = BST_CHECKED == btn_check_flip_vertical_camera_. GetCheck (); //Do not rotate in this superposition mode, otherwise the distortion will be severe, either set an angle and adjust the width and height, but do not dynamically rotate camera_layer_c0->conf_.rotate_degress_ = 0 ; layer_conf_wrappers_. push_back (camera_layer_c0); //The first layer is the screen auto screen_layer_c1 = std::make_shared<nt_pb_sdk::ScreenLayerConfigWrapper>(index++, true , 0 , 0 , cap.width_/2 , cap.height_/2 ); screen_layer_c1->conf_.reserve_ = nullptr ; screen_layer_c1->conf_.clip_region_.x_ = 0 ; screen_layer_c1->conf_.clip_region_.y_ = 0 ; screen_layer_c1->conf_.clip_region_.width_ = cap.width_/2 ; screen_layer_c1->conf_.clip_region_.height_ = cap.height_/2 ; layer_conf_wrappers_. push_back (screen_layer_c1); SetLayersConfig (); publisher_api_. SetFrameRate (publisher_handle_, cur_sel_camera_frame_rate_index_ + 1 ); } } else if (BST_CHECKED == btn_check_desktop_input_. GetCheck () && BST_CHECKED == btn_check_scale_desktop_. GetCheck ()) { int left = 0 , top = 0 , w = 0 , h = 0 , scale_w = 0 , scale_h = 0 ; GetScreenScaleConfigInfo (left, top, w, h, scale_w, scale_h); auto index = 0 ; //The 0th layer is filled with an RGBA rectangle, the purpose is to ensure the frame rate, and the color is filled with all black auto rgba_layer_c0 = std::make_shared<nt_pb_sdk::RGBARectangleLayerConfigWrapper>(index++, true , 0 , 0 , scale_w, scale_h); rgba_layer_c0->conf_.red_ = 0 ; rgba_layer_c0->conf_.green_ = 0 ; rgba_layer_c0->conf_.blue_ = 0 ; rgba_layer_c0->conf_.alpha_ = 255 ; layer_conf_wrappers_. push_back (rgba_layer_c0); //The first layer is the screen auto screen_layer_c1 = std::make_shared<nt_pb_sdk::ScreenLayerConfigWrapperV2>(index++, true , 0 , 0 , scale_w, scale_h); screen_layer_c1->conf_.reserve1_ = nullptr ; screen_layer_c1->conf_.reserve2_ = 0 ; screen_layer_c1->conf_.clip_region_.x_ = left; screen_layer_c1->conf_.clip_region_.y_ = top; screen_layer_c1->conf_.clip_region_.width_ = w; screen_layer_c1->conf_.clip_region_.height_ = h; screen_layer_c1->conf_.scale_filter_mode_ = 3 ; //The screen zoom takes into account the definition, here 3 is used, the highest zoom quality layer_conf_wrappers_. push_back (screen_layer_c1); SetLayersConfig (); publisher_api_. SetFrameRate (publisher_handle_, GetIntValueFromEdit (&edit_frame_rate_)); } else if (BST_CHECKED == btn_check_desktop_input_. GetCheck () && BST_CHECKED != btn_check_scale_desktop_. GetCheck ()) { publisher_api_. SetScreenClip (publisher_handle_, GetIntValueFromEdit (&edit_clip_left_), GetIntValueFromEdit (&edit_clip_top_), GetIntValueFromEdit (&edit_clip_width_), GetIntValueFromEdit (&edit_clip_height_)); publisher_api_. SetFrameRate (publisher_handle_, GetIntValueFromEdit (&edit_frame_rate_)); } else if (BST_CHECKED == btn_check_window_input_. GetCheck ()) { if ( nullptr != cur_sel_capture_window_) { publisher_api_. SetCaptureWindow (publisher_handle_, cur_sel_capture_window_); publisher_api_. SetFrameRate (publisher_handle_, GetIntValueFromEdit (&edit_frame_rate_)); //Window layer test and demo code++ /*auto index = 0; auto rgba_layer_c0 = std::make_shared<nt_pb_sdk::RGBARectangleLayerConfigWrapper>(index++, true, 0, 0, 1280, 720); rgba_layer_c0->conf_.red_ = 0; rgba_layer_c0->conf_.green_ = 255; rgba_layer_c0->conf_.blue_ = 0; rgba_layer_c0->conf_.alpha_ = 255; layer_conf_wrappers_.push_back(rgba_layer_c0); auto window_layer_c1 = std::make_shared<nt_pb_sdk::WindowLayerConfigWrapper>(index++, true, 100, 100, 800, 600); window_layer_c1->conf_.window_ = cur_sel_capture_window_; layer_conf_wrappers_.push_back(window_layer_c1); SetLayersConfig(); */ //Window layer test and demo code-- } } else if (BST_CHECKED == btn_check_camera_input_. GetCheck ()) { if (CB_ERR != cur_sel_camera_index_ && CB_ERR != cur_sel_camera_resolutions_index_ && CB_ERR != cur_sel_camera_frame_rate_index_) { auto & camera = cameras_[cur_sel_camera_index_]; auto & cap = camera.capabilities_[cur_sel_camera_resolutions_index_]; publisher_api_. SetVideoCaptureDeviceBaseParameter (publisher_handle_, camera.id_. c_str (), cap.width_, cap.height_); publisher_api_. SetFrameRate (publisher_handle_, cur_sel_camera_frame_rate_index_+ 1 ); publisher_api_. FlipVerticalCamera (publisher_handle_, BST_CHECKED == btn_check_flip_vertical_camera_. GetCheck ()); publisher_api_. FlipHorizontalCamera (publisher_handle_, BST_CHECKED == btn_check_flip_horizontal_camera_. GetCheck ()); auto degress = GetCameraRotateDegress (); publisher_api_. RotateCamera (publisher_handle_, degress); } } publisher_api_. EnableDXGIScreenCapturer (publisher_handle_, BST_CHECKED == btn_check_dxgi_screen_capturer_. GetCheck ()); publisher_api_. DisableAeroScreenCapturer (publisher_handle_, BST_CHECKED == btn_check_capturer_disable_aero_. GetCheck ()); publisher_api_. EnableScreenCaptureLayeredWindow (publisher_handle_, BST_CHECKED == check_capture_layered_window_. GetCheck ()? 1 : 0 ); publisher_api_. SetCaptureWindowWay (publisher_handle_, BST_CHECKED == btn_check_wr_way_capture_window_. GetCheck ()? 2 : 1 ); auto cur_video_codec_id = btn_check_h265_encoder_. GetCheck () != BST_CHECKED? NT_MEDIA_CODEC_ID_H264: NT_MEDIA_CODEC_ID_H265; NT_INT32 cur_sel_encoder_id = 0 ; NT_INT32 cur_sel_gpu = 0 ; auto is_hw_encoder = btn_check_video_hardware_encoder_. GetCheck () == BST_CHECKED; if (is_hw_encoder) { auto cur_sel_hw = combobox_video_encoders_. GetCurSel (); if (cur_sel_hw >= 0 ) { cur_sel_encoder_id = combobox_video_encoders_. GetItemData (cur_sel_hw); cur_sel_gpu = -1 ; auto cur_sel_hw_dev = combobox_video_hardware_encoder_devices_. GetCurSel (); if (cur_sel_hw_dev >= 0 ) { cur_sel_gpu = combobox_video_hardware_encoder_devices_. GetItemData (cur_sel_hw_dev); } } else { is_hw_encoder = false ; } } if (!is_hw_encoder) { if (NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id) { cur_sel_encoder_id = btn_check_openh264_encoder_. GetCheck () == BST_CHECKED? 1 : 0 ; } } publisher_api_. SetVideoEncoder (publisher_handle_, is_hw_encoder? 1 : 0 , cur_sel_encoder_id, cur_video_codec_id, cur_sel_gpu); if (BST_CHECKED != btn_check_window_input_. GetCheck ()) { publisher_api_. SetVideoBitRate (publisher_handle_, GetIntValueFromEdit (&edit_bit_rate_)); } else { //The resolution of the window will change, so set a set of bitrates and go down auto frame_rate = GetIntValueFromEdit (&edit_frame_rate_); SetBitrateGroup (publisher_handle_, frame_rate); } publisher_api_. SetVideoQualityV2 (publisher_handle_, GetIntValueFromEdit (&edit_video_quality_)); publisher_api_. SetVideoMaxBitRate (publisher_handle_, GetIntValueFromEdit (&edit_video_max_bitrate_)); publisher_api_. SetVideoKeyFrameInterval (publisher_handle_, GetIntValueFromEdit (&edit_key_frame_)); if (NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id) { auto profile_sel = combox_h264_profile_. GetCurSel (); if (profile_sel != CB_ERR) { publisher_api_. SetVideoEncoderProfile (publisher_handle_, profile_sel + 1 ); } } publisher_api_. SetVideoEncoderSpeed (publisher_handle_, GetIntValueFromEdit (&edit_video_encode_speed_)); //Clear all the specific parameters of the encoder publisher_api_. ClearVideoEncoderSpecialOptions (publisher_handle_); if (cur_sel_encoder_id == 1 ) { //qp_max and qp_min are currently only valid for openh264, here only set the configuration value publisher_api_ in the scenario where openh264 is used . SetVideoEncoderQPMax (publisher_handle_, GetIntValueFromEdit (&edit_qp_max_)); publisher_api_. SetVideoEncoderQPMin (publisher_handle_, GetIntValueFromEdit (&edit_qp_min_)); //openh264 configures specific parameters publisher_api_. SetVideoEncoderSpecialInt32Option (publisher_handle_, "usage_type" , btn_check_openh264_ppt_usage_type_. GetCheck () == BST_CHECKED? 1 : 0 ); publisher_api_. SetVideoEncoderSpecialInt32Option (publisher_handle_, "rc_mode" , btn_check_openh264_rc_bitrate_mode_. GetCheck () == BST_CHECKED? 1 : 0 ); publisher_api_. SetVideoEncoderSpecialInt32Option (publisher_handle_, "enable_frame_skip" , btn_check_openh264_frame_skip_. GetCheck () == BST_CHECKED? 1 : 0 ); } else { publisher_api_. SetVideoEncoderQPMax (publisher_handle_, -1 ); publisher_api_. SetVideoEncoderQPMin (publisher_handle_, -1 ); } } //Audio related settings if (BST_CHECKED == btn_check_auido_mic_input_. GetCheck ()) { auto count = combox_auido_input_devices_. GetCount (); if (count != CB_ERR && count> 0 ) { auto cur_sel = combox_auido_input_devices_. GetCurSel (); if (cur_sel != CB_ERR) { publisher_api_. SetAuidoInputDeviceId (publisher_handle_, cur_sel); } } } //Do mute compensation when only collecting speakers if (BST_CHECKED != btn_check_auido_mic_input_. GetCheck () && BST_CHECKED == btn_check_auido_speaker_input_. GetCheck ()) { publisher_api_. SetCaptureSpeakerCompensateMute (publisher_handle_, 1 ); } if (BST_CHECKED == btn_check_speex_encoder_. GetCheck ()) { publisher_api_. SetPublisherAudioCodecType (publisher_handle_, 2 ); publisher_api_. SetPublisherSpeexEncoderQuality (publisher_handle_, GetIntValueFromEdit (&edit_speex_quality_)); } else { publisher_api_. SetPublisherAudioCodecType (publisher_handle_, 1 ); } if (BST_CHECKED == btn_check_auido_mic_input_. GetCheck () || BST_CHECKED == btn_check_auido_speaker_input_. GetCheck ()) { if (BST_CHECKED == btn_check_set_mute_. GetCheck ()) { publisher_api_. SetMute (publisher_handle_, 1 ); } } if (BST_CHECKED == btn_check_echo_cancel_. GetCheck ()) { publisher_api_. SetEchoCancellation (publisher_handle_, 1 , GetIntValueFromEdit (&edit_echo_delay_)); } else { publisher_api_. SetEchoCancellation (publisher_handle_, 0 , 0 ); } if (BST_CHECKED == btn_check_noise_suppression_. GetCheck ()) { publisher_api_. SetNoiseSuppression (publisher_handle_, 1 ); } else { publisher_api_. SetNoiseSuppression (publisher_handle_, 0 ); } if (BST_CHECKED == btn_check_agc_. GetCheck ()) { publisher_api_. SetAGC (publisher_handle_, 1 ); } else { publisher_api_. SetAGC (publisher_handle_, 0 ); } if (BST_CHECKED == btn_check_vad_. GetCheck ()) { publisher_api_. SetVAD (publisher_handle_, 1 ); } else { publisher_api_. SetVAD (publisher_handle_, 0 ); } if (BST_CHECKED == btn_check_auido_mic_input_. GetCheck () && BST_CHECKED == btn_check_auido_speaker_input_. GetCheck ()) { publisher_api_. SetInputAudioVolume (publisher_handle_, 0 , GetDoubleValueFromEdit (&edit_audio_input_volume_)); publisher_api_. SetInputAudioVolume (publisher_handle_, 1 , GetDoubleValueFromEdit (&edit_audio_speaker_input_volume_)); } else if (BST_CHECKED == btn_check_auido_mic_input_. GetCheck ()) { publisher_api_. SetInputAudioVolume (publisher_handle_, 0 , GetDoubleValueFromEdit (&edit_audio_input_volume_)); } else if (BST_CHECKED == btn_check_auido_speaker_input_. GetCheck ()) { publisher_api_. SetInputAudioVolume (publisher_handle_, 0 , GetDoubleValueFromEdit (&edit_audio_speaker_input_volume_)); } } Copy code

Next, set up the preview callback:

. publisher_api_ SetVideoPreviewImageCallBack (publisher_handle_, NT_PB_E_IMAGE_FORMAT_RGB32, GetSafeHwnd (), & NT_PB_SDKVideoPreviewImageHandle); duplicated code

Then, call the preview interface:

void CSmartPublisherDemoDlg::OnBnClickedButtonStartPreview () { if (BST_CHECKED == btn_check_window_input_. GetCheck ()) { if ( nullptr == cur_sel_capture_window_) { AfxMessageBox (_T( "Please drop down to select the collection window first" )); return ; } } if (publisher_handle_ == NULL ) { if (! OpenPublisherHandle ()) { return ; } } if (publisher_handle_count_ < 1 ) { SetCommonOptionToPublisherSDK (); } ASSERT (publisher_handle_ != NULL ); publisher_api_. SetVideoPreviewImageCallBack (publisher_handle_, NT_PB_E_IMAGE_FORMAT_RGB32, GetSafeHwnd (), &NT_PB_SDKVideoPreviewImageHandle); if (NT_ERC_OK != publisher_api_. StartPreview (publisher_handle_, 0 , NULL )) { if ( 0 == publisher_handle_count_) { publisher_api_. Close (publisher_handle_); publisher_handle_ = NULL ; } AfxMessageBox (_T( "Preview failed, please make sure to select the video capture option" )); return ; } publisher_handle_count_++; btn_preview_. EnableWindow (FALSE); btn_stop_preview_. EnableWindow (TRUE); if ( 1 == publisher_handle_count_) { if (BST_CHECKED == btn_check_desktop_input_. GetCheck ()) { btn_choose_screen_region_. SetWindowTextW ( L "Move the screen area" ); } btn_check_dxgi_screen_capturer_. EnableWindow (FALSE); check_capture_layered_window_. EnableWindow (FALSE); btn_check_wr_way_capture_window_. EnableWindow (FALSE); btn_check_desktop_camera_switch_. EnableWindow (FALSE); btn_check_camera_overlay_to_desktop_. EnableWindow (FALSE); btn_check_desktop_overlay_to_camera_. EnableWindow (FALSE); btn_add_image_watermark_. EnableWindow (FALSE); if (BST_CHECKED == btn_check_desktop_camera_switch_. GetCheck () || BST_CHECKED == btn_check_camera_overlay_to_desktop_. GetCheck () || BST_CHECKED == btn_check_desktop_overlay_to_camera_. GetCheck ()) { } else { btn_check_desktop_input_. EnableWindow (FALSE); btn_check_scale_desktop_. EnableWindow (FALSE); edit_desktop_scale_. EnableWindow (FALSE); btn_check_camera_input_. EnableWindow (FALSE); btn_check_window_input_. EnableWindow (FALSE); } DisableAuidoInputControl (); } CreatePreViewWindow (); ShowPreViewWindow ( true ); } Copy code

NT_PB_SDKVideoPreviewImageHandle related callback implementation:

extern "C" NT_VOID NT_CALLBACK NT_PB_SDKVideoPreviewImageHandle (NT_HANDLE handle, NT_PVOID user_data, const NT_PB_Image* image) { if (image != nullptr && image->format_ == NT_PB_E_IMAGE_FORMAT_RGB32 && image->width_> 0 && image->height_> 0 && image->plane_[ 0 ] != nullptr && image->stride_[ 0 ]> 0 ) { std::unique_ptr<nt_pb_rgbx_image> rgbx_image ( new nt_pb_rgbx_image()) ; rgbx_image->width_ = image->width_; rgbx_image->height_ = image->height_; rgbx_image->stride_ = image->stride_[ 0 ]; rgbx_image->size_ = image->stride_[ 0 ] * image->height_; rgbx_image->data_. reset ( new NT_BYTE[rgbx_image->size_]); if (rgbx_image->data_) { memcpy (rgbx_image->data_. get (), image->plane_[ 0 ], rgbx_image->size_); HWND hwnd = reinterpret_cast <HWND>(user_data); if (hwnd != NULL && :: IsWindow (hwnd)) { :: PostMessage (hwnd, WM_USER_PB_SDK_PREVIEW_RGBX_IMAGE, (WPARAM)(rgbx_image. release ()), 0 ); } } } /*std::ostringstream ss; ss << "OnPreviewImage handle:" << handle << "w=" << image->width_ << "h=" << image->height_ << "format=" << image->format_ <<"/r\n"; OutputDebugStringA(ss.str().c_str());*/ } Copy code

Here, the local real-time preview is completed by calling the encapsulated nt_pb_ui_preview_wnd:

LRESULT CSmartPublisherDemoDlg::OnSDKPreviewRGBXImage (WPARAM wParam, LPARAM lParam) { nt_pb_rgbx_image* rgbx_image = (nt_pb_rgbx_image*)(wParam); if (rgbx_image != nullptr ) { std::shared_ptr<nt_pb_rgbx_image> sp_rgbx_image (rgbx_image) ; preview_wnd_. OnRGBXImage (sp_rgbx_image); } return S_OK; } Copy code

The specific implementation is as follows:

void nt_pb_ui_preview_wnd::OnRGBXImage(const std::shared_ptr<nt_pb_rgbx_image>& sp_image) { rgbx_image_ = sp_image; if (m_hWnd != NULL && ::IsWindow(m_hWnd) && ::IsWindowVisible(m_hWnd)) { InvalidateRect(NULL, FALSE); } } Copy code

OnPaint() refreshes the data in real time:

void nt_pb_ui_preview_wnd::OnPaint () { CPaintDC dc ( this ) ;//device context for painting //TODO: Add your message handler code here //Do not call CWnd::OnPaint() for painting messages if ( IsIconic ()) { return ; } CRect rc_client ( 0 , 0 , 0 , 0 ) ; GetClientRect (rc_client); if (rc_client. IsRectNull () || rc_client. IsRectEmpty ()) { return ; } auto mem_dc = :: CreateCompatibleDC (dc. GetSafeHdc ()); auto mem_bitmap = :: CreateCompatibleBitmap (dc. GetSafeHdc (), rc_client. Width (), rc_client. Height ()); :: SelectObject (mem_dc, mem_bitmap); HBRUSH brush = :: CreateSolidBrush ( RGB ( 238 , 243 , 250 )); :: FillRect (mem_dc, &rc_client, brush); :: DeleteObject (brush); DrawImage (mem_dc, rc_client); :: BitBlt (dc. GetSafeHdc (), 0 , 0 , rc_client. Width (), rc_client. Height (), mem_dc, 0 , 0 , SRCCOPY); :: DeleteObject (mem_bitmap); :: DeleteDC (mem_dc); } Copy code

DrawImage() package implementation

void nt_pb_ui_preview_wnd::DrawImage (HDC hdc, const CRect& rc_client) { ASSERT (hdc != NULL ); ASSERT (!rc_client. IsRectNull ()); ASSERT (!rc_client. IsRectEmpty ()); if (!rgbx_image_) return ; if (rc_client. Width () < 100 || rc_client. Height () < 100 ) return ; if (draw_api_ == nullptr ) return ; NT_PB_Image image; memset (&image, 0 , sizeof (image)); image.cb_size_ = sizeof (image); image.format_ = NT_PB_E_IMAGE_FORMAT_RGB32; image.width_ = rgbx_image_->width_; image.height_ = rgbx_image_->height_; image.timestamp_ = 0 ; image.plane_[ 0 ] = rgbx_image_->data_. get (); image.stride_[ 0 ] = rgbx_image_->stride_; image.plane_size_[ 0 ] = rgbx_image_->size_; auto limit_w = rc_client. Width () -8 ; auto limit_h = rc_client. Height () -8 ; auto d_w = 0 , d_h = 0 ; if (rgbx_image_->width_ <= limit_w && rgbx_image_->height_ <= limit_h) { d_w = rgbx_image_->width_; d_h = rgbx_image_->height_; } else { CalScaleSize (limit_w, limit_h, rgbx_image_->width_, rgbx_image_->height_, d_w, d_h); } if (d_w> 0 && d_h> 0 ) { auto d_x = rc_client. Width ()/2 -d_w/2 ; auto d_y = rc_client. Height ()/2 -d_h/2 ; draw_api_-> Draw (hdc, d_x, d_y, d_w, d_h, 0 , 0 , rgbx_image_->width_, rgbx_image_->height_, &image); } } Copy code

Stop preview

Just call StopPreview().

void CSmartPublisherDemoDlg::OnBnClickedButtonStopPreview () { publisher_handle_count_--; publisher_api_. StopPreview (publisher_handle_); if ( 0 == publisher_handle_count_) { publisher_api_. Close (publisher_handle_); publisher_handle_ = NULL ; } btn_preview_. EnableWindow (TRUE); btn_stop_preview_. EnableWindow (FALSE); if ( 0 == publisher_handle_count_) { if (BST_CHECKED == btn_check_desktop_input_. GetCheck ()) { btn_choose_screen_region_. SetWindowTextW ( L"Select screen region" ); } btn_check_dxgi_screen_capturer_. EnableWindow (TRUE); check_capture_layered_window_. EnableWindow (TRUE); btn_check_wr_way_capture_window_. EnableWindow (TRUE); btn_desktop_camera_switch_. SetWindowTextW ( L "Switch to camera" ); btn_disable_image_watermark_. SetWindowTextW ( L"Stop Watermark" ); btn_disable_camera_overlay_. SetWindowTextW ( L "Stop overlaying camera" ); btn_disable_desktop_overlay_. SetWindowTextW ( L"Stop screen overlay" ); btn_check_desktop_camera_switch_. EnableWindow (TRUE); btn_check_camera_overlay_to_desktop_. EnableWindow (TRUE); btn_check_desktop_overlay_to_camera_. EnableWindow (TRUE); btn_add_image_watermark_. EnableWindow (TRUE); if (BST_CHECKED == btn_check_desktop_camera_switch_. GetCheck () || BST_CHECKED == btn_check_camera_overlay_to_desktop_. GetCheck () || BST_CHECKED == btn_check_desktop_overlay_to_camera_. GetCheck ()) { } else { btn_check_desktop_input_. EnableWindow (TRUE); btn_check_scale_desktop_. EnableWindow (TRUE); edit_desktop_scale_. EnableWindow (TRUE); btn_check_camera_input_. EnableWindow (TRUE); btn_check_window_input_. EnableWindow (TRUE); } EnableAuidoInputControl (); } ShowPreViewWindow ( false ); } Copy code