Skip to content

fall back to software H264 encoder when hardware ProcessInput fails #1062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
richiemcilroy merged 3 commits into from
Sep 24, 2025
Merged

fall back to software H264 encoder when hardware ProcessInput fails #1062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a379cf
fall back to software H264 encoder when hardware ProcessInput fails
richiemcilroy Sep 24, 2025
92014e0
cleanup
richiemcilroy Sep 24, 2025
3227e2c
0.3.72
richiemcilroy Sep 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion apps/desktop/src-tauri/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "cap-desktop"
version = "0.3.71"
version = "0.3.72"
description = "Beautiful screen recordings, owned by you."
authors = ["you"]
edition = "2024"
Expand Down
25 changes: 19 additions & 6 deletions crates/enc-mediafoundation/src/mft.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,29 @@ pub struct EncoderDevice {

impl EncoderDevice {
pub fn enumerate(major_type: GUID, subtype: GUID) -> Result<Vec<EncoderDevice>> {
Self::enumerate_with_flags(
major_type,
subtype,
MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY | MFT_ENUM_FLAG_SORTANDFILTER,
)
}

pub fn enumerate_with_flags(
major_type: GUID,
subtype: GUID,
flags: MFT_ENUM_FLAG,
) -> Result<Vec<EncoderDevice>> {
let output_info = MFT_REGISTER_TYPE_INFO {
guidMajorType: major_type,
guidSubtype: subtype,
};
let encoders = enumerate_mfts(
&MFT_CATEGORY_VIDEO_ENCODER,
MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY | MFT_ENUM_FLAG_SORTANDFILTER,
None,
Some(&output_info),
)?;
let flags = if flags.0 == 0 {
MFT_ENUM_FLAG_SORTANDFILTER
} else {
flags | MFT_ENUM_FLAG_SORTANDFILTER
};
let encoders =
enumerate_mfts(&MFT_CATEGORY_VIDEO_ENCODER, flags, None, Some(&output_info))?;
let mut encoder_devices = Vec::new();
for encoder in encoders {
let display_name = if let Some(display_name) =
Expand Down
94 changes: 74 additions & 20 deletions crates/enc-mediafoundation/src/video/h264.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ use windows::{
MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE,
MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK,
MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType,
MFCreateSample, MFMediaType_Video, MFT_MESSAGE_COMMAND_FLUSH,
MFCreateSample, MFMediaType_Video, MFT_ENUM_FLAG, MFT_ENUM_FLAG_HARDWARE,
MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_FLUSH,
MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM,
MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM,
MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY,
Expand Down Expand Up @@ -97,13 +98,15 @@ pub enum HandleNeedsInputError {
unsafe impl Send for H264Encoder {}

impl H264Encoder {
pub fn new_with_scaled_output(
fn new_with_scaled_output_with_flags(
d3d_device: &ID3D11Device,
format: DXGI_FORMAT,
input_resolution: SizeInt32,
output_resolution: SizeInt32,
frame_rate: u32,
bitrate_multipler: f32,
flags: MFT_ENUM_FLAG,
enable_hardware_transforms: bool,
) -> Result<Self, NewVideoEncoderError> {
Comment on lines +101 to 110
Copy link
Contributor

@coderabbitai coderabbitai bot Sep 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Drop the now-ineffective enable_hardware_transforms parameter and adjust call sites.

Selection is controlled at enumeration time via flags; the parameter is unused after removing the SetUINT32.

Apply these diffs:

Signature:

-        flags: MFT_ENUM_FLAG,
-        enable_hardware_transforms: bool,
+        flags: MFT_ENUM_FLAG,

Call site (hardware path):

-            MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY,
-            true,
+            MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY,

Call site (software path):

-            MFT_ENUM_FLAG_TRANSCODE_ONLY,
-            false,
+            MFT_ENUM_FLAG_TRANSCODE_ONLY,

Also applies to: 300-318, 320-338

let bitrate = calculate_bitrate(
output_resolution.Width as u32,
Expand All @@ -112,13 +115,14 @@ impl H264Encoder {
bitrate_multipler,
);

let transform = EncoderDevice::enumerate(MFMediaType_Video, MFVideoFormat_H264)
.map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?
.first()
.cloned()
.ok_or(NewVideoEncoderError::NoVideoEncoderDevice)?
.create_transform()
.map_err(NewVideoEncoderError::EncoderTransform)?;
let transform =
EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags)
.map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?
.first()
.cloned()
.ok_or(NewVideoEncoderError::NoVideoEncoderDevice)?
.create_transform()
.map_err(NewVideoEncoderError::EncoderTransform)?;

Comment on lines +118 to 126
Copy link
Contributor

@coderabbitai coderabbitai bot Sep 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Don’t assume the first enumerated MFT will activate—try all candidates.

If the first device fails to activate, you currently error out instead of trying others. Iterate until one succeeds to improve reliability.

Apply this diff:

-        let transform =
-            EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags)
-                .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?
-                .first()
-                .cloned()
-                .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)?
-                .create_transform()
-                .map_err(NewVideoEncoderError::EncoderTransform)?;
+        let transform = {
+            let devices =
+                EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags)
+                    .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?;
+            devices
+                .into_iter()
+                .find_map(|d| d.create_transform().ok())
+                .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)?
+        };
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
let transform =
EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags)
.map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?
.first()
.cloned()
.ok_or(NewVideoEncoderError::NoVideoEncoderDevice)?
.create_transform()
.map_err(NewVideoEncoderError::EncoderTransform)?;
let transform = {
let devices =
EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags)
.map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?;
devices
.into_iter()
.find_map(|d| d.create_transform().ok())
.ok_or(NewVideoEncoderError::NoVideoEncoderDevice)?
};
🤖 Prompt for AI Agents 
In crates/enc-mediafoundation/src/video/h264.rs around lines 118 to 126, the
code currently takes the first enumerated encoder device and errors if it fails
to activate; instead iterate over all candidates returned by
EncoderDevice::enumerate_with_flags, attempt to activate/create_transform for
each one, and return the first successful transform; if none succeed, convert
the last (or aggregated) error into NewVideoEncoderError::EncoderTransform or
NoVideoEncoderDevice as appropriate. Concretely: call enumerate_with_flags(...)?
to get the list, if empty return NoVideoEncoderDevice; then for each candidate
call cloned.create_transform().map_err(NewVideoEncoderError::EncoderTransform)
and on the first Ok return it, otherwise continue; after the loop return
NoVideoEncoderDevice or the last EncoderTransform error.

let video_processor = VideoProcessor::new(
d3d_device.clone(),
Expand All @@ -130,7 +134,6 @@ impl H264Encoder {
)
.map_err(NewVideoEncoderError::VideoProcessor)?;

// Create MF device manager
let mut device_manager_reset_token: u32 = 0;
let media_device_manager = {
let mut media_device_manager = None;
Expand All @@ -149,7 +152,6 @@ impl H264Encoder {
.map_err(NewVideoEncoderError::DeviceManager)?
};

// Setup MFTransform
let event_generator: IMFMediaEventGenerator = transform
.cast()
.map_err(NewVideoEncoderError::EventGenerator)?;
Expand All @@ -163,7 +165,10 @@ impl H264Encoder {
.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1)
.map_err(NewVideoEncoderError::EventGenerator)?;
attributes
.SetUINT32(&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1)
.SetUINT32(
&MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS,
enable_hardware_transforms as u32,
)
.map_err(NewVideoEncoderError::EventGenerator)?;
};

Expand All @@ -182,13 +187,6 @@ impl H264Encoder {
match result {
Ok(_) => {}
Err(error) => {
// https://docs.microsoft.com/en-us/windows/win32/api/mftransform/nf-mftransform-imftransform-getstreamids
// This method can return E_NOTIMPL if both of the following conditions are true:
// * The transform has a fixed number of streams.
// * The streams are numbered consecutively from 0 to n – 1, where n is the
// number of input streams or output streams. In other words, the first
// input stream is 0, the second is 1, and so on; and the first output
// stream is 0, the second is 1, and so on.
if error.code() == E_NOTIMPL {
for i in 0..number_of_input_streams {
input_stream_ids[i as usize] = i;
Expand All @@ -206,7 +204,6 @@ impl H264Encoder {
let input_stream_id = input_stream_ids[0];
let output_stream_id = output_stream_ids[0];

// TOOD: Avoid this AddRef?
unsafe {
let temp = media_device_manager.clone();
transform
Expand Down Expand Up @@ -300,6 +297,46 @@ impl H264Encoder {
})
}

pub fn new_with_scaled_output(
d3d_device: &ID3D11Device,
format: DXGI_FORMAT,
input_resolution: SizeInt32,
output_resolution: SizeInt32,
frame_rate: u32,
bitrate_multipler: f32,
) -> Result<Self, NewVideoEncoderError> {
Self::new_with_scaled_output_with_flags(
d3d_device,
format,
input_resolution,
output_resolution,
frame_rate,
bitrate_multipler,
MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY,
true,
)
}

pub fn new_with_scaled_output_software(
d3d_device: &ID3D11Device,
format: DXGI_FORMAT,
input_resolution: SizeInt32,
output_resolution: SizeInt32,
frame_rate: u32,
bitrate_multipler: f32,
) -> Result<Self, NewVideoEncoderError> {
Self::new_with_scaled_output_with_flags(
d3d_device,
format,
input_resolution,
output_resolution,
frame_rate,
bitrate_multipler,
MFT_ENUM_FLAG_TRANSCODE_ONLY,
false,
)
}

pub fn new(
d3d_device: &ID3D11Device,
format: DXGI_FORMAT,
Expand All @@ -317,6 +354,23 @@ impl H264Encoder {
)
}

pub fn new_software(
d3d_device: &ID3D11Device,
format: DXGI_FORMAT,
resolution: SizeInt32,
frame_rate: u32,
bitrate_multipler: f32,
) -> Result<Self, NewVideoEncoderError> {
Self::new_with_scaled_output_software(
d3d_device,
format,
resolution,
resolution,
frame_rate,
bitrate_multipler,
)
}

pub fn bitrate(&self) -> u32 {
self.bitrate
}
Expand Down
Loading
Loading