From 70ec155627b63dbd4c356940c7616a3781b77ac9 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 28 May 2026 06:47:37 -0500 Subject: [PATCH 01/30] Add JUCE audio recorder and fix FFmpegWriter FLAC duration - Add AudioRecorder with input device capture, monitoring, recording, stats, level snapshots, and waveform snapshots - Add AudioRecorder settings, level data, waveform chunk data, and frame factory helpers - Expose AudioRecorder and related APIs to Python/SWIG - Add AudioDevices input device listing support - Register AudioRecorder in CMake and OpenShot public headers - Write recorded audio through FFmpegWriter using configured codec, sample rate, channels, layout, and bitrate - Finalize recorder writers cleanly on stop/close, including prepared recordings that never start - Update FFmpegWriter audio timestamps, packet durations, encoder flush behavior, and stream duration metadata - Fix FLAC outputs so FFmpegReader reports valid duration and video length after recording - Improve AudioWaveformer extraction for audio files with missing/unknown duration metadata - Add AudioRecorder unit tests for validation, monitoring, waveform, levels, and synthetic recording behavior - Add AudioWaveformer/FFmpegWriter regression coverage for FLAC duration and waveform extraction --- bindings/python/openshot.i | 2 + src/AudioDevices.cpp | 21 ++ src/AudioDevices.h | 3 + src/AudioRecorder.cpp | 532 +++++++++++++++++++++++++++++++++++++ src/AudioRecorder.h | 223 ++++++++++++++++ src/AudioWaveformer.cpp | 67 +++-- src/CMakeLists.txt | 1 + src/FFmpegWriter.cpp | 96 ++++++- src/OpenShot.h | 1 + tests/AudioRecorder.cpp | 150 +++++++++++ tests/AudioWaveformer.cpp | 50 ++++ tests/CMakeLists.txt | 1 + 12 files changed, 1122 insertions(+), 25 deletions(-) create mode 100644 src/AudioRecorder.cpp create mode 100644 src/AudioRecorder.h create mode 100644 tests/AudioRecorder.cpp diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index ff59079a9..57a8934ca 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -75,6 +75,7 @@ class QWidget; #include "ReaderBase.h" #include "WriterBase.h" #include "AudioDevices.h" +#include "AudioRecorder.h" #include "AudioWaveformer.h" #include "CacheBase.h" #include "CacheDisk.h" @@ -480,6 +481,7 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "ReaderBase.h" %include "WriterBase.h" %include "AudioDevices.h" +%include "AudioRecorder.h" %include "AudioWaveformer.h" %include "CacheBase.h" %include "CacheDisk.h" diff --git a/src/AudioDevices.cpp b/src/AudioDevices.cpp index 0d1f1ebb8..137de4e85 100644 --- a/src/AudioDevices.cpp +++ b/src/AudioDevices.cpp @@ -37,3 +37,24 @@ AudioDeviceList AudioDevices::getNames() { } return m_devices; } + +// Build a list of audio input devices found, and return +AudioDeviceList AudioDevices::getInputNames() { + // A temporary device manager, used to scan device names. + // Its initialize() is never called, and devices are not opened. + std::unique_ptr + manager(new juce::AudioDeviceManager()); + + m_devices.clear(); + + auto &types = manager->getAvailableDeviceTypes(); + for (auto* t : types) { + t->scanForDevices(); + const auto names = t->getDeviceNames(true); + for (const auto& name : names) { + m_devices.emplace_back( + name.toStdString(), t->getTypeName().toStdString()); + } + } + return m_devices; +} diff --git a/src/AudioDevices.h b/src/AudioDevices.h index 9fb715970..7bc28eefe 100644 --- a/src/AudioDevices.h +++ b/src/AudioDevices.h @@ -50,6 +50,9 @@ class AudioDevices /// Return a vector of std::pair<> objects holding the /// device name and type for each audio device detected AudioDeviceList getNames(); + + /// Return only audio input devices, for recording workflows + AudioDeviceList getInputNames(); private: AudioDeviceList m_devices; }; diff --git a/src/AudioRecorder.cpp b/src/AudioRecorder.cpp new file mode 100644 index 000000000..3e6cabb9e --- /dev/null +++ b/src/AudioRecorder.cpp @@ -0,0 +1,532 @@ +/** + * @file + * @brief Source file for audio recording classes + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "AudioRecorder.h" + +#include +#include +#include + +#include "Exceptions.h" +#include "FFmpegWriter.h" +#include "Frame.h" + +using namespace openshot; + +AudioLevelData AudioRecorderLevelMeter::ProcessBlock(const AudioRecorderBlock& block) const +{ + AudioLevelData result; + result.timestamp = block.sample_rate > 0 + ? static_cast(block.first_sample) / static_cast(block.sample_rate) + : 0.0; + + const int channels = static_cast(block.channels.size()); + result.peak.assign(channels, 0.0f); + result.rms.assign(channels, 0.0f); + + for (int channel = 0; channel < channels; ++channel) { + const auto& samples = block.channels[channel]; + double squared_sum = 0.0; + float peak = 0.0f; + + for (float sample : samples) { + const float abs_sample = std::abs(sample); + peak = std::max(peak, abs_sample); + squared_sum += static_cast(sample) * static_cast(sample); + if (abs_sample >= 1.0f) { + result.clipped = true; + } + } + + result.peak[channel] = peak; + if (!samples.empty()) { + result.rms[channel] = static_cast(std::sqrt(squared_sum / static_cast(samples.size()))); + } + } + + return result; +} + +AudioRecorderWaveformAccumulator::AudioRecorderWaveformAccumulator(int new_sample_rate, int new_samples_per_second) + : sample_rate(new_sample_rate) + , samples_per_second(new_samples_per_second) + , sample_divisor(1) + , pending_samples(0) + , pending_max(0.0f) + , pending_squared_sum(0.0) + , emitted_visual_samples(0) +{ + if (sample_rate <= 0 || samples_per_second <= 0) { + throw InvalidOptions("Audio waveform settings require a valid sample rate and samples-per-second value."); + } + + sample_divisor = std::max(1, sample_rate / samples_per_second); +} + +std::vector AudioRecorderWaveformAccumulator::ProcessBlock(const AudioRecorderBlock& block) +{ + std::vector chunks; + if (block.channels.empty() || block.Samples() <= 0) { + return chunks; + } + + AudioWaveformChunk chunk; + chunk.samples_per_second = samples_per_second; + chunk.start_time = static_cast(emitted_visual_samples) / static_cast(samples_per_second); + + const int channels = static_cast(block.channels.size()); + const int samples = block.Samples(); + for (int sample_index = 0; sample_index < samples; ++sample_index) { + for (int channel = 0; channel < channels; ++channel) { + if (sample_index >= static_cast(block.channels[channel].size())) { + continue; + } + const float sample = block.channels[channel][sample_index]; + pending_max = std::max(pending_max, std::abs(sample)); + pending_squared_sum += static_cast(sample) * static_cast(sample); + } + + pending_samples++; + if (pending_samples >= sample_divisor) { + const double denominator = static_cast(pending_samples * channels); + const float rms = denominator > 0.0 + ? static_cast(std::sqrt(pending_squared_sum / denominator)) + : 0.0f; + + max_samples.push_back(pending_max); + rms_samples.push_back(rms); + chunk.max_samples.push_back(pending_max); + chunk.rms_samples.push_back(rms); + emitted_visual_samples++; + + pending_samples = 0; + pending_max = 0.0f; + pending_squared_sum = 0.0; + } + } + + if (!chunk.max_samples.empty()) { + chunk.duration = static_cast(chunk.max_samples.size()) / static_cast(samples_per_second); + chunks.push_back(std::move(chunk)); + } + + return chunks; +} + +AudioWaveformData AudioRecorderWaveformAccumulator::Snapshot() const +{ + AudioWaveformData result; + result.max_samples = max_samples; + result.rms_samples = rms_samples; + return result; +} + +void AudioRecorderWaveformAccumulator::Reset() +{ + pending_samples = 0; + pending_max = 0.0f; + pending_squared_sum = 0.0; + emitted_visual_samples = 0; + max_samples.clear(); + rms_samples.clear(); +} + +std::shared_ptr AudioRecordingFrameFactory::CreateFrame( + const AudioRecorderBlock& block, + ChannelLayout channel_layout, + int64_t frame_number) +{ + const int channels = static_cast(block.channels.size()); + const int samples = block.Samples(); + auto frame = std::make_shared(frame_number, samples, channels); + frame->SampleRate(block.sample_rate); + frame->ChannelsLayout(channel_layout); + + for (int channel = 0; channel < channels; ++channel) { + frame->AddAudio(true, channel, 0, block.channels[channel].data(), samples, 1.0f); + } + + return frame; +} + +AudioRecorder::AudioRecorder(const AudioRecorderSettings& new_settings) + : settings(new_settings) + , writer(nullptr) + , waveform_accumulator(nullptr) + , is_open(false) + , is_recording(false) + , is_monitoring(false) + , writer_should_stop(false) + , samples_recorded(0) + , dropped_blocks(0) + , next_frame_number(1) +{ + ValidateSettings(); +} + +AudioRecorder::~AudioRecorder() +{ + Close(); +} + +void AudioRecorder::ValidateSettings() const +{ + if (settings.path.empty()) { + throw InvalidOptions("Audio recorder requires an output path."); + } + if (settings.codec.empty()) { + throw InvalidOptions("Audio recorder requires an audio codec."); + } + if (settings.sample_rate < 8000) { + throw InvalidSampleRate("Audio recorder requires a sample rate of at least 8000 Hz.", settings.path); + } + if (settings.channels <= 0) { + throw InvalidChannels("Audio recorder requires at least one input channel.", settings.path); + } + if (settings.buffer_size <= 0) { + throw InvalidOptions("Audio recorder requires a positive audio buffer size.", settings.path); + } + if (settings.waveform_samples_per_second <= 0) { + throw InvalidOptions("Audio recorder requires a positive waveform sample rate.", settings.path); + } + if (settings.max_queue_seconds <= 0) { + throw InvalidOptions("Audio recorder requires a positive maximum queue duration.", settings.path); + } +} + +void AudioRecorder::Open() +{ + if (is_open) { + return; + } + + if (!settings.device_type.empty()) { + device_manager.setCurrentAudioDeviceType(settings.device_type, true); + } + + juce::AudioDeviceManager::AudioDeviceSetup setup; + setup.inputChannels.clear(); + for (int channel = 0; channel < settings.channels; ++channel) { + setup.inputChannels.setBit(channel); + } + setup.outputChannels.clear(); + setup.sampleRate = settings.sample_rate; + setup.bufferSize = settings.buffer_size; + setup.inputDeviceName = settings.device_name; + + const juce::String error = device_manager.initialise( + settings.channels, + 0, + nullptr, + true, + settings.device_name, + &setup); + if (error.isNotEmpty()) { + throw InvalidOptions(error.toStdString(), settings.path); + } + + if (auto* device = device_manager.getCurrentAudioDevice()) { + const double actual_rate = device->getCurrentSampleRate(); + if (actual_rate > 0.0) { + settings.sample_rate = static_cast(std::llround(actual_rate)); + } + } + + waveform_accumulator = std::make_unique( + settings.sample_rate, + settings.waveform_samples_per_second); + + is_open = true; +} + +void AudioRecorder::OpenWriter() +{ + if (writer) { + return; + } + + writer = std::make_unique(settings.path); + writer->SetAudioOptions(true, settings.codec, settings.sample_rate, settings.channels, settings.channel_layout, settings.bit_rate); + writer->Open(); +} + +void AudioRecorder::Start() +{ + if (is_recording) { + return; + } + + PrepareRecording(); + + writer_should_stop = false; + is_recording = true; + device_manager.addAudioCallback(this); + writer_thread = std::thread(&AudioRecorder::WriterLoop, this); +} + +void AudioRecorder::PrepareRecording() +{ + if (!is_open) { + Open(); + } + StopMonitoring(); + OpenWriter(); + if (waveform_accumulator) { + waveform_accumulator->Reset(); + } + samples_recorded = 0; + dropped_blocks = 0; + next_frame_number = 1; +} + +void AudioRecorder::Stop() +{ + if (!is_recording && !writer_thread.joinable()) { + if (writer) { + writer->Close(); + writer.reset(); + } + return; + } + + is_recording = false; + device_manager.removeAudioCallback(this); + writer_should_stop = true; + queue_condition.notify_all(); + + if (writer_thread.joinable()) { + writer_thread.join(); + } + + if (writer) { + writer->Close(); + writer.reset(); + } +} + +void AudioRecorder::StartMonitoring() +{ + if (is_recording || is_monitoring) { + return; + } + + if (!is_open) { + Open(); + } + + is_monitoring = true; + device_manager.addAudioCallback(this); +} + +void AudioRecorder::StopMonitoring() +{ + if (!is_monitoring) { + return; + } + + is_monitoring = false; + device_manager.removeAudioCallback(this); +} + +void AudioRecorder::Close() +{ + StopMonitoring(); + Stop(); + + if (is_open) { + device_manager.closeAudioDevice(); + if (writer) { + writer->Close(); + writer.reset(); + } + is_open = false; + } +} + +bool AudioRecorder::IsOpen() const +{ + return is_open; +} + +bool AudioRecorder::IsRecording() const +{ + return is_recording; +} + +bool AudioRecorder::IsMonitoring() const +{ + return is_monitoring; +} + +AudioRecorderStats AudioRecorder::GetStats() const +{ + AudioRecorderStats stats; + stats.is_open = is_open; + stats.is_recording = is_recording; + stats.sample_rate = settings.sample_rate; + stats.channels = settings.channels; + stats.samples_recorded = samples_recorded; + stats.dropped_blocks = dropped_blocks; + stats.duration = settings.sample_rate > 0 + ? static_cast(stats.samples_recorded) / static_cast(settings.sample_rate) + : 0.0; + + std::lock_guard lock(queue_mutex); + stats.queued_blocks = static_cast(queue.size()); + return stats; +} + +AudioWaveformData AudioRecorder::GetWaveformSnapshot() const +{ + std::lock_guard lock(state_mutex); + return waveform_accumulator ? waveform_accumulator->Snapshot() : AudioWaveformData(); +} + +AudioLevelData AudioRecorder::GetLevelSnapshot() const +{ + std::lock_guard lock(state_mutex); + return last_level; +} + +void AudioRecorder::SetLevelCallback(std::function callback) +{ + std::lock_guard lock(state_mutex); + level_callback = std::move(callback); +} + +void AudioRecorder::SetWaveformCallback(std::function callback) +{ + std::lock_guard lock(state_mutex); + waveform_callback = std::move(callback); +} + +void AudioRecorder::audioDeviceIOCallbackWithContext( + const float* const* inputChannelData, + int numInputChannels, + float* const* outputChannelData, + int numOutputChannels, + int numSamples, + const juce::AudioIODeviceCallbackContext&) +{ + for (int channel = 0; channel < numOutputChannels; ++channel) { + if (outputChannelData[channel]) { + std::fill(outputChannelData[channel], outputChannelData[channel] + numSamples, 0.0f); + } + } + + if ((!is_recording && !is_monitoring) || numSamples <= 0) { + return; + } + + AudioRecorderBlock block; + block.sample_rate = settings.sample_rate; + block.first_sample = samples_recorded.load(); + block.channels.resize(settings.channels); + + for (int channel = 0; channel < settings.channels; ++channel) { + block.channels[channel].assign(numSamples, 0.0f); + if (channel < numInputChannels && inputChannelData[channel]) { + std::copy(inputChannelData[channel], inputChannelData[channel] + numSamples, block.channels[channel].begin()); + } + } + + AudioLevelData level = level_meter.ProcessBlock(block); + std::function level_cb; + { + std::lock_guard lock(state_mutex); + last_level = level; + level_cb = level_callback; + } + if (level_cb) { + level_cb(level); + } + + if (!is_recording) { + samples_recorded += numSamples; + return; + } + + const int64_t max_queue_blocks = static_cast( + std::max(1, (settings.max_queue_seconds * settings.sample_rate) / std::max(1, numSamples))); + { + std::lock_guard lock(queue_mutex); + if (static_cast(queue.size()) >= max_queue_blocks) { + dropped_blocks++; + } else { + queue.push_back(std::move(block)); + queue_condition.notify_one(); + } + } + + samples_recorded += numSamples; +} + +void AudioRecorder::audioDeviceAboutToStart(juce::AudioIODevice* device) +{ + (void) device; +} + +void AudioRecorder::audioDeviceStopped() +{ +} + +bool AudioRecorder::PopBlock(AudioRecorderBlock& block) +{ + std::unique_lock lock(queue_mutex); + queue_condition.wait(lock, [this]() { + return writer_should_stop || !queue.empty(); + }); + + if (queue.empty()) { + return false; + } + + block = std::move(queue.front()); + queue.pop_front(); + return true; +} + +void AudioRecorder::WriterLoop() +{ + while (true) { + AudioRecorderBlock block; + if (!PopBlock(block)) { + if (writer_should_stop) { + break; + } + continue; + } + + std::vector waveform_chunks; + std::function waveform_cb; + { + std::lock_guard lock(state_mutex); + if (waveform_accumulator) { + waveform_chunks = waveform_accumulator->ProcessBlock(block); + } + waveform_cb = waveform_callback; + } + + if (waveform_cb) { + for (const auto& chunk : waveform_chunks) { + waveform_cb(chunk); + } + } + + if (writer) { + writer->WriteFrame(AudioRecordingFrameFactory::CreateFrame( + block, + settings.channel_layout, + next_frame_number++)); + } + } +} diff --git a/src/AudioRecorder.h b/src/AudioRecorder.h new file mode 100644 index 000000000..9521d5a66 --- /dev/null +++ b/src/AudioRecorder.h @@ -0,0 +1,223 @@ +/** + * @file + * @brief Header file for audio recording classes + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_AUDIORECORDER_H +#define OPENSHOT_AUDIORECORDER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "AudioWaveformer.h" +#include "ChannelLayouts.h" + +namespace openshot +{ + class FFmpegWriter; + class Frame; + + struct AudioRecorderSettings + { + std::string path; + std::string device_name; + std::string device_type; + std::string codec = "pcm_s16le"; + int sample_rate = 48000; + int channels = 1; + openshot::ChannelLayout channel_layout = openshot::LAYOUT_MONO; + int bit_rate = 192000; + int buffer_size = 512; + int waveform_samples_per_second = 30; + int max_queue_seconds = 10; + std::map options; + }; + + struct AudioLevelData + { + double timestamp = 0.0; + std::vector peak; + std::vector rms; + bool clipped = false; + + std::vector> vectors() const + { + std::vector> output; + output.push_back(peak); + output.push_back(rms); + return output; + } + }; + + struct AudioWaveformChunk + { + double start_time = 0.0; + double duration = 0.0; + int samples_per_second = 0; + std::vector max_samples; + std::vector rms_samples; + + std::vector> vectors() const + { + std::vector> output; + output.push_back(max_samples); + output.push_back(rms_samples); + return output; + } + }; + + struct AudioRecorderStats + { + bool is_open = false; + bool is_recording = false; + int sample_rate = 0; + int channels = 0; + int64_t samples_recorded = 0; + int64_t dropped_blocks = 0; + int64_t queued_blocks = 0; + double duration = 0.0; + }; + + struct AudioRecorderBlock + { + int sample_rate = 0; + int64_t first_sample = 0; + std::vector> channels; + + int Samples() const + { + return channels.empty() ? 0 : static_cast(channels.front().size()); + } + }; + + class AudioRecorderLevelMeter + { + public: + AudioLevelData ProcessBlock(const AudioRecorderBlock& block) const; + }; + + class AudioRecorderWaveformAccumulator + { + public: + AudioRecorderWaveformAccumulator(int sample_rate, int samples_per_second); + + std::vector ProcessBlock(const AudioRecorderBlock& block); + openshot::AudioWaveformData Snapshot() const; + void Reset(); + + private: + int sample_rate; + int samples_per_second; + int sample_divisor; + int pending_samples; + float pending_max; + double pending_squared_sum; + int64_t emitted_visual_samples; + std::vector max_samples; + std::vector rms_samples; + }; + + class AudioRecordingFrameFactory + { + public: + static std::shared_ptr CreateFrame( + const AudioRecorderBlock& block, + openshot::ChannelLayout channel_layout, + int64_t frame_number); + }; + + class AudioRecorder +#ifndef SWIG + : private juce::AudioIODeviceCallback +#endif + { + public: + explicit AudioRecorder(const AudioRecorderSettings& settings); + ~AudioRecorder() +#ifndef SWIG + override +#endif + ; + + void Open(); + void PrepareRecording(); + void Start(); + void Stop(); + void StartMonitoring(); + void StopMonitoring(); + void Close(); + + bool IsOpen() const; + bool IsRecording() const; + bool IsMonitoring() const; + AudioRecorderStats GetStats() const; + openshot::AudioWaveformData GetWaveformSnapshot() const; + AudioLevelData GetLevelSnapshot() const; + +#ifndef SWIG + void SetLevelCallback(std::function callback); + void SetWaveformCallback(std::function callback); + void audioDeviceIOCallbackWithContext( + const float* const* inputChannelData, + int numInputChannels, + float* const* outputChannelData, + int numOutputChannels, + int numSamples, + const juce::AudioIODeviceCallbackContext& context) override; + void audioDeviceAboutToStart(juce::AudioIODevice* device) override; + void audioDeviceStopped() override; +#endif + + private: + void ValidateSettings() const; + void OpenWriter(); + void WriterLoop(); + bool PopBlock(AudioRecorderBlock& block); + + AudioRecorderSettings settings; + juce::AudioDeviceManager device_manager; + std::unique_ptr writer; + std::unique_ptr waveform_accumulator; + AudioRecorderLevelMeter level_meter; + + mutable std::mutex state_mutex; + mutable std::mutex queue_mutex; + std::condition_variable queue_condition; + std::deque queue; + std::thread writer_thread; + + std::function level_callback; + std::function waveform_callback; + AudioLevelData last_level; + + std::atomic is_open; + std::atomic is_recording; + std::atomic is_monitoring; + std::atomic writer_should_stop; + std::atomic samples_recorded; + std::atomic dropped_blocks; + int64_t next_frame_number; + }; +} + +#endif diff --git a/src/AudioWaveformer.cpp b/src/AudioWaveformer.cpp index 424dc2fa7..c50135a21 100644 --- a/src/AudioWaveformer.cpp +++ b/src/AudioWaveformer.cpp @@ -276,8 +276,9 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r return data; } - int total_samples = static_cast(std::ceil(reader_duration * num_per_second)); - if (total_samples <= 0 || source_reader->info.channels == 0) { + const bool known_duration = reader_duration > 0.0f && reader_video_length > 0; + int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0; + if (source_reader->info.channels == 0) { return data; } @@ -285,9 +286,16 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r return data; } - // Resize and clear audio buffers - data.resize(total_samples); - data.zero(total_samples); + // Resize and clear audio buffers when duration is known. Some audio-only + // formats, especially raw FLAC, may not expose a container duration even + // though frames are readable, so those are appended dynamically below. + if (known_duration) { + data.resize(total_samples); + data.zero(total_samples); + } else { + data.max_samples.reserve(num_per_second); + data.rms_samples.reserve(num_per_second); + } int extracted_index = 0; int sample_index = 0; @@ -297,10 +305,38 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r int channel_count = (channel == -1) ? source_reader->info.channels : 1; std::vector channels(source_reader->info.channels, nullptr); + const double frame_rate = fps_value > 0.0 ? fps_value : 30.0; + const int64_t max_unknown_frames = static_cast(std::ceil(frame_rate * 60.0 * 60.0 * 12.0)); + int empty_audio_frames = 0; + + auto append_sample = [&](float max_sample, float rms_sample) { + if (known_duration) { + if (extracted_index >= total_samples) { + return; + } + data.max_samples[extracted_index] = max_sample; + data.rms_samples[extracted_index] = rms_sample; + } else { + data.max_samples.push_back(max_sample); + data.rms_samples.push_back(rms_sample); + } + samples_max = std::max(samples_max, max_sample); + extracted_index++; + }; try { - for (int64_t f = 1; f <= reader_video_length && extracted_index < total_samples; f++) { + for (int64_t f = 1; + (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames); + f++) { std::shared_ptr frame = get_frame_with_retry(f); + int sample_count = frame->GetAudioSamplesCount(); + if (sample_count <= 0) { + if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) { + break; + } + continue; + } + empty_audio_frames = 0; for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { if (channel == channel_index || channel == -1) { @@ -308,7 +344,6 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r } } - int sample_count = frame->GetAudioSamplesCount(); for (int s = 0; s < sample_count; s++) { for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { if (channel == channel_index || channel == -1) { @@ -330,18 +365,15 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_divisor * channel_count)); } - if (extracted_index < total_samples) { - data.max_samples[extracted_index] = chunk_max; - data.rms_samples[extracted_index] = std::sqrt(avg_squared_sum); - samples_max = std::max(samples_max, chunk_max); - extracted_index++; + if (!known_duration || extracted_index < total_samples) { + append_sample(chunk_max, std::sqrt(avg_squared_sum)); } sample_index = 0; chunk_max = 0.0f; chunk_squared_sum = 0.0; - if (extracted_index >= total_samples) { + if (known_duration && extracted_index >= total_samples) { break; } } @@ -351,21 +383,18 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r throw; } - if (sample_index > 0 && extracted_index < total_samples) { + if (sample_index > 0 && (!known_duration || extracted_index < total_samples)) { float avg_squared_sum = 0.0f; if (channel_count > 0) { avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_index * channel_count)); } - data.max_samples[extracted_index] = chunk_max; - data.rms_samples[extracted_index] = std::sqrt(avg_squared_sum); - samples_max = std::max(samples_max, chunk_max); - extracted_index++; + append_sample(chunk_max, std::sqrt(avg_squared_sum)); } if (normalize && samples_max > 0.0f) { float scale = 1.0f / samples_max; - data.scale(total_samples, scale); + data.scale(static_cast(data.max_samples.size()), scale); } return data; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9c09c088..69a11866b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,6 +47,7 @@ add_feature_info("IWYU (include-what-you-use)" ENABLE_IWYU "Scan all source file # Main library sources set(OPENSHOT_SOURCES AudioBufferSource.cpp + AudioRecorder.cpp AnimatedCurve.cpp AudioDevices.cpp AudioReaderSource.cpp diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index c183b86fc..a2667fb04 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -754,6 +754,13 @@ void FFmpegWriter::WriteTrailer() { // Flush encoders (who sometimes hold on to frames) flush_encoders(); + if (info.has_audio && audio_st && audio_codec_ctx && audio_timestamp > 0) { + audio_st->duration = av_rescale_q( + audio_timestamp, + audio_codec_ctx->time_base, + audio_st->time_base); + } + /* write the trailer, if any. The trailer must be written * before you close the CodecContexts open when you wrote the * header; otherwise write_trailer may try to use memory that @@ -870,10 +877,61 @@ void FFmpegWriter::flush_encoders() { int error_code = 0; int got_packet = 0; #if IS_FFMPEG_3_2 + if (!audio_codec_ctx->codec || !(audio_codec_ctx->codec->capabilities & AV_CODEC_CAP_DELAY)) { + av_packet_free(&pkt); + break; + } error_code = avcodec_send_frame(audio_codec_ctx, NULL); + if (error_code < 0 && error_code != AVERROR_EOF) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegWriter::flush_encoders ERROR [" + + av_err2string(error_code) + "]", + "error_code", error_code); + } + while (true) { + error_code = avcodec_receive_packet(audio_codec_ctx, pkt); + if (error_code == AVERROR(EAGAIN) || error_code == AVERROR_EOF) { + got_packet = 0; + break; + } + if (error_code < 0) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegWriter::flush_encoders ERROR [" + + av_err2string(error_code) + "]", + "error_code", error_code); + got_packet = 0; + break; + } + + got_packet = 1; + if (pkt->pts == AV_NOPTS_VALUE) { + pkt->pts = audio_timestamp; + } + if (pkt->dts == AV_NOPTS_VALUE) { + pkt->dts = pkt->pts; + } + if (pkt->duration <= 0) { + pkt->duration = audio_codec_ctx->frame_size > 0 ? audio_codec_ctx->frame_size : audio_input_frame_size; + } + const int64_t packet_duration = pkt->duration; + av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base); + pkt->stream_index = audio_st->index; + pkt->flags |= AV_PKT_FLAG_KEY; + + error_code = av_interleaved_write_frame(oc, pkt); + if (error_code < 0) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegWriter::flush_encoders ERROR [" + + av_err2string(error_code) + "]", + "error_code", error_code); + } + audio_timestamp += packet_duration; + AV_FREE_PACKET(pkt); + } + av_packet_free(&pkt); + break; #else error_code = avcodec_encode_audio2(audio_codec_ctx, pkt, NULL, &got_packet); -#endif if (error_code < 0) { ZmqLogger::Instance()->AppendDebugMethod( "FFmpegWriter::flush_encoders ERROR [" @@ -887,6 +945,9 @@ void FFmpegWriter::flush_encoders() { // Since the PTS can change during encoding, set the value again. This seems like a huge hack, // but it fixes lots of PTS related issues when I do this. pkt->pts = pkt->dts = audio_timestamp; + if (pkt->duration <= 0) { + pkt->duration = audio_codec_ctx->frame_size > 0 ? audio_codec_ctx->frame_size : audio_input_frame_size; + } // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase) av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base); @@ -909,6 +970,7 @@ void FFmpegWriter::flush_encoders() { // deallocate memory for packet AV_FREE_PACKET(pkt); +#endif } } @@ -1062,6 +1124,9 @@ AVStream *FFmpegWriter::add_audio_stream() { // Set sample rate c->sample_rate = info.sample_rate; + c->time_base = AVRational{1, c->sample_rate}; + st->time_base = c->time_base; + uint64_t channel_layout = info.channel_layout; #if HAVE_CH_LAYOUT // Set a valid number of channels (or throw error) @@ -1782,6 +1847,13 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr 0 || is_final) { // Get remaining samples needed for this packet @@ -1821,6 +1893,7 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptrsample_fmt)) { ZmqLogger::Instance()->AppendDebugMethod( "FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", @@ -1878,7 +1951,7 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptrnb_samples = audio_input_frame_size; + frame_final->nb_samples = frame_nb_samples; #if HAVE_CH_LAYOUT av_channel_layout_from_mask(&frame_final->ch_layout, info.channel_layout); #else @@ -1931,7 +2004,14 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptrsample_fmt)); // Init the nb_samples property - frame_final->nb_samples = audio_input_frame_size; + frame_final->nb_samples = frame_nb_samples; + frame_final->format = audio_codec_ctx->sample_fmt; +#if HAVE_CH_LAYOUT + av_channel_layout_copy(&frame_final->ch_layout, &audio_codec_ctx->ch_layout); +#else + frame_final->channels = audio_codec_ctx->channels; + frame_final->channel_layout = audio_codec_ctx->channel_layout; +#endif // Fill the final_frame AVFrame with audio (non planar) #if HAVE_CH_LAYOUT @@ -1953,9 +2033,9 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptrdata = audio_encoder_buffer; pkt->size = audio_encoder_buffer_size; +#endif // Set the packet's PTS prior to encoding pkt->pts = pkt->dts = audio_timestamp; @@ -1969,7 +2049,9 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptrcodec + && (audio_codec_ctx->codec->capabilities & AV_CODEC_CAP_DELAY)) { avcodec_send_frame(audio_codec_ctx, NULL); } else { @@ -1979,7 +2061,6 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr= 0) frame_finished = 1; if(ret == AVERROR(EINVAL) || ret == AVERROR_EOF) { - avcodec_flush_buffers(audio_codec_ctx); ret = 0; } if (ret >= 0) { @@ -2001,6 +2082,9 @@ void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptrpts = pkt->dts = audio_timestamp; + if (pkt->duration <= 0) { + pkt->duration = frame_nb_samples; + } // Scale the PTS to the audio stream timebase (which is sometimes different than the codec's timebase) av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_st->time_base); diff --git a/src/OpenShot.h b/src/OpenShot.h index dc97b88af..91b8e287d 100644 --- a/src/OpenShot.h +++ b/src/OpenShot.h @@ -106,6 +106,7 @@ #include "AudioBufferSource.h" #include "AudioLocation.h" #include "AudioReaderSource.h" +#include "AudioRecorder.h" #include "AudioResampler.h" #include "CacheDisk.h" #include "CacheMemory.h" diff --git a/tests/AudioRecorder.cpp b/tests/AudioRecorder.cpp new file mode 100644 index 000000000..4cb254e9f --- /dev/null +++ b/tests/AudioRecorder.cpp @@ -0,0 +1,150 @@ +/** + * @file + * @brief Unit tests for openshot::AudioRecorder helper classes + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include + +#include "openshot_catch.h" + +#include "AudioRecorder.h" +#include "Exceptions.h" +#include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include "Frame.h" + +using namespace openshot; + +namespace { +AudioRecorderBlock MakeBlock(int sample_rate, int64_t first_sample, std::vector left, std::vector right = {}) +{ + AudioRecorderBlock block; + block.sample_rate = sample_rate; + block.first_sample = first_sample; + block.channels.push_back(std::move(left)); + if (!right.empty()) { + block.channels.push_back(std::move(right)); + } + return block; +} +} + +TEST_CASE("Audio recorder settings validation", "[libopenshot][audiorecorder]") +{ + AudioRecorderSettings settings; + settings.path = "settings-validation.wav"; + CHECK_NOTHROW([&settings]() { AudioRecorder recorder(settings); }()); + + settings.path = ""; + CHECK_THROWS_AS([&settings]() { AudioRecorder recorder(settings); }(), InvalidOptions); + + settings.path = "settings-validation.wav"; + settings.channels = 0; + CHECK_THROWS_AS([&settings]() { AudioRecorder recorder(settings); }(), InvalidChannels); + + settings.channels = 1; + settings.sample_rate = 7999; + CHECK_THROWS_AS([&settings]() { AudioRecorder recorder(settings); }(), InvalidSampleRate); +} + +TEST_CASE("Audio recorder level meter calculates peak and RMS", "[libopenshot][audiorecorder]") +{ + AudioRecorderLevelMeter meter; + auto level = meter.ProcessBlock(MakeBlock( + 48000, + 24000, + {0.0f, -0.5f, 0.25f, 1.0f}, + {0.0f, 0.25f, -0.25f, 0.5f})); + + REQUIRE(level.peak.size() == 2); + REQUIRE(level.rms.size() == 2); + CHECK(level.timestamp == Approx(0.5)); + CHECK(level.peak[0] == Approx(1.0f)); + CHECK(level.peak[1] == Approx(0.5f)); + CHECK(level.rms[0] == Approx(std::sqrt((0.0 + 0.25 + 0.0625 + 1.0) / 4.0))); + CHECK(level.rms[1] == Approx(std::sqrt((0.0 + 0.0625 + 0.0625 + 0.25) / 4.0))); + CHECK(level.clipped); +} + +TEST_CASE("Audio recorder waveform accumulator downsamples max and RMS", "[libopenshot][audiorecorder]") +{ + AudioRecorderWaveformAccumulator accumulator(8, 2); + auto chunks = accumulator.ProcessBlock(MakeBlock( + 8, + 0, + {0.0f, 0.5f, -1.0f, 0.25f, 0.25f, -0.25f, 0.75f, -0.5f})); + + REQUIRE(chunks.size() == 1); + CHECK(chunks[0].samples_per_second == 2); + CHECK(chunks[0].start_time == Approx(0.0)); + CHECK(chunks[0].duration == Approx(1.0)); + REQUIRE(chunks[0].max_samples.size() == 2); + REQUIRE(chunks[0].rms_samples.size() == 2); + CHECK(chunks[0].max_samples[0] == Approx(1.0f)); + CHECK(chunks[0].max_samples[1] == Approx(0.75f)); + CHECK(chunks[0].rms_samples[0] == Approx(std::sqrt((0.0 + 0.25 + 1.0 + 0.0625) / 4.0))); + CHECK(chunks[0].rms_samples[1] == Approx(std::sqrt((0.0625 + 0.0625 + 0.5625 + 0.25) / 4.0))); + + auto snapshot = accumulator.Snapshot(); + CHECK(snapshot.max_samples == chunks[0].max_samples); + CHECK(snapshot.rms_samples == chunks[0].rms_samples); +} + +TEST_CASE("Audio recording frame factory creates audio-only frames", "[libopenshot][audiorecorder]") +{ + auto block = MakeBlock( + 44100, + 0, + {0.1f, 0.2f, 0.3f}, + {-0.1f, -0.2f, -0.3f}); + + auto frame = AudioRecordingFrameFactory::CreateFrame(block, LAYOUT_STEREO, 42); + REQUIRE(frame); + CHECK(frame->number == 42); + CHECK(frame->SampleRate() == 44100); + CHECK(frame->ChannelsLayout() == LAYOUT_STEREO); + CHECK(frame->GetAudioChannelsCount() == 2); + CHECK(frame->GetAudioSamplesCount() == 3); + CHECK(frame->GetAudioSamples(0)[1] == Approx(0.2f)); + CHECK(frame->GetAudioSamples(1)[2] == Approx(-0.3f)); +} + +TEST_CASE("Audio recorder frames write through FFmpegWriter", "[libopenshot][audiorecorder][ffmpegwriter]") +{ + const std::string path = "AudioRecorder-output.wav"; + std::remove(path.c_str()); + + const int sample_rate = 8000; + const int samples = sample_rate / 10; + std::vector tone(samples); + for (int i = 0; i < samples; ++i) { + tone[i] = 0.25f * static_cast(std::sin(2.0 * M_PI * 440.0 * static_cast(i) / sample_rate)); + } + + FFmpegWriter writer(path); + writer.SetAudioOptions(true, "pcm_s16le", sample_rate, 1, LAYOUT_MONO, 128000); + writer.Open(); + writer.WriteFrame(AudioRecordingFrameFactory::CreateFrame(MakeBlock(sample_rate, 0, tone), LAYOUT_MONO, 1)); + writer.Close(); + + FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.has_audio); + CHECK(reader.info.sample_rate == sample_rate); + CHECK(reader.info.channels == 1); + CHECK(reader.info.duration > 0.05f); + auto frame = reader.GetFrame(1); + CHECK(frame->GetAudioChannelsCount() == 1); + CHECK(frame->GetAudioSamplesCount() > 0); + reader.Close(); +} diff --git a/tests/AudioWaveformer.cpp b/tests/AudioWaveformer.cpp index 1be2d5d30..2ceac1ee4 100644 --- a/tests/AudioWaveformer.cpp +++ b/tests/AudioWaveformer.cpp @@ -15,6 +15,8 @@ #include "Clip.h" #include "Exceptions.h" #include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include "Frame.h" #include "Timeline.h" #include @@ -27,6 +29,17 @@ using namespace openshot; +namespace { +std::shared_ptr MakeWaveformFrame(int frame_number, int sample_rate, const std::vector& samples) +{ + auto frame = std::make_shared(frame_number, static_cast(samples.size()), 1); + frame->SampleRate(sample_rate); + frame->ChannelsLayout(LAYOUT_MONO); + frame->AddAudio(true, 0, 0, samples.data(), static_cast(samples.size()), 1.0f); + return frame; +} +} + TEST_CASE( "Extract waveform data piano.wav", "[libopenshot][audiowaveformer]" ) { // Create a reader @@ -139,6 +152,43 @@ TEST_CASE( "Channel selection returns data and rejects invalid channel", "[libop r.Close(); } +TEST_CASE( "FFmpegWriter FLAC includes duration metadata for waveform extraction", "[libopenshot][ffmpegwriter][audiowaveformer][flac]" ) +{ + const std::string path = "AudioWaveformer-duration-unknown.flac"; + std::remove(path.c_str()); + + const int sample_rate = 48000; + const int samples = sample_rate / 2; + std::vector tone(samples); + for (int i = 0; i < samples; ++i) { + tone[i] = 0.25f * static_cast(std::sin(2.0 * M_PI * 440.0 * static_cast(i) / sample_rate)); + } + + FFmpegWriter writer(path); + writer.SetAudioOptions(true, "flac", sample_rate, 1, LAYOUT_MONO, 192000); + writer.Open(); + writer.WriteFrame(MakeWaveformFrame(1, sample_rate, tone)); + writer.WriteFrame(MakeWaveformFrame(2, sample_rate, tone)); + writer.Close(); + + FFmpegReader reader(path); + reader.Open(); + REQUIRE(reader.info.has_audio); + CHECK(reader.info.duration > 0.0f); + CHECK(reader.info.video_length > 0); + + AudioWaveformer waveformer(&reader); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, 20, true); + + CHECK_FALSE(waveform.max_samples.empty()); + CHECK(waveform.max_samples.size() == waveform.rms_samples.size()); + CHECK(*std::max_element(waveform.max_samples.begin(), waveform.max_samples.end()) > 0.0f); + CHECK(*std::max_element(waveform.max_samples.begin(), waveform.max_samples.end()) <= Approx(1.0f).margin(0.0001f)); + + reader.Close(); + std::remove(path.c_str()); +} + TEST_CASE( "Waveform extraction does not mutate source reader video flag", "[libopenshot][audiowaveformer][mutation]" ) { std::stringstream path; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4f915b307..95ef5a422 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries(openshot-benchmark openshot) ### set(OPENSHOT_TESTS AudioDeviceManager + AudioRecorder AnimatedCurve AudioWaveformer CacheDisk From 3e9de94ecc34d3be45640aa5cd67eb883405bbb4 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 29 May 2026 10:03:43 -0500 Subject: [PATCH 02/30] Reverting changes to python detection in libopenshot, so we don't break our Mac builder by including the wrong version of python at this moment. We'll revisit in the future. --- bindings/python/CMakeLists.txt | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index f3d05cb09..d748f252d 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -20,22 +20,13 @@ if (POLICY CMP0086) cmake_policy(SET CMP0086 NEW) endif() -if (CMAKE_VERSION VERSION_LESS 3.12) - find_package(PythonInterp 3) - find_package(PythonLibs 3) -else() - find_package(Python3 3 COMPONENTS Interpreter Development) - if (Python3_FOUND) - set(PYTHON_EXECUTABLE "${Python3_EXECUTABLE}") - set(PYTHON_INCLUDE_PATH "${Python3_INCLUDE_DIRS}") - set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") - set(PYTHON_VERSION_MAJOR "${Python3_VERSION_MAJOR}") - set(PYTHON_VERSION_MINOR "${Python3_VERSION_MINOR}") - set(PYTHONINTERP_FOUND TRUE) - set(PYTHONLIBS_FOUND TRUE) - endif() +if (POLICY CMP0148) + cmake_policy(SET CMP0148 OLD) endif() +find_package(PythonInterp 3) +find_package(PythonLibs 3) + if (NOT PYTHONLIBS_FOUND OR NOT PYTHONINTERP_FOUND) return() endif() From 8bae3afb627f0e86faef38bc87dfbc8c6cb66749 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 15 Jun 2026 23:09:52 -0500 Subject: [PATCH 03/30] Add Linux screen and camera capture readers Introduce FFmpeg-backed live capture readers for X11 screen capture and v4l2 camera devices, including JSON/settings support, Python bindings, public OpenShot includes, and focused metadata validation tests. --- bindings/python/openshot.i | 4 + src/CMakeLists.txt | 6 +- src/CameraCaptureReader.cpp | 177 ++++++++++++++ src/CameraCaptureReader.h | 72 ++++++ src/OpenShot.h | 2 + src/ScreenCaptureReader.cpp | 444 ++++++++++++++++++++++++++++++++++ src/ScreenCaptureReader.h | 102 ++++++++ tests/CMakeLists.txt | 2 + tests/CameraCaptureReader.cpp | 69 ++++++ tests/ScreenCaptureReader.cpp | 78 ++++++ 10 files changed, 954 insertions(+), 2 deletions(-) create mode 100644 src/CameraCaptureReader.cpp create mode 100644 src/CameraCaptureReader.h create mode 100644 src/ScreenCaptureReader.cpp create mode 100644 src/ScreenCaptureReader.h create mode 100644 tests/CameraCaptureReader.cpp create mode 100644 tests/ScreenCaptureReader.cpp diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index 57a8934ca..f90ec070c 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -77,6 +77,7 @@ class QWidget; #include "AudioDevices.h" #include "AudioRecorder.h" #include "AudioWaveformer.h" +#include "CameraCaptureReader.h" #include "CacheBase.h" #include "CacheDisk.h" #include "CacheMemory.h" @@ -102,6 +103,7 @@ class QWidget; #include "PlayerBase.h" #include "Point.h" #include "Profiles.h" +#include "ScreenCaptureReader.h" #include "QtHtmlReader.h" #include "QtImageReader.h" #include "QtPlayer.h" @@ -483,6 +485,7 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "AudioDevices.h" %include "AudioRecorder.h" %include "AudioWaveformer.h" +%include "CameraCaptureReader.h" %include "CacheBase.h" %include "CacheDisk.h" %include "CacheMemory.h" @@ -508,6 +511,7 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "PlayerBase.h" %include "Point.h" %include "Profiles.h" +%include "ScreenCaptureReader.h" %include "QtHtmlReader.h" %include "QtImageReader.h" %include "QtPlayer.h" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 69a11866b..6d900a559 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -53,6 +53,7 @@ set(OPENSHOT_SOURCES AudioReaderSource.cpp AudioResampler.cpp AudioWaveformer.cpp + CameraCaptureReader.cpp CacheBase.cpp CacheDisk.cpp CacheMemory.cpp @@ -66,6 +67,7 @@ set(OPENSHOT_SOURCES DummyReader.cpp ReaderBase.cpp RendererBase.cpp + ScreenCaptureReader.cpp WriterBase.cpp EffectBase.cpp EffectInfo.cpp @@ -393,11 +395,11 @@ mark_as_advanced(QT_VERSION_STR) ################### FFMPEG ##################### # Find FFmpeg libraries (used for video encoding / decoding) find_package(FFmpeg REQUIRED - COMPONENTS avcodec avformat avutil swscale + COMPONENTS avcodec avdevice avformat avutil swscale OPTIONAL_COMPONENTS swresample ) -set(all_comps avcodec avformat avutil swscale) +set(all_comps avcodec avdevice avformat avutil swscale) set(version_comps avcodec avformat avutil) # Pick a resampler. Prefer swresample if possible diff --git a/src/CameraCaptureReader.cpp b/src/CameraCaptureReader.cpp new file mode 100644 index 000000000..b2781f516 --- /dev/null +++ b/src/CameraCaptureReader.cpp @@ -0,0 +1,177 @@ +/** + * @file + * @brief Source file for live FFmpeg camera capture readers + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "CameraCaptureReader.h" + +#include + +#include "Exceptions.h" + +using namespace openshot; + +CameraCaptureReader::CameraCaptureReader(const CameraCaptureSettings& new_settings) + : settings(new_settings) + , reader(nullptr) +{ + if (settings.backend == CAMERA_CAPTURE_AUTO) { + settings.backend = DefaultBackend(); + } + ValidateSettings(); + RebuildReader(); +} + +CameraCaptureReader::~CameraCaptureReader() +{ + Close(); +} + +bool CameraCaptureReader::IsBackendSupported(CameraCaptureBackend backend) +{ +#if defined(__linux__) + return backend == CAMERA_CAPTURE_V4L2 || backend == CAMERA_CAPTURE_AUTO; +#else + (void) backend; + return false; +#endif +} + +CameraCaptureBackend CameraCaptureReader::DefaultBackend() +{ +#if defined(__linux__) + return CAMERA_CAPTURE_V4L2; +#else + return CAMERA_CAPTURE_AUTO; +#endif +} + +void CameraCaptureReader::ValidateSettings() const +{ + if (!IsBackendSupported(settings.backend)) { + throw InvalidOptions("Camera capture backend is not supported on this OS."); + } + if (settings.backend != CAMERA_CAPTURE_V4L2) { + throw InvalidOptions("Only the v4l2 camera capture backend is implemented in this build."); + } + if (settings.device.empty()) { + throw InvalidOptions("Camera capture requires a device path."); + } + if (settings.width <= 0 || settings.height <= 0) { + throw InvalidOptions("Camera capture requires a positive width and height."); + } + if (settings.fps.num <= 0 || settings.fps.den <= 0) { + throw InvalidOptions("Camera capture requires a positive frame rate."); + } +} + +ScreenCaptureSettings CameraCaptureReader::ToDeviceSettings() const +{ + ScreenCaptureSettings converted; + converted.backend = SCREEN_CAPTURE_X11; + converted.display = settings.device; + converted.width = settings.width; + converted.height = settings.height; + converted.fps = settings.fps; + converted.options = settings.options; + converted.options["input_format_name"] = "v4l2"; + return converted; +} + +void CameraCaptureReader::RebuildReader() +{ + reader = std::make_unique(ToDeviceSettings()); + info = reader->info; + info.metadata["capture_type"] = "camera"; +} + +void CameraCaptureReader::Open() +{ + reader->Open(); + info = reader->info; + info.metadata["capture_type"] = "camera"; +} + +void CameraCaptureReader::Close() +{ + if (reader) { + reader->Close(); + } +} + +openshot::CaptureReaderStats CameraCaptureReader::GetStats() const +{ + return reader ? reader->GetStats() : CaptureReaderStats(); +} + +std::shared_ptr CameraCaptureReader::GetFrame(int64_t number) +{ + return reader->GetFrame(number); +} + +std::string CameraCaptureReader::Json() const +{ + return JsonValue().toStyledString(); +} + +Json::Value CameraCaptureReader::JsonValue() const +{ + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "CameraCaptureReader"; + root["backend"] = settings.backend; + root["device"] = settings.device; + root["width"] = settings.width; + root["height"] = settings.height; + root["fps"]["num"] = settings.fps.num; + root["fps"]["den"] = settings.fps.den; + root["options"] = Json::Value(Json::objectValue); + for (const auto& option : settings.options) { + root["options"][option.first] = option.second; + } + return root; +} + +void CameraCaptureReader::SetJson(const std::string value) +{ + try { + SetJsonValue(openshot::stringToJson(value)); + } catch (const std::exception&) { + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +void CameraCaptureReader::SetJsonValue(const Json::Value root) +{ + if (!root["backend"].isNull()) + settings.backend = static_cast(root["backend"].asInt()); + if (!root["device"].isNull()) + settings.device = root["device"].asString(); + if (!root["width"].isNull()) + settings.width = root["width"].asInt(); + if (!root["height"].isNull()) + settings.height = root["height"].asInt(); + if (!root["fps"].isNull() && root["fps"].isObject()) { + if (!root["fps"]["num"].isNull()) + settings.fps.num = root["fps"]["num"].asInt(); + if (!root["fps"]["den"].isNull()) + settings.fps.den = root["fps"]["den"].asInt(); + } + if (!root["options"].isNull() && root["options"].isObject()) { + settings.options.clear(); + for (const auto& key : root["options"].getMemberNames()) { + settings.options[key] = root["options"][key].asString(); + } + } + if (settings.backend == CAMERA_CAPTURE_AUTO) { + settings.backend = DefaultBackend(); + } + ValidateSettings(); + RebuildReader(); +} diff --git a/src/CameraCaptureReader.h b/src/CameraCaptureReader.h new file mode 100644 index 000000000..638ec7fde --- /dev/null +++ b/src/CameraCaptureReader.h @@ -0,0 +1,72 @@ +/** + * @file + * @brief Header file for live camera capture readers + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_CAMERACAPTUREREADER_H +#define OPENSHOT_CAMERACAPTUREREADER_H + +#include "ScreenCaptureReader.h" + +#include + +namespace openshot +{ + enum CameraCaptureBackend + { + CAMERA_CAPTURE_AUTO = 0, + CAMERA_CAPTURE_V4L2 = 1, + CAMERA_CAPTURE_WINDOWS_DSHOW = 2, + CAMERA_CAPTURE_MAC_AVFOUNDATION = 3 + }; + + struct CameraCaptureSettings + { + CameraCaptureBackend backend = CAMERA_CAPTURE_AUTO; + std::string device = "/dev/video0"; + int width = 1280; + int height = 720; + openshot::Fraction fps = openshot::Fraction(30, 1); + std::map options; + }; + + class CameraCaptureReader : public ReaderBase + { + public: + explicit CameraCaptureReader(const CameraCaptureSettings& settings); + ~CameraCaptureReader() override; + + void Close() override; + CacheBase* GetCache() override { return NULL; }; + std::shared_ptr GetFrame(int64_t number) override; + bool IsOpen() override { return reader && reader->IsOpen(); }; + std::string Name() override { return "CameraCaptureReader"; }; + std::string Json() const override; + void SetJson(const std::string value) override; + Json::Value JsonValue() const override; + void SetJsonValue(const Json::Value root) override; + void Open() override; + + openshot::CaptureReaderStats GetStats() const; + CameraCaptureSettings GetSettings() const { return settings; }; + static bool IsBackendSupported(CameraCaptureBackend backend); + static CameraCaptureBackend DefaultBackend(); + + private: + void ValidateSettings() const; + ScreenCaptureSettings ToDeviceSettings() const; + void RebuildReader(); + + CameraCaptureSettings settings; + std::unique_ptr reader; + }; +} + +#endif diff --git a/src/OpenShot.h b/src/OpenShot.h index 91b8e287d..1a753377e 100644 --- a/src/OpenShot.h +++ b/src/OpenShot.h @@ -108,6 +108,7 @@ #include "AudioReaderSource.h" #include "AudioRecorder.h" #include "AudioResampler.h" +#include "CameraCaptureReader.h" #include "CacheDisk.h" #include "CacheMemory.h" #include "ChunkReader.h" @@ -122,6 +123,7 @@ #include "Enums.h" #include "Exceptions.h" #include "ReaderBase.h" +#include "ScreenCaptureReader.h" #include "WriterBase.h" #include "FFmpegReader.h" #include "FFmpegWriter.h" diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp new file mode 100644 index 000000000..1028d280b --- /dev/null +++ b/src/ScreenCaptureReader.cpp @@ -0,0 +1,444 @@ +/** + * @file + * @brief Source file for live FFmpeg screen capture readers + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "ScreenCaptureReader.h" + +#include +#include +#include + +extern "C" { + #include + #include +} + +#include "Exceptions.h" +#include "Frame.h" + +using namespace openshot; + +namespace +{ + std::string fraction_to_string(const Fraction& value) + { + std::stringstream stream; + stream << value.num << "/" << value.den; + return stream.str(); + } + + void set_option(AVDictionary** options, const std::string& key, const std::string& value) + { + av_dict_set(options, key.c_str(), value.c_str(), 0); + } +} + +ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settings) + : settings(new_settings) + , is_open(false) + , video_stream(-1) + , frames_read(0) + , dropped_packets(0) + , format_context(nullptr) + , codec_context(nullptr) + , source_frame(nullptr) + , rgba_frame(nullptr) + , packet(nullptr) + , sws_context(nullptr) +{ + if (settings.backend == SCREEN_CAPTURE_AUTO) { + settings.backend = DefaultBackend(); + } + ValidateSettings(); + PopulateInfo(); +} + +ScreenCaptureReader::~ScreenCaptureReader() +{ + Close(); +} + +bool ScreenCaptureReader::IsBackendSupported(ScreenCaptureBackend backend) +{ +#if defined(__linux__) + return backend == SCREEN_CAPTURE_X11 || backend == SCREEN_CAPTURE_AUTO; +#else + (void) backend; + return false; +#endif +} + +ScreenCaptureBackend ScreenCaptureReader::DefaultBackend() +{ +#if defined(__linux__) + const char* session = std::getenv("XDG_SESSION_TYPE"); + if (!session || std::string(session) == "x11") { + return SCREEN_CAPTURE_X11; + } +#endif + return SCREEN_CAPTURE_AUTO; +} + +void ScreenCaptureReader::ValidateSettings() const +{ + const bool using_v4l2_device = settings.options.count("input_format_name") && settings.options.at("input_format_name") == "v4l2"; + if (!IsBackendSupported(settings.backend)) { + throw InvalidOptions("Screen capture backend is not supported on this OS or session."); + } + if (settings.backend != SCREEN_CAPTURE_X11 && !using_v4l2_device) { + throw InvalidOptions("Only the X11 screen capture backend is implemented in this build."); + } + if (settings.width <= 0 || settings.height <= 0) { + throw InvalidOptions("Screen capture requires a positive width and height."); + } + if (settings.fps.num <= 0 || settings.fps.den <= 0) { + throw InvalidOptions("Screen capture requires a positive frame rate."); + } +} + +std::string ScreenCaptureReader::InputFormatName() const +{ + const auto override_format = settings.options.find("input_format_name"); + if (override_format != settings.options.end()) { + return override_format->second; + } + if (settings.backend == SCREEN_CAPTURE_X11) { + return "x11grab"; + } + return ""; +} + +std::string ScreenCaptureReader::InputName() const +{ + if (InputFormatName() == "v4l2") { + return settings.display.empty() ? "/dev/video0" : settings.display; + } + + std::string display = settings.display; + if (display.empty()) { + const char* env_display = std::getenv("DISPLAY"); + display = env_display ? env_display : ":0.0"; + } + if (InputFormatName() == "x11grab" && settings.options.count("window_id")) { + return display; + } + + std::stringstream input; + input << display << "+" << settings.x << "," << settings.y; + return input.str(); +} + +void ScreenCaptureReader::PopulateInfo() +{ + info.has_video = true; + info.has_audio = false; + info.has_single_image = false; + info.duration = 60.0f * 60.0f; + info.file_size = 0; + info.width = settings.width; + info.height = settings.height; + info.pixel_format = AV_PIX_FMT_RGBA; + info.fps = settings.fps; + info.video_bit_rate = 0; + info.pixel_ratio = Fraction(1, 1); + info.display_ratio = Fraction(settings.width, settings.height); + info.display_ratio.Reduce(); + info.vcodec = "rawvideo"; + info.video_length = static_cast(info.duration * settings.fps.ToFloat()); + info.video_stream_index = -1; + info.video_timebase = settings.fps.Reciprocal(); + info.interlaced_frame = false; + info.top_field_first = false; + info.acodec = ""; + info.audio_bit_rate = 0; + info.sample_rate = 0; + info.channels = 0; + info.channel_layout = LAYOUT_MONO; + info.audio_stream_index = -1; + info.audio_timebase = Fraction(1, 1); +} + +void ScreenCaptureReader::Open() +{ + if (is_open) { + return; + } + OpenDevice(); + OpenDecoder(); + is_open = true; +} + +void ScreenCaptureReader::OpenDevice() +{ + avdevice_register_all(); + + const AVInputFormat* input_format = av_find_input_format(InputFormatName().c_str()); + if (!input_format) { + throw InvalidOptions("FFmpeg input device is not available: " + InputFormatName(), InputName()); + } + + AVDictionary* options = nullptr; + set_option(&options, "framerate", fraction_to_string(settings.fps)); + set_option(&options, "video_size", std::to_string(settings.width) + "x" + std::to_string(settings.height)); + if (InputFormatName() == "x11grab") { + set_option(&options, "draw_mouse", settings.include_cursor ? "1" : "0"); + set_option(&options, "show_region", settings.show_region ? "1" : "0"); + } + for (const auto& option : settings.options) { + if (option.first == "input_format_name") { + continue; + } + set_option(&options, option.first, option.second); + } + + const int result = avformat_open_input(&format_context, InputName().c_str(), input_format, &options); + av_dict_free(&options); + if (result < 0) { + throw InvalidFile("Unable to open screen capture input: " + std::string(av_err2str(result)), InputName()); + } +} + +void ScreenCaptureReader::OpenDecoder() +{ + if (avformat_find_stream_info(format_context, nullptr) < 0) { + throw InvalidFile("Unable to read capture stream information.", InputName()); + } + + for (unsigned int index = 0; index < format_context->nb_streams; ++index) { + if (format_context->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + video_stream = static_cast(index); + break; + } + } + if (video_stream < 0) { + throw InvalidFile("No video stream found in capture input.", InputName()); + } + + AVStream* stream = format_context->streams[video_stream]; + const AVCodec* codec = avcodec_find_decoder(stream->codecpar->codec_id); + if (!codec) { + throw InvalidCodec("No decoder available for capture stream.", InputName()); + } + + codec_context = ffmpeg_get_codec_context(stream, codec); + if (!codec_context || avcodec_open2(codec_context, codec, nullptr) < 0) { + throw InvalidCodec("Unable to open capture stream decoder.", InputName()); + } + + info.width = codec_context->width > 0 ? codec_context->width : settings.width; + info.height = codec_context->height > 0 ? codec_context->height : settings.height; + info.video_stream_index = video_stream; + info.pixel_format = codec_context->pix_fmt; + info.display_ratio = Fraction(info.width, info.height); + info.display_ratio.Reduce(); + if (codec_context->framerate.num > 0 && codec_context->framerate.den > 0) { + info.fps = Fraction(codec_context->framerate.num, codec_context->framerate.den); + info.video_timebase = info.fps.Reciprocal(); + } else if (stream->avg_frame_rate.num > 0 && stream->avg_frame_rate.den > 0) { + info.fps = Fraction(stream->avg_frame_rate.num, stream->avg_frame_rate.den); + info.video_timebase = info.fps.Reciprocal(); + } else if (stream->r_frame_rate.num > 0 && stream->r_frame_rate.den > 0) { + info.fps = Fraction(stream->r_frame_rate.num, stream->r_frame_rate.den); + info.video_timebase = info.fps.Reciprocal(); + } + + source_frame = av_frame_alloc(); + rgba_frame = av_frame_alloc(); + packet = av_packet_alloc(); + if (!source_frame || !rgba_frame || !packet) { + throw OutOfMemory("Unable to allocate capture decoder frames.", InputName()); + } +} + +std::shared_ptr ScreenCaptureReader::GetFrame(int64_t number) +{ + if (!is_open) { + throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame()."); + } + + const std::lock_guard lock(getFrameMutex); + return DecodeNextFrame(number); +} + +std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) +{ + while (av_read_frame(format_context, packet) >= 0) { + if (packet->stream_index != video_stream) { + av_packet_unref(packet); + dropped_packets++; + continue; + } + + const int send_result = avcodec_send_packet(codec_context, packet); + av_packet_unref(packet); + if (send_result < 0) { + continue; + } + + const int receive_result = avcodec_receive_frame(codec_context, source_frame); + if (receive_result == AVERROR(EAGAIN)) { + continue; + } + if (receive_result < 0) { + throw InvalidFile("Unable to decode capture frame: " + std::string(av_err2str(receive_result)), InputName()); + } + + const int width = source_frame->width > 0 ? source_frame->width : info.width; + const int height = source_frame->height > 0 ? source_frame->height : info.height; + const PixelFormat src_fmt = static_cast(source_frame->format); + + sws_context = sws_getCachedContext( + sws_context, + width, + height, + src_fmt, + width, + height, + AV_PIX_FMT_RGBA, + SWS_BILINEAR, + nullptr, + nullptr, + nullptr); + if (!sws_context) { + throw InvalidFile("Unable to create capture pixel conversion context.", InputName()); + } + + const int bytes_per_pixel = 4; + unsigned char* buffer = static_cast(malloc(static_cast(width) * height * bytes_per_pixel)); + if (!buffer) { + throw OutOfMemory("Unable to allocate capture frame buffer.", InputName()); + } + + av_image_fill_arrays(rgba_frame->data, rgba_frame->linesize, buffer, AV_PIX_FMT_RGBA, width, height, 1); + const int scaled_lines = sws_scale(sws_context, source_frame->data, source_frame->linesize, 0, height, rgba_frame->data, rgba_frame->linesize); + av_frame_unref(source_frame); + + if (scaled_lines <= 0) { + free(buffer); + continue; + } + + auto frame = std::make_shared(number, width, height, "#000000"); + frame->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer); + frames_read++; + return frame; + } + + throw InvalidFile("Capture input ended before a frame could be read.", InputName()); +} + +void ScreenCaptureReader::Close() +{ + if (packet) { + av_packet_free(&packet); + } + if (source_frame) { + av_frame_free(&source_frame); + } + if (rgba_frame) { + av_frame_free(&rgba_frame); + } + if (sws_context) { + sws_freeContext(sws_context); + sws_context = nullptr; + } + if (codec_context) { + avcodec_free_context(&codec_context); + } + if (format_context) { + avformat_close_input(&format_context); + } + is_open = false; + video_stream = -1; +} + +openshot::CaptureReaderStats ScreenCaptureReader::GetStats() const +{ + CaptureReaderStats stats; + stats.is_open = is_open; + stats.frames_read = frames_read; + stats.dropped_packets = dropped_packets; + const double fps = settings.fps.den != 0 ? static_cast(settings.fps.num) / static_cast(settings.fps.den) : 0.0; + stats.duration = fps > 0.0 ? static_cast(frames_read) / fps : 0.0; + return stats; +} + +std::string ScreenCaptureReader::Json() const +{ + return JsonValue().toStyledString(); +} + +Json::Value ScreenCaptureReader::JsonValue() const +{ + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "ScreenCaptureReader"; + root["backend"] = settings.backend; + root["display"] = settings.display; + root["x"] = settings.x; + root["y"] = settings.y; + root["width"] = settings.width; + root["height"] = settings.height; + root["fps"]["num"] = settings.fps.num; + root["fps"]["den"] = settings.fps.den; + root["include_cursor"] = settings.include_cursor; + root["show_region"] = settings.show_region; + root["options"] = Json::Value(Json::objectValue); + for (const auto& option : settings.options) { + root["options"][option.first] = option.second; + } + return root; +} + +void ScreenCaptureReader::SetJson(const std::string value) +{ + try { + SetJsonValue(openshot::stringToJson(value)); + } catch (const std::exception&) { + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +void ScreenCaptureReader::SetJsonValue(const Json::Value root) +{ + if (!root["backend"].isNull()) + settings.backend = static_cast(root["backend"].asInt()); + if (!root["display"].isNull()) + settings.display = root["display"].asString(); + if (!root["x"].isNull()) + settings.x = root["x"].asInt(); + if (!root["y"].isNull()) + settings.y = root["y"].asInt(); + if (!root["width"].isNull()) + settings.width = root["width"].asInt(); + if (!root["height"].isNull()) + settings.height = root["height"].asInt(); + if (!root["fps"].isNull() && root["fps"].isObject()) { + if (!root["fps"]["num"].isNull()) + settings.fps.num = root["fps"]["num"].asInt(); + if (!root["fps"]["den"].isNull()) + settings.fps.den = root["fps"]["den"].asInt(); + } + if (!root["include_cursor"].isNull()) + settings.include_cursor = root["include_cursor"].asBool(); + if (!root["show_region"].isNull()) + settings.show_region = root["show_region"].asBool(); + if (!root["options"].isNull() && root["options"].isObject()) { + settings.options.clear(); + for (const auto& key : root["options"].getMemberNames()) { + settings.options[key] = root["options"][key].asString(); + } + } + if (settings.backend == SCREEN_CAPTURE_AUTO) { + settings.backend = DefaultBackend(); + } + ValidateSettings(); + PopulateInfo(); +} diff --git a/src/ScreenCaptureReader.h b/src/ScreenCaptureReader.h new file mode 100644 index 000000000..16ef6450f --- /dev/null +++ b/src/ScreenCaptureReader.h @@ -0,0 +1,102 @@ +/** + * @file + * @brief Header file for live screen capture readers + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_SCREENCAPTUREREADER_H +#define OPENSHOT_SCREENCAPTUREREADER_H + +#include "ReaderBase.h" + +#include +#include +#include + +#include "FFmpegUtilities.h" + +namespace openshot +{ + enum ScreenCaptureBackend + { + SCREEN_CAPTURE_AUTO = 0, + SCREEN_CAPTURE_X11 = 1, + SCREEN_CAPTURE_WAYLAND = 2, + SCREEN_CAPTURE_WINDOWS_GDI = 3, + SCREEN_CAPTURE_MAC_AVFOUNDATION = 4 + }; + + struct ScreenCaptureSettings + { + ScreenCaptureBackend backend = SCREEN_CAPTURE_AUTO; + std::string display; + int x = 0; + int y = 0; + int width = 1280; + int height = 720; + openshot::Fraction fps = openshot::Fraction(30, 1); + bool include_cursor = true; + bool show_region = false; + std::map options; + }; + + struct CaptureReaderStats + { + bool is_open = false; + int64_t frames_read = 0; + int dropped_packets = 0; + double duration = 0.0; + }; + + class ScreenCaptureReader : public ReaderBase + { + public: + explicit ScreenCaptureReader(const ScreenCaptureSettings& settings); + ~ScreenCaptureReader() override; + + void Close() override; + CacheBase* GetCache() override { return NULL; }; + std::shared_ptr GetFrame(int64_t number) override; + bool IsOpen() override { return is_open; }; + std::string Name() override { return "ScreenCaptureReader"; }; + std::string Json() const override; + void SetJson(const std::string value) override; + Json::Value JsonValue() const override; + void SetJsonValue(const Json::Value root) override; + void Open() override; + + openshot::CaptureReaderStats GetStats() const; + ScreenCaptureSettings GetSettings() const { return settings; }; + static bool IsBackendSupported(ScreenCaptureBackend backend); + static ScreenCaptureBackend DefaultBackend(); + + private: + void ValidateSettings() const; + void OpenDevice(); + void OpenDecoder(); + std::string InputFormatName() const; + std::string InputName() const; + std::shared_ptr DecodeNextFrame(int64_t number); + void PopulateInfo(); + + ScreenCaptureSettings settings; + bool is_open; + int video_stream; + int64_t frames_read; + int dropped_packets; + AVFormatContext* format_context; + AVCodecContext* codec_context; + AVFrame* source_frame; + AVFrame* rgba_frame; + AVPacket* packet; + SwsContext* sws_context; + }; +} + +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 95ef5a422..afe0ebe4c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,6 +29,7 @@ set(OPENSHOT_TESTS AudioRecorder AnimatedCurve AudioWaveformer + CameraCaptureReader CacheDisk CacheMemory Caption @@ -49,6 +50,7 @@ set(OPENSHOT_TESTS QtImageReader ReaderBase Settings + ScreenCaptureReader SphericalMetadata Timeline VideoCacheThread diff --git a/tests/CameraCaptureReader.cpp b/tests/CameraCaptureReader.cpp new file mode 100644 index 000000000..160647a5d --- /dev/null +++ b/tests/CameraCaptureReader.cpp @@ -0,0 +1,69 @@ +/** + * @file + * @brief Unit tests for openshot::CameraCaptureReader settings and metadata + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "openshot_catch.h" + +#include "CameraCaptureReader.h" +#include "Exceptions.h" + +using namespace openshot; + +TEST_CASE("Camera capture settings validation", "[libopenshot][cameracapturereader]") +{ +#if defined(__linux__) + CameraCaptureSettings settings; + settings.backend = CAMERA_CAPTURE_V4L2; + settings.device = "/dev/video0"; + settings.width = 640; + settings.height = 480; + settings.fps = Fraction(30, 1); + + CHECK_NOTHROW([&settings]() { CameraCaptureReader reader(settings); }()); + + settings.device = ""; + CHECK_THROWS_AS([&settings]() { CameraCaptureReader reader(settings); }(), InvalidOptions); + + settings.device = "/dev/video0"; + settings.height = 0; + CHECK_THROWS_AS([&settings]() { CameraCaptureReader reader(settings); }(), InvalidOptions); +#else + CHECK_FALSE(CameraCaptureReader::IsBackendSupported(CAMERA_CAPTURE_V4L2)); +#endif +} + +TEST_CASE("Camera capture reader reports configured video info", "[libopenshot][cameracapturereader]") +{ +#if defined(__linux__) + CameraCaptureSettings settings; + settings.backend = CAMERA_CAPTURE_V4L2; + settings.device = "/dev/video9"; + settings.width = 1280; + settings.height = 720; + settings.fps = Fraction(24, 1); + + CameraCaptureReader reader(settings); + CHECK(reader.Name() == "CameraCaptureReader"); + CHECK(reader.info.has_video == true); + CHECK(reader.info.has_audio == false); + CHECK(reader.info.width == 1280); + CHECK(reader.info.height == 720); + CHECK(reader.info.fps.num == 24); + + const Json::Value json = reader.JsonValue(); + CHECK(json["type"].asString() == "CameraCaptureReader"); + CHECK(json["device"].asString() == "/dev/video9"); + CHECK(json["width"].asInt() == 1280); + CHECK(json["height"].asInt() == 720); +#else + CHECK_FALSE(CameraCaptureReader::IsBackendSupported(CAMERA_CAPTURE_V4L2)); +#endif +} diff --git a/tests/ScreenCaptureReader.cpp b/tests/ScreenCaptureReader.cpp new file mode 100644 index 000000000..6dfcd2d62 --- /dev/null +++ b/tests/ScreenCaptureReader.cpp @@ -0,0 +1,78 @@ +/** + * @file + * @brief Unit tests for openshot::ScreenCaptureReader settings and metadata + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "openshot_catch.h" + +#include "Exceptions.h" +#include "ScreenCaptureReader.h" + +using namespace openshot; + +TEST_CASE("Screen capture settings validation", "[libopenshot][screencapturereader]") +{ +#if defined(__linux__) + ScreenCaptureSettings settings; + settings.backend = SCREEN_CAPTURE_X11; + settings.display = ":0.0"; + settings.width = 640; + settings.height = 360; + settings.fps = Fraction(30, 1); + + CHECK_NOTHROW([&settings]() { ScreenCaptureReader reader(settings); }()); + + settings.width = 0; + CHECK_THROWS_AS([&settings]() { ScreenCaptureReader reader(settings); }(), InvalidOptions); + + settings.width = 640; + settings.fps = Fraction(0, 1); + CHECK_THROWS_AS([&settings]() { ScreenCaptureReader reader(settings); }(), InvalidOptions); +#else + CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); +#endif +} + +TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][screencapturereader]") +{ +#if defined(__linux__) + ScreenCaptureSettings settings; + settings.backend = SCREEN_CAPTURE_X11; + settings.display = ":99.0"; + settings.x = 10; + settings.y = 20; + settings.width = 800; + settings.height = 450; + settings.fps = Fraction(25, 1); + settings.include_cursor = false; + settings.show_region = true; + settings.options["window_id"] = "12345"; + + ScreenCaptureReader reader(settings); + CHECK(reader.Name() == "ScreenCaptureReader"); + CHECK(reader.info.has_video == true); + CHECK(reader.info.has_audio == false); + CHECK(reader.info.width == 800); + CHECK(reader.info.height == 450); + CHECK(reader.info.fps.num == 25); + CHECK(reader.info.fps.den == 1); + + const Json::Value json = reader.JsonValue(); + CHECK(json["type"].asString() == "ScreenCaptureReader"); + CHECK(json["display"].asString() == ":99.0"); + CHECK(json["x"].asInt() == 10); + CHECK(json["y"].asInt() == 20); + CHECK(json["include_cursor"].asBool() == false); + CHECK(json["show_region"].asBool() == true); + CHECK(json["options"]["window_id"].asString() == "12345"); +#else + CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); +#endif +} From 5f49df808ddcae4eeb745c2eca72c8bd7530ad10 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 16 Jun 2026 23:00:11 -0500 Subject: [PATCH 04/30] Add Wayland screen capture backend --- .gitlab-ci.yml | 3 +- CMakeLists.txt | 1 + doc/INSTALL-LINUX.md | 9 + src/CMakeLists.txt | 25 ++ src/ScreenCaptureReader.cpp | 65 ++- src/ScreenCaptureReader.h | 20 +- src/WaylandScreenCaptureReader.cpp | 656 +++++++++++++++++++++++++++++ tests/ScreenCaptureReader.cpp | 48 +++ 8 files changed, 823 insertions(+), 4 deletions(-) create mode 100644 src/WaylandScreenCaptureReader.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bbbdc779b..9086afacf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,8 @@ linux-builder: - export LD_LIBRARY_PATH="$OPENCV_ROOT/lib:$LD_LIBRARY_PATH" - export PATH="$OPENCV_ROOT/bin:$PATH" - mkdir -p build; cd build; - - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 ../ + - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 + - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON ../ - cmake --build . --parallel $(nproc) - make install - ctest --output-on-failure -VV diff --git a/CMakeLists.txt b/CMakeLists.txt index d11f80a49..634784791 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ option(USE_SYSTEM_JSONCPP "Use system installed JsonCpp, if found" ON) option(DISABLE_BUNDLED_JSONCPP "Don't fall back to bundled JsonCpp" OFF) option(ENABLE_MAGICK "Use ImageMagick, if available" ON) option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)" ON) +option(ENABLE_WAYLAND_CAPTURE "Build Wayland screen capture support with xdg-desktop-portal and PipeWire (Linux only)" ON) option(USE_HW_ACCEL "Enable hardware-accelerated encoding-decoding with FFmpeg 3.4+" ON) option(ENABLE_VULKAN_BENCHMARK "Build the experimental Vulkan benchmark example" OFF) diff --git a/doc/INSTALL-LINUX.md b/doc/INSTALL-LINUX.md index d297f038c..7633da879 100644 --- a/doc/INSTALL-LINUX.md +++ b/doc/INSTALL-LINUX.md @@ -34,6 +34,11 @@ list below to help distinguish between them. * http://www.ffmpeg.org/ `(Library)` * This library is used to decode and encode video, audio, and image files. It is also used to obtain information about media files, such as frame rate, sample rate, aspect ratio, and other common attributes. +### PipeWire and xdg-desktop-portal (libpipewire, libspa, GIO) + * https://pipewire.org/ `(Library)` + * https://flatpak.github.io/xdg-desktop-portal/ `(Runtime Service)` + * These libraries and services are used for Wayland screen capture. The development packages are needed at build time, and a desktop portal implementation must be available at runtime. + ### ImageMagick++ (libMagick++, libMagickWand, libMagickCore) * http://www.imagemagick.org/script/magick++.php `(Library)` * This library is **optional**, and used to decode and encode images. @@ -154,6 +159,8 @@ software packages available to download and install. libjsoncpp-dev \ libmagick++-dev \ libopenshot-audio-dev \ + libpipewire-0.3-dev \ + libspa-0.2-dev \ libswscale-dev \ libunittest++-dev \ libxcursor-dev \ @@ -165,6 +172,8 @@ software packages available to download and install. qtbase5-dev \ qtmultimedia5-dev \ swig \ + xdg-desktop-portal \ + xdg-desktop-portal-gtk \ python3-zmq \ python3-pyqt5.qtwebengine diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d900a559..503b72d80 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -95,6 +95,21 @@ set(OPENSHOT_SOURCES ZmqLogger.cpp ) +set(OPENSHOT_WAYLAND_CAPTURE FALSE) +if(ENABLE_WAYLAND_CAPTURE AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + find_package(PkgConfig) + if(PkgConfig_FOUND) + pkg_check_modules(PC_PIPEWIRE QUIET IMPORTED_TARGET libpipewire-0.3) + pkg_check_modules(PC_LIBSPA QUIET IMPORTED_TARGET libspa-0.2) + pkg_check_modules(PC_GIO QUIET IMPORTED_TARGET gio-2.0) + pkg_check_modules(PC_GIO_UNIX QUIET IMPORTED_TARGET gio-unix-2.0) + if(PC_PIPEWIRE_FOUND AND PC_LIBSPA_FOUND AND PC_GIO_FOUND AND PC_GIO_UNIX_FOUND) + set(OPENSHOT_WAYLAND_CAPTURE TRUE) + list(APPEND OPENSHOT_SOURCES WaylandScreenCaptureReader.cpp) + endif() + endif() +endif() + # OpenCV related classes set(OPENSHOT_CV_SOURCES CVTracker.cpp @@ -195,6 +210,16 @@ target_include_directories(openshot $ $) +if(OPENSHOT_WAYLAND_CAPTURE) + target_compile_definitions(openshot PRIVATE HAVE_WAYLAND_CAPTURE=1) + target_link_libraries(openshot PRIVATE + PkgConfig::PC_PIPEWIRE + PkgConfig::PC_LIBSPA + PkgConfig::PC_GIO + PkgConfig::PC_GIO_UNIX) +endif() +add_feature_info("Wayland screen capture" OPENSHOT_WAYLAND_CAPTURE "Use xdg-desktop-portal ScreenCast and PipeWire") + ################# LIBOPENSHOT-AUDIO ################### # Find JUCE-based openshot Audio libraries if(NOT TARGET OpenShot::Audio) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 1028d280b..84a53bb9b 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -41,8 +41,15 @@ namespace } } +#if defined(HAVE_WAYLAND_CAPTURE) +std::unique_ptr CreateWaylandScreenCaptureReader( + const ScreenCaptureSettings& settings, + ReaderInfo& info); +#endif + ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settings) : settings(new_settings) + , backend_reader(nullptr) , is_open(false) , video_stream(-1) , frames_read(0) @@ -59,6 +66,16 @@ ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settin } ValidateSettings(); PopulateInfo(); + if (UsesWaylandPortal()) { + #if defined(HAVE_WAYLAND_CAPTURE) + backend_reader = CreateWaylandScreenCaptureReader(settings, info); + if (!backend_reader) { + throw InvalidOptions("Wayland screen capture backend is unavailable in this build."); + } + #else + throw InvalidOptions("Wayland screen capture backend is unavailable in this build."); + #endif + } } ScreenCaptureReader::~ScreenCaptureReader() @@ -66,10 +83,23 @@ ScreenCaptureReader::~ScreenCaptureReader() Close(); } +bool ScreenCaptureReader::IsOpen() +{ + return backend_reader ? backend_reader->IsOpen() : is_open; +} + bool ScreenCaptureReader::IsBackendSupported(ScreenCaptureBackend backend) { #if defined(__linux__) - return backend == SCREEN_CAPTURE_X11 || backend == SCREEN_CAPTURE_AUTO; + if (backend == SCREEN_CAPTURE_X11 || backend == SCREEN_CAPTURE_AUTO) { + return true; + } +#if defined(HAVE_WAYLAND_CAPTURE) + if (backend == SCREEN_CAPTURE_WAYLAND) { + return true; + } +#endif + return false; #else (void) backend; return false; @@ -80,6 +110,9 @@ ScreenCaptureBackend ScreenCaptureReader::DefaultBackend() { #if defined(__linux__) const char* session = std::getenv("XDG_SESSION_TYPE"); + if (session && std::string(session) == "wayland" && IsBackendSupported(SCREEN_CAPTURE_WAYLAND)) { + return SCREEN_CAPTURE_WAYLAND; + } if (!session || std::string(session) == "x11") { return SCREEN_CAPTURE_X11; } @@ -93,7 +126,7 @@ void ScreenCaptureReader::ValidateSettings() const if (!IsBackendSupported(settings.backend)) { throw InvalidOptions("Screen capture backend is not supported on this OS or session."); } - if (settings.backend != SCREEN_CAPTURE_X11 && !using_v4l2_device) { + if (!UsesFFmpegDevice() && !UsesWaylandPortal() && !using_v4l2_device) { throw InvalidOptions("Only the X11 screen capture backend is implemented in this build."); } if (settings.width <= 0 || settings.height <= 0) { @@ -104,6 +137,17 @@ void ScreenCaptureReader::ValidateSettings() const } } +bool ScreenCaptureReader::UsesFFmpegDevice() const +{ + const bool using_v4l2_device = settings.options.count("input_format_name") && settings.options.at("input_format_name") == "v4l2"; + return settings.backend == SCREEN_CAPTURE_X11 || using_v4l2_device; +} + +bool ScreenCaptureReader::UsesWaylandPortal() const +{ + return settings.backend == SCREEN_CAPTURE_WAYLAND; +} + std::string ScreenCaptureReader::InputFormatName() const { const auto override_format = settings.options.find("input_format_name"); @@ -168,6 +212,10 @@ void ScreenCaptureReader::PopulateInfo() void ScreenCaptureReader::Open() { + if (backend_reader) { + backend_reader->Open(); + return; + } if (is_open) { return; } @@ -260,6 +308,13 @@ void ScreenCaptureReader::OpenDecoder() std::shared_ptr ScreenCaptureReader::GetFrame(int64_t number) { + if (backend_reader) { + if (!backend_reader->IsOpen()) { + throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame()."); + } + const std::lock_guard lock(getFrameMutex); + return backend_reader->GetFrame(number); + } if (!is_open) { throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame()."); } @@ -337,6 +392,9 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) void ScreenCaptureReader::Close() { + if (backend_reader) { + backend_reader->Close(); + } if (packet) { av_packet_free(&packet); } @@ -362,6 +420,9 @@ void ScreenCaptureReader::Close() openshot::CaptureReaderStats ScreenCaptureReader::GetStats() const { + if (backend_reader) { + return backend_reader->GetStats(); + } CaptureReaderStats stats; stats.is_open = is_open; stats.frames_read = frames_read; diff --git a/src/ScreenCaptureReader.h b/src/ScreenCaptureReader.h index 16ef6450f..0e04bcc3e 100644 --- a/src/ScreenCaptureReader.h +++ b/src/ScreenCaptureReader.h @@ -57,13 +57,26 @@ namespace openshot class ScreenCaptureReader : public ReaderBase { public: +#ifndef SWIG + class CaptureBackendReader + { + public: + virtual ~CaptureBackendReader() = default; + virtual void Open() = 0; + virtual void Close() = 0; + virtual bool IsOpen() const = 0; + virtual std::shared_ptr GetFrame(int64_t number) = 0; + virtual openshot::CaptureReaderStats GetStats() const = 0; + }; +#endif + explicit ScreenCaptureReader(const ScreenCaptureSettings& settings); ~ScreenCaptureReader() override; void Close() override; CacheBase* GetCache() override { return NULL; }; std::shared_ptr GetFrame(int64_t number) override; - bool IsOpen() override { return is_open; }; + bool IsOpen() override; std::string Name() override { return "ScreenCaptureReader"; }; std::string Json() const override; void SetJson(const std::string value) override; @@ -83,9 +96,14 @@ namespace openshot std::string InputFormatName() const; std::string InputName() const; std::shared_ptr DecodeNextFrame(int64_t number); + bool UsesFFmpegDevice() const; + bool UsesWaylandPortal() const; void PopulateInfo(); ScreenCaptureSettings settings; +#ifndef SWIG + std::unique_ptr backend_reader; +#endif bool is_open; int video_stream; int64_t frames_read; diff --git a/src/WaylandScreenCaptureReader.cpp b/src/WaylandScreenCaptureReader.cpp new file mode 100644 index 000000000..2fd687e94 --- /dev/null +++ b/src/WaylandScreenCaptureReader.cpp @@ -0,0 +1,656 @@ +/** + * @file + * @brief Wayland screen capture backend using xdg-desktop-portal and PipeWire + * @author Jonathan Thomas + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "ScreenCaptureReader.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "Exceptions.h" +#include "Frame.h" + +using namespace openshot; + +namespace +{ + const char* PORTAL_BUS = "org.freedesktop.portal.Desktop"; + const char* PORTAL_PATH = "/org/freedesktop/portal/desktop"; + const char* SCREENCAST_IFACE = "org.freedesktop.portal.ScreenCast"; + + std::string gerror_message(GError* error) + { + if (!error) { + return "unknown error"; + } + std::string message = error->message ? error->message : "unknown error"; + g_error_free(error); + return message; + } + + std::string token_for(const char* prefix) + { + static uint64_t counter = 0; + std::stringstream token; + token << prefix << "_" << ++counter; + return token.str(); + } + + struct PortalResponse + { + bool done = false; + bool timed_out = false; + uint32_t code = 1; + GVariant* results = nullptr; + GMainLoop* loop = nullptr; + + ~PortalResponse() + { + if (results) { + g_variant_unref(results); + } + } + }; + + void portal_response_callback( + GDBusConnection*, + const gchar*, + const gchar*, + const gchar*, + const gchar*, + GVariant* parameters, + gpointer user_data) + { + auto* response = static_cast(user_data); + GVariant* results = nullptr; + g_variant_get(parameters, "(u@a{sv})", &response->code, &results); + response->results = results; + response->done = true; + if (response->loop) { + g_main_loop_quit(response->loop); + } + } + + gboolean portal_response_timeout(gpointer user_data) + { + auto* response = static_cast(user_data); + response->timed_out = true; + if (response->loop) { + g_main_loop_quit(response->loop); + } + return G_SOURCE_REMOVE; + } + + GVariant* wait_for_portal_response(GDBusConnection* connection, const std::string& handle) + { + PortalResponse response; + response.loop = g_main_loop_new(nullptr, FALSE); + const guint timeout = g_timeout_add_seconds(60, portal_response_timeout, &response); + const guint subscription = g_dbus_connection_signal_subscribe( + connection, + PORTAL_BUS, + "org.freedesktop.portal.Request", + "Response", + handle.c_str(), + nullptr, + G_DBUS_SIGNAL_FLAGS_NO_MATCH_RULE, + portal_response_callback, + &response, + nullptr); + + g_main_loop_run(response.loop); + g_dbus_connection_signal_unsubscribe(connection, subscription); + if (!response.timed_out) { + g_source_remove(timeout); + } + g_main_loop_unref(response.loop); + response.loop = nullptr; + + if (response.timed_out) { + throw InvalidOptions("Timed out waiting for Wayland screen capture permission."); + } + if (!response.done || response.code != 0) { + throw InvalidOptions("Wayland screen capture permission was denied or cancelled."); + } + GVariant* results = response.results; + response.results = nullptr; + return results; + } + + std::string call_portal_request( + GDBusConnection* connection, + const char* method, + GVariant* parameters) + { + GError* error = nullptr; + GVariant* result = g_dbus_connection_call_sync( + connection, + PORTAL_BUS, + PORTAL_PATH, + SCREENCAST_IFACE, + method, + parameters, + G_VARIANT_TYPE("(o)"), + G_DBUS_CALL_FLAGS_NONE, + -1, + nullptr, + &error); + if (!result) { + throw InvalidOptions("Wayland portal " + std::string(method) + " failed: " + gerror_message(error)); + } + + const char* handle = nullptr; + g_variant_get(result, "(&o)", &handle); + std::string handle_path = handle ? handle : ""; + g_variant_unref(result); + return handle_path; + } + + uint32_t stream_node_id_from_results(GVariant* results) + { + GVariant* streams = g_variant_lookup_value(results, "streams", G_VARIANT_TYPE("a(ua{sv})")); + if (!streams || g_variant_n_children(streams) == 0) { + if (streams) { + g_variant_unref(streams); + } + throw InvalidOptions("Wayland portal did not return a PipeWire stream."); + } + + uint32_t node_id = 0; + GVariant* properties = nullptr; + GVariant* child = g_variant_get_child_value(streams, 0); + g_variant_get(child, "(u@a{sv})", &node_id, &properties); + if (properties) { + g_variant_unref(properties); + } + g_variant_unref(child); + g_variant_unref(streams); + return node_id; + } + + int open_pipewire_remote(GDBusConnection* connection, const std::string& session_handle) + { + GVariantBuilder options; + g_variant_builder_init(&options, G_VARIANT_TYPE_VARDICT); + + GError* error = nullptr; + GUnixFDList* out_fds = nullptr; + GVariant* result = g_dbus_connection_call_with_unix_fd_list_sync( + connection, + PORTAL_BUS, + PORTAL_PATH, + SCREENCAST_IFACE, + "OpenPipeWireRemote", + g_variant_new("(oa{sv})", session_handle.c_str(), &options), + G_VARIANT_TYPE("(h)"), + G_DBUS_CALL_FLAGS_NONE, + -1, + nullptr, + &out_fds, + nullptr, + &error); + if (!result) { + throw InvalidOptions("Wayland portal OpenPipeWireRemote failed: " + gerror_message(error)); + } + + int fd_index = -1; + g_variant_get(result, "(h)", &fd_index); + g_variant_unref(result); + int fd = g_unix_fd_list_get(out_fds, fd_index, &error); + g_object_unref(out_fds); + if (fd < 0) { + throw InvalidOptions("Unable to obtain PipeWire remote file descriptor: " + gerror_message(error)); + } + return fd; + } + + struct CapturedFrame + { + int width = 0; + int height = 0; + std::vector rgba; + }; +} + +class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBackendReader +{ +public: + WaylandScreenCaptureReader(const ScreenCaptureSettings& new_settings, ReaderInfo& new_info) + : settings(new_settings) + , info(new_info) + , connection(nullptr) + , thread_loop(nullptr) + , context(nullptr) + , core(nullptr) + , stream(nullptr) + , node_id(0) + , frames_read(0) + , dropped_packets(0) + , open(false) + , streaming(false) + , stream_error(false) + { + } + + ~WaylandScreenCaptureReader() override + { + Close(); + } + + void Open() override + { + if (open) { + return; + } + + GError* error = nullptr; + connection = g_bus_get_sync(G_BUS_TYPE_SESSION, nullptr, &error); + if (!connection) { + throw InvalidOptions("Unable to connect to the session bus for Wayland capture: " + gerror_message(error)); + } + + std::string session_handle = CreatePortalSession(); + SelectPortalSources(session_handle); + node_id = StartPortalSession(session_handle); + const int pipewire_fd = open_pipewire_remote(connection, session_handle); + OpenPipeWireStream(pipewire_fd); + open = true; + } + + void Close() override + { + open = false; + streaming = false; + + if (thread_loop) { + pw_thread_loop_stop(thread_loop); + } + if (stream) { + pw_stream_destroy(stream); + stream = nullptr; + } + if (core) { + pw_core_disconnect(core); + core = nullptr; + } + if (context) { + pw_context_destroy(context); + context = nullptr; + } + if (thread_loop) { + pw_thread_loop_destroy(thread_loop); + thread_loop = nullptr; + } + if (connection) { + g_object_unref(connection); + connection = nullptr; + } + } + + bool IsOpen() const override + { + return open; + } + + CaptureReaderStats GetStats() const override + { + CaptureReaderStats stats; + stats.is_open = open; + stats.frames_read = frames_read; + stats.dropped_packets = dropped_packets; + const double fps = settings.fps.den != 0 ? static_cast(settings.fps.num) / static_cast(settings.fps.den) : 0.0; + stats.duration = fps > 0.0 ? static_cast(frames_read) / fps : 0.0; + return stats; + } + + std::shared_ptr GetFrame(int64_t number) override + { + CapturedFrame captured; + { + std::unique_lock lock(queue_mutex); + const auto timeout = std::chrono::seconds(5); + if (!queue_condition.wait_for(lock, timeout, [this]() { return !frame_queue.empty() || stream_error || !open; })) { + throw InvalidFile("Timed out waiting for a Wayland capture frame.", "wayland"); + } + if (stream_error) { + throw InvalidFile("Wayland capture stream failed.", "wayland"); + } + if (frame_queue.empty()) { + throw ReaderClosed("The Wayland screen capture stream is closed."); + } + captured = std::move(frame_queue.front()); + frame_queue.pop_front(); + } + + const int bytes_per_pixel = 4; + const size_t buffer_size = static_cast(captured.width) * captured.height * bytes_per_pixel; + unsigned char* buffer = static_cast(malloc(buffer_size)); + if (!buffer) { + throw OutOfMemory("Unable to allocate Wayland capture frame buffer.", "wayland"); + } + std::memcpy(buffer, captured.rgba.data(), buffer_size); + + auto frame = std::make_shared(number, captured.width, captured.height, "#000000"); + frame->AddImage(captured.width, captured.height, bytes_per_pixel, QImage::Format_RGBA8888, buffer); + frames_read++; + return frame; + } + +private: + std::string CreatePortalSession() + { + GVariantBuilder options; + g_variant_builder_init(&options, G_VARIANT_TYPE_VARDICT); + g_variant_builder_add(&options, "{sv}", "handle_token", g_variant_new_string(token_for("openshot_create").c_str())); + g_variant_builder_add(&options, "{sv}", "session_handle_token", g_variant_new_string(token_for("openshot_session").c_str())); + + const std::string handle = call_portal_request(connection, "CreateSession", g_variant_new("(a{sv})", &options)); + GVariant* results = wait_for_portal_response(connection, handle); + + const char* session = nullptr; + if (!g_variant_lookup(results, "session_handle", "&s", &session) || !session) { + g_variant_unref(results); + throw InvalidOptions("Wayland portal did not return a screencast session handle."); + } + std::string session_handle = session; + g_variant_unref(results); + return session_handle; + } + + void SelectPortalSources(const std::string& session_handle) + { + const uint32_t source_monitor = 1; + const uint32_t source_window = 2; + const uint32_t cursor_hidden = 1; + const uint32_t cursor_embedded = 2; + + GVariantBuilder options; + g_variant_builder_init(&options, G_VARIANT_TYPE_VARDICT); + g_variant_builder_add(&options, "{sv}", "handle_token", g_variant_new_string(token_for("openshot_select").c_str())); + g_variant_builder_add(&options, "{sv}", "types", g_variant_new_uint32(source_monitor | source_window)); + g_variant_builder_add(&options, "{sv}", "multiple", g_variant_new_boolean(FALSE)); + g_variant_builder_add(&options, "{sv}", "cursor_mode", g_variant_new_uint32(settings.include_cursor ? cursor_embedded : cursor_hidden)); + + const std::string handle = call_portal_request( + connection, + "SelectSources", + g_variant_new("(oa{sv})", session_handle.c_str(), &options)); + GVariant* results = wait_for_portal_response(connection, handle); + g_variant_unref(results); + } + + uint32_t StartPortalSession(const std::string& session_handle) + { + GVariantBuilder options; + g_variant_builder_init(&options, G_VARIANT_TYPE_VARDICT); + g_variant_builder_add(&options, "{sv}", "handle_token", g_variant_new_string(token_for("openshot_start").c_str())); + + const std::string handle = call_portal_request( + connection, + "Start", + g_variant_new("(osa{sv})", session_handle.c_str(), "", &options)); + GVariant* results = wait_for_portal_response(connection, handle); + const uint32_t result_node_id = stream_node_id_from_results(results); + g_variant_unref(results); + return result_node_id; + } + + void OpenPipeWireStream(int pipewire_fd) + { + pw_init(nullptr, nullptr); + + thread_loop = pw_thread_loop_new("openshot-wayland-capture", nullptr); + if (!thread_loop) { + throw InvalidOptions("Unable to create PipeWire thread loop."); + } + context = pw_context_new(pw_thread_loop_get_loop(thread_loop), nullptr, 0); + if (!context) { + throw InvalidOptions("Unable to create PipeWire context."); + } + core = pw_context_connect_fd(context, pipewire_fd, nullptr, 0); + if (!core) { + close(pipewire_fd); + throw InvalidOptions("Unable to connect to the portal PipeWire remote."); + } + + pw_properties* props = pw_properties_new( + PW_KEY_MEDIA_TYPE, "Video", + PW_KEY_MEDIA_CATEGORY, "Capture", + PW_KEY_MEDIA_ROLE, "Screen", + nullptr); + stream = pw_stream_new(core, "OpenShot Wayland Screen Capture", props); + if (!stream) { + throw InvalidOptions("Unable to create PipeWire capture stream."); + } + + pw_stream_add_listener(stream, &stream_listener, &stream_events, this); + + if (pw_thread_loop_start(thread_loop) < 0) { + throw InvalidOptions("Unable to start PipeWire capture thread loop."); + } + + uint8_t format_buffer[1024]; + spa_pod_builder builder = SPA_POD_BUILDER_INIT(format_buffer, sizeof(format_buffer)); + const spa_pod* params[1]; + params[0] = static_cast(spa_pod_builder_add_object( + &builder, + SPA_TYPE_OBJECT_Format, + SPA_PARAM_EnumFormat, + SPA_FORMAT_mediaType, + SPA_POD_Id(SPA_MEDIA_TYPE_video), + SPA_FORMAT_mediaSubtype, + SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), + SPA_FORMAT_VIDEO_format, + SPA_POD_CHOICE_ENUM_Id( + 4, + SPA_VIDEO_FORMAT_BGRx, + SPA_VIDEO_FORMAT_RGBA, + SPA_VIDEO_FORMAT_BGRA, + SPA_VIDEO_FORMAT_RGBx))); + + pw_thread_loop_lock(thread_loop); + const int result = pw_stream_connect( + stream, + PW_DIRECTION_INPUT, + node_id, + static_cast(PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS), + params, + 1); + if (result < 0) { + pw_thread_loop_unlock(thread_loop); + throw InvalidOptions("Unable to connect to PipeWire capture stream."); + } + + while (!streaming && !stream_error) { + if (pw_thread_loop_timed_wait(thread_loop, 10) < 0) { + stream_error = true; + break; + } + } + pw_thread_loop_unlock(thread_loop); + + if (stream_error) { + throw InvalidOptions("PipeWire capture stream failed to start or timed out."); + } + } + + static void OnStreamStateChanged(void* data, pw_stream_state, pw_stream_state state, const char*) + { + auto* self = static_cast(data); + if (state == PW_STREAM_STATE_STREAMING) { + self->streaming = true; + pw_thread_loop_signal(self->thread_loop, false); + } else if (state == PW_STREAM_STATE_ERROR || state == PW_STREAM_STATE_UNCONNECTED) { + self->stream_error = true; + pw_thread_loop_signal(self->thread_loop, false); + self->queue_condition.notify_all(); + } + } + + static void OnStreamParamChanged(void* data, uint32_t id, const spa_pod* param) + { + if (id != SPA_PARAM_Format || !param) { + return; + } + auto* self = static_cast(data); + spa_video_info info = {}; + if (spa_format_parse(param, &info.media_type, &info.media_subtype) < 0 || + info.media_type != SPA_MEDIA_TYPE_video || + info.media_subtype != SPA_MEDIA_SUBTYPE_raw || + spa_format_video_raw_parse(param, &info.info.raw) < 0) { + return; + } + + self->video_format = info.info.raw.format; + self->stream_width = static_cast(info.info.raw.size.width); + self->stream_height = static_cast(info.info.raw.size.height); + if (self->stream_width > 0 && self->stream_height > 0) { + self->info.width = self->stream_width; + self->info.height = self->stream_height; + self->info.display_ratio = Fraction(self->stream_width, self->stream_height); + self->info.display_ratio.Reduce(); + } + } + + static void OnStreamProcess(void* data) + { + auto* self = static_cast(data); + pw_buffer* buffer = pw_stream_dequeue_buffer(self->stream); + if (!buffer) { + return; + } + + self->CopyPipeWireBuffer(buffer); + pw_stream_queue_buffer(self->stream, buffer); + } + + void CopyPipeWireBuffer(pw_buffer* buffer) + { + spa_buffer* spa_buffer = buffer->buffer; + if (!spa_buffer || spa_buffer->n_datas == 0 || !spa_buffer->datas[0].data || stream_width <= 0 || stream_height <= 0) { + dropped_packets++; + return; + } + + spa_data& data = spa_buffer->datas[0]; + const spa_chunk* chunk = data.chunk; + const uint8_t* src = static_cast(data.data) + (chunk ? chunk->offset : 0); + const int stride = chunk && chunk->stride > 0 ? chunk->stride : stream_width * 4; + const int readable_rows = chunk && chunk->size > 0 ? static_cast(chunk->size) / stride : stream_height; + const int rows = std::min(stream_height, readable_rows); + + CapturedFrame frame; + frame.width = stream_width; + frame.height = stream_height; + frame.rgba.assign(static_cast(frame.width) * frame.height * 4, 0); + + for (int y = 0; y < rows; ++y) { + const uint8_t* row = src + static_cast(y) * stride; + for (int x = 0; x < stream_width; ++x) { + const uint8_t* pixel = row + static_cast(x) * 4; + uint8_t r = 0; + uint8_t g = 0; + uint8_t b = 0; + uint8_t a = 255; + switch (video_format) { + case SPA_VIDEO_FORMAT_RGBA: + r = pixel[0]; g = pixel[1]; b = pixel[2]; a = pixel[3]; + break; + case SPA_VIDEO_FORMAT_RGBx: + r = pixel[0]; g = pixel[1]; b = pixel[2]; + break; + case SPA_VIDEO_FORMAT_BGRA: + b = pixel[0]; g = pixel[1]; r = pixel[2]; a = pixel[3]; + break; + case SPA_VIDEO_FORMAT_BGRx: + default: + b = pixel[0]; g = pixel[1]; r = pixel[2]; + break; + } + const size_t dst = (static_cast(y) * stream_width + x) * 4; + frame.rgba[dst + 0] = r; + frame.rgba[dst + 1] = g; + frame.rgba[dst + 2] = b; + frame.rgba[dst + 3] = a; + } + } + + { + std::lock_guard lock(queue_mutex); + if (frame_queue.size() > 3) { + frame_queue.pop_front(); + dropped_packets++; + } + frame_queue.push_back(std::move(frame)); + } + queue_condition.notify_one(); + } + + ScreenCaptureSettings settings; + ReaderInfo& info; + GDBusConnection* connection; + pw_thread_loop* thread_loop; + pw_context* context; + pw_core* core; + pw_stream* stream; + spa_hook stream_listener; + uint32_t node_id; + int stream_width = 0; + int stream_height = 0; + spa_video_format video_format = SPA_VIDEO_FORMAT_BGRx; + int64_t frames_read; + int dropped_packets; + bool open; + bool streaming; + bool stream_error; + std::mutex queue_mutex; + std::condition_variable queue_condition; + std::deque frame_queue; + + static const pw_stream_events stream_events; +}; + +const pw_stream_events WaylandScreenCaptureReader::stream_events = { + PW_VERSION_STREAM_EVENTS, + nullptr, + WaylandScreenCaptureReader::OnStreamStateChanged, + nullptr, + nullptr, + WaylandScreenCaptureReader::OnStreamParamChanged, + nullptr, + nullptr, + WaylandScreenCaptureReader::OnStreamProcess, + nullptr, + nullptr, + nullptr +}; + +std::unique_ptr CreateWaylandScreenCaptureReader( + const ScreenCaptureSettings& settings, + ReaderInfo& info) +{ + return std::make_unique(settings, info); +} diff --git a/tests/ScreenCaptureReader.cpp b/tests/ScreenCaptureReader.cpp index 6dfcd2d62..224fbea90 100644 --- a/tests/ScreenCaptureReader.cpp +++ b/tests/ScreenCaptureReader.cpp @@ -15,6 +15,9 @@ #include "Exceptions.h" #include "ScreenCaptureReader.h" +#include +#include + using namespace openshot; TEST_CASE("Screen capture settings validation", "[libopenshot][screencapturereader]") @@ -76,3 +79,48 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); #endif } + +TEST_CASE("Screen capture backend support follows platform build features", "[libopenshot][screencapturereader]") +{ +#if defined(__linux__) + CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); + CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); + + ScreenCaptureSettings wayland_settings; + wayland_settings.backend = SCREEN_CAPTURE_WAYLAND; + wayland_settings.width = 640; + wayland_settings.height = 360; + + if (ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_WAYLAND)) { + CHECK_NOTHROW([&wayland_settings]() { ScreenCaptureReader reader(wayland_settings); }()); + } else { + CHECK_THROWS_AS([&wayland_settings]() { ScreenCaptureReader reader(wayland_settings); }(), InvalidOptions); + } +#else + CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); + CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); + CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_WAYLAND)); +#endif +} + +TEST_CASE("Screen capture auto backend prefers Wayland only when supported", "[libopenshot][screencapturereader]") +{ +#if defined(__linux__) + const char* previous_session = std::getenv("XDG_SESSION_TYPE"); + const std::string previous_value = previous_session ? previous_session : ""; + setenv("XDG_SESSION_TYPE", "wayland", 1); + + const ScreenCaptureBackend default_backend = ScreenCaptureReader::DefaultBackend(); + if (ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_WAYLAND)) { + CHECK(default_backend == SCREEN_CAPTURE_WAYLAND); + } else { + CHECK(default_backend == SCREEN_CAPTURE_AUTO); + } + + if (previous_session) { + setenv("XDG_SESSION_TYPE", previous_value.c_str(), 1); + } else { + unsetenv("XDG_SESSION_TYPE"); + } +#endif +} From 1e75b9d856ba3007641cd8647d214dffb9b98f9d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 17 Jun 2026 13:17:22 -0500 Subject: [PATCH 05/30] Refine Wayland portal capture handling --- src/WaylandScreenCaptureReader.cpp | 133 ++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 14 deletions(-) diff --git a/src/WaylandScreenCaptureReader.cpp b/src/WaylandScreenCaptureReader.cpp index 2fd687e94..c030e93f6 100644 --- a/src/WaylandScreenCaptureReader.cpp +++ b/src/WaylandScreenCaptureReader.cpp @@ -169,7 +169,15 @@ namespace return handle_path; } - uint32_t stream_node_id_from_results(GVariant* results) + struct PortalStreamInfo + { + uint32_t node_id = 0; + uint64_t pipewire_serial = 0; + int width = 0; + int height = 0; + }; + + PortalStreamInfo stream_info_from_results(GVariant* results) { GVariant* streams = g_variant_lookup_value(results, "streams", G_VARIANT_TYPE("a(ua{sv})")); if (!streams || g_variant_n_children(streams) == 0) { @@ -179,16 +187,26 @@ namespace throw InvalidOptions("Wayland portal did not return a PipeWire stream."); } - uint32_t node_id = 0; + PortalStreamInfo stream_info; GVariant* properties = nullptr; GVariant* child = g_variant_get_child_value(streams, 0); - g_variant_get(child, "(u@a{sv})", &node_id, &properties); + g_variant_get(child, "(u@a{sv})", &stream_info.node_id, &properties); if (properties) { + uint64_t serial = 0; + int width = 0; + int height = 0; + if (g_variant_lookup(properties, "pipewire-serial", "t", &serial)) { + stream_info.pipewire_serial = serial; + } + if (g_variant_lookup(properties, "size", "(ii)", &width, &height)) { + stream_info.width = width; + stream_info.height = height; + } g_variant_unref(properties); } g_variant_unref(child); g_variant_unref(streams); - return node_id; + return stream_info; } int open_pipewire_remote(GDBusConnection* connection, const std::string& session_handle) @@ -246,7 +264,7 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack , context(nullptr) , core(nullptr) , stream(nullptr) - , node_id(0) + , session_closed_subscription(0) , frames_read(0) , dropped_packets(0) , open(false) @@ -272,9 +290,11 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack throw InvalidOptions("Unable to connect to the session bus for Wayland capture: " + gerror_message(error)); } - std::string session_handle = CreatePortalSession(); + session_handle = CreatePortalSession(); + SubscribePortalSessionClosed(); SelectPortalSources(session_handle); - node_id = StartPortalSession(session_handle); + stream_info = StartPortalSession(session_handle); + ApplyPortalStreamInfo(); const int pipewire_fd = open_pipewire_remote(connection, session_handle); OpenPipeWireStream(pipewire_fd); open = true; @@ -288,6 +308,31 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack if (thread_loop) { pw_thread_loop_stop(thread_loop); } + if (connection && session_closed_subscription) { + g_dbus_connection_signal_unsubscribe(connection, session_closed_subscription); + session_closed_subscription = 0; + } + if (connection && !session_handle.empty()) { + GError* error = nullptr; + GVariant* result = g_dbus_connection_call_sync( + connection, + PORTAL_BUS, + session_handle.c_str(), + "org.freedesktop.portal.Session", + "Close", + nullptr, + nullptr, + G_DBUS_CALL_FLAGS_NONE, + -1, + nullptr, + &error); + if (result) { + g_variant_unref(result); + } else if (error) { + g_error_free(error); + } + session_handle.clear(); + } if (stream) { pw_stream_destroy(stream); stream = nullptr; @@ -331,8 +376,15 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack CapturedFrame captured; { std::unique_lock lock(queue_mutex); - const auto timeout = std::chrono::seconds(5); - if (!queue_condition.wait_for(lock, timeout, [this]() { return !frame_queue.empty() || stream_error || !open; })) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (frame_queue.empty() && !stream_error && open && std::chrono::steady_clock::now() < deadline) { + lock.unlock(); + while (g_main_context_iteration(nullptr, FALSE)) { + } + lock.lock(); + queue_condition.wait_for(lock, std::chrono::milliseconds(100)); + } + if (frame_queue.empty() && !stream_error && open) { throw InvalidFile("Timed out waiting for a Wayland capture frame.", "wayland"); } if (stream_error) { @@ -380,6 +432,24 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack return session_handle; } + void SubscribePortalSessionClosed() + { + if (!connection || session_handle.empty()) { + return; + } + session_closed_subscription = g_dbus_connection_signal_subscribe( + connection, + PORTAL_BUS, + "org.freedesktop.portal.Session", + "Closed", + session_handle.c_str(), + nullptr, + G_DBUS_SIGNAL_FLAGS_NO_MATCH_RULE, + OnPortalSessionClosed, + this, + nullptr); + } + void SelectPortalSources(const std::string& session_handle) { const uint32_t source_monitor = 1; @@ -402,7 +472,7 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack g_variant_unref(results); } - uint32_t StartPortalSession(const std::string& session_handle) + PortalStreamInfo StartPortalSession(const std::string& session_handle) { GVariantBuilder options; g_variant_builder_init(&options, G_VARIANT_TYPE_VARDICT); @@ -413,9 +483,20 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack "Start", g_variant_new("(osa{sv})", session_handle.c_str(), "", &options)); GVariant* results = wait_for_portal_response(connection, handle); - const uint32_t result_node_id = stream_node_id_from_results(results); + const PortalStreamInfo result_stream_info = stream_info_from_results(results); g_variant_unref(results); - return result_node_id; + return result_stream_info; + } + + void ApplyPortalStreamInfo() + { + if (stream_info.width <= 0 || stream_info.height <= 0) { + return; + } + info.width = stream_info.width; + info.height = stream_info.height; + info.display_ratio = Fraction(stream_info.width, stream_info.height); + info.display_ratio.Reduce(); } void OpenPipeWireStream(int pipewire_fd) @@ -441,6 +522,9 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack PW_KEY_MEDIA_CATEGORY, "Capture", PW_KEY_MEDIA_ROLE, "Screen", nullptr); + if (stream_info.pipewire_serial > 0) { + pw_properties_set(props, PW_KEY_TARGET_OBJECT, std::to_string(stream_info.pipewire_serial).c_str()); + } stream = pw_stream_new(core, "OpenShot Wayland Screen Capture", props); if (!stream) { throw InvalidOptions("Unable to create PipeWire capture stream."); @@ -472,10 +556,11 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack SPA_VIDEO_FORMAT_RGBx))); pw_thread_loop_lock(thread_loop); + const uint32_t target_id = stream_info.pipewire_serial > 0 ? PW_ID_ANY : stream_info.node_id; const int result = pw_stream_connect( stream, PW_DIRECTION_INPUT, - node_id, + target_id, static_cast(PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS), params, 1); @@ -497,6 +582,24 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack } } + static void OnPortalSessionClosed( + GDBusConnection*, + const gchar*, + const gchar*, + const gchar*, + const gchar*, + GVariant*, + gpointer user_data) + { + auto* self = static_cast(user_data); + self->open = false; + self->stream_error = true; + if (self->thread_loop) { + pw_thread_loop_signal(self->thread_loop, false); + } + self->queue_condition.notify_all(); + } + static void OnStreamStateChanged(void* data, pw_stream_state, pw_stream_state state, const char*) { auto* self = static_cast(data); @@ -617,7 +720,9 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack pw_core* core; pw_stream* stream; spa_hook stream_listener; - uint32_t node_id; + PortalStreamInfo stream_info; + std::string session_handle; + guint session_closed_subscription; int stream_width = 0; int stream_height = 0; spa_video_format video_format = SPA_VIDEO_FORMAT_BGRx; From 9660b5ccca91072309c1de5ca93687b5070c81ec Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 17 Jun 2026 13:48:42 -0500 Subject: [PATCH 06/30] Handle Wayland cropped window streams --- src/WaylandScreenCaptureReader.cpp | 77 +++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/src/WaylandScreenCaptureReader.cpp b/src/WaylandScreenCaptureReader.cpp index c030e93f6..f15f15955 100644 --- a/src/WaylandScreenCaptureReader.cpp +++ b/src/WaylandScreenCaptureReader.cpp @@ -26,12 +26,15 @@ #include #include #include +#include +#include #include #include #include #include "Exceptions.h" #include "Frame.h" +#include "ZmqLogger.h" using namespace openshot; @@ -173,6 +176,7 @@ namespace { uint32_t node_id = 0; uint64_t pipewire_serial = 0; + uint32_t source_type = 0; int width = 0; int height = 0; }; @@ -193,11 +197,15 @@ namespace g_variant_get(child, "(u@a{sv})", &stream_info.node_id, &properties); if (properties) { uint64_t serial = 0; + uint32_t source_type = 0; int width = 0; int height = 0; if (g_variant_lookup(properties, "pipewire-serial", "t", &serial)) { stream_info.pipewire_serial = serial; } + if (g_variant_lookup(properties, "source_type", "u", &source_type)) { + stream_info.source_type = source_type; + } if (g_variant_lookup(properties, "size", "(ii)", &width, &height)) { stream_info.width = width; stream_info.height = height; @@ -270,6 +278,7 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack , open(false) , streaming(false) , stream_error(false) + , crop_logged(false) { } @@ -295,6 +304,11 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack SelectPortalSources(session_handle); stream_info = StartPortalSession(session_handle); ApplyPortalStreamInfo(); + ZmqLogger::Instance()->Log( + "Wayland portal stream selected: node_id=" + std::to_string(stream_info.node_id) + + " pipewire_serial=" + std::to_string(stream_info.pipewire_serial) + + " source_type=" + std::to_string(stream_info.source_type) + + " portal_size=" + std::to_string(stream_info.width) + "x" + std::to_string(stream_info.height)); const int pipewire_fd = open_pipewire_remote(connection, session_handle); OpenPipeWireStream(pipewire_fd); open = true; @@ -631,6 +645,10 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack self->stream_width = static_cast(info.info.raw.size.width); self->stream_height = static_cast(info.info.raw.size.height); if (self->stream_width > 0 && self->stream_height > 0) { + ZmqLogger::Instance()->Log( + "Wayland PipeWire stream format: " + + std::to_string(self->stream_width) + "x" + std::to_string(self->stream_height) + + " format=" + std::to_string(self->video_format)); self->info.width = self->stream_width; self->info.height = self->stream_height; self->info.display_ratio = Fraction(self->stream_width, self->stream_height); @@ -663,17 +681,55 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack const uint8_t* src = static_cast(data.data) + (chunk ? chunk->offset : 0); const int stride = chunk && chunk->stride > 0 ? chunk->stride : stream_width * 4; const int readable_rows = chunk && chunk->size > 0 ? static_cast(chunk->size) / stride : stream_height; - const int rows = std::min(stream_height, readable_rows); + int crop_x = 0; + int crop_y = 0; + int crop_width = stream_width; + int crop_height = stream_height; + auto* crop = static_cast( + spa_buffer_find_meta_data(spa_buffer, SPA_META_VideoCrop, sizeof(spa_meta_region))); + if (crop && spa_meta_region_is_valid(crop)) { + crop_x = std::max(0, crop->region.position.x); + crop_y = std::max(0, crop->region.position.y); + crop_width = std::min(static_cast(crop->region.size.width), stream_width - crop_x); + crop_height = std::min(static_cast(crop->region.size.height), stream_height - crop_y); + if (!crop_logged) { + ZmqLogger::Instance()->Log( + "Wayland PipeWire video crop: x=" + std::to_string(crop_x) + + " y=" + std::to_string(crop_y) + + " width=" + std::to_string(crop_width) + + " height=" + std::to_string(crop_height)); + crop_logged = true; + } + } + if (crop_width <= 0 || crop_height <= 0) { + dropped_packets++; + return; + } + if (crop_width % 2 != 0) { + crop_width--; + } + if (crop_height % 2 != 0) { + crop_height--; + } + if (crop_width <= 0 || crop_height <= 0) { + dropped_packets++; + return; + } + const int rows = std::min(crop_height, readable_rows - crop_y); + if (rows <= 0) { + dropped_packets++; + return; + } CapturedFrame frame; - frame.width = stream_width; - frame.height = stream_height; + frame.width = crop_width; + frame.height = crop_height; frame.rgba.assign(static_cast(frame.width) * frame.height * 4, 0); for (int y = 0; y < rows; ++y) { - const uint8_t* row = src + static_cast(y) * stride; - for (int x = 0; x < stream_width; ++x) { - const uint8_t* pixel = row + static_cast(x) * 4; + const uint8_t* row = src + static_cast(y + crop_y) * stride; + for (int x = 0; x < crop_width; ++x) { + const uint8_t* pixel = row + static_cast(x + crop_x) * 4; uint8_t r = 0; uint8_t g = 0; uint8_t b = 0; @@ -693,7 +749,7 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack b = pixel[0]; g = pixel[1]; r = pixel[2]; break; } - const size_t dst = (static_cast(y) * stream_width + x) * 4; + const size_t dst = (static_cast(y) * crop_width + x) * 4; frame.rgba[dst + 0] = r; frame.rgba[dst + 1] = g; frame.rgba[dst + 2] = b; @@ -709,6 +765,12 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack } frame_queue.push_back(std::move(frame)); } + if (info.width != crop_width || info.height != crop_height) { + info.width = crop_width; + info.height = crop_height; + info.display_ratio = Fraction(crop_width, crop_height); + info.display_ratio.Reduce(); + } queue_condition.notify_one(); } @@ -731,6 +793,7 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack bool open; bool streaming; bool stream_error; + bool crop_logged; std::mutex queue_mutex; std::condition_variable queue_condition; std::deque frame_queue; From caae5b57dd71cd53f7deb8dae3a5fc43608faada Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 17 Jun 2026 14:07:19 -0500 Subject: [PATCH 07/30] Request Wayland stream crop metadata --- src/WaylandScreenCaptureReader.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/WaylandScreenCaptureReader.cpp b/src/WaylandScreenCaptureReader.cpp index f15f15955..36062eb2c 100644 --- a/src/WaylandScreenCaptureReader.cpp +++ b/src/WaylandScreenCaptureReader.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -654,6 +655,27 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack self->info.display_ratio = Fraction(self->stream_width, self->stream_height); self->info.display_ratio.Reduce(); } + + uint8_t params_buffer[256]; + spa_pod_builder builder = SPA_POD_BUILDER_INIT(params_buffer, sizeof(params_buffer)); + const spa_pod* params[2]; + params[0] = static_cast(spa_pod_builder_add_object( + &builder, + SPA_TYPE_OBJECT_ParamMeta, + SPA_PARAM_Meta, + SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Header), + SPA_PARAM_META_size, + SPA_POD_Int(sizeof(spa_meta_header)))); + params[1] = static_cast(spa_pod_builder_add_object( + &builder, + SPA_TYPE_OBJECT_ParamMeta, + SPA_PARAM_Meta, + SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_VideoCrop), + SPA_PARAM_META_size, + SPA_POD_Int(sizeof(spa_meta_region)))); + pw_stream_update_params(self->stream, params, 2); } static void OnStreamProcess(void* data) From 1cde209af60e65c9eb2909195d5320eebfde1bd1 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 19 Jun 2026 22:52:31 -0500 Subject: [PATCH 08/30] Add Windows capture backends for recording --- .gitlab-ci.yml | 4 ++ src/CameraCaptureReader.cpp | 80 +++++++++++++++++++++++++++++++++-- src/CameraCaptureReader.h | 2 + src/ScreenCaptureReader.cpp | 28 +++++++++--- tests/CameraCaptureReader.cpp | 22 +++++++++- tests/ScreenCaptureReader.cpp | 28 +++++++++++- 6 files changed, 151 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9086afacf..cd3d9e59a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,6 +105,8 @@ windows-builder-x64: - $env:OpenCV_DIR = "$env:OPENCV_ROOT\lib\cmake\opencv4" - $env:Path = "$env:OPENCV_ROOT\bin;C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; - $env:MSYSTEM = "MINGW64" + - ffmpeg -hide_banner -devices | findstr /I "gdigrab" + - ffmpeg -hide_banner -devices | findstr /I "dshow" - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" - cmake --build build --parallel $([Environment]::ProcessorCount) - ctest --test-dir build --output-on-failure -VV @@ -134,6 +136,8 @@ windows-builder-x86: - $env:OpenCV_DIR = "$env:OPENCV_ROOT\lib\cmake\opencv4" - $env:Path = "$env:OPENCV_ROOT\bin;C:\msys64\mingw32\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; - $env:MSYSTEM = "MINGW32" + - ffmpeg -hide_banner -devices | findstr /I "gdigrab" + - ffmpeg -hide_banner -devices | findstr /I "dshow" - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32" - cmake --build build --parallel $([Environment]::ProcessorCount) - cmake --install build diff --git a/src/CameraCaptureReader.cpp b/src/CameraCaptureReader.cpp index b2781f516..f47a074cf 100644 --- a/src/CameraCaptureReader.cpp +++ b/src/CameraCaptureReader.cpp @@ -14,6 +14,10 @@ #include +extern "C" { + #include +} + #include "Exceptions.h" using namespace openshot; @@ -38,6 +42,8 @@ bool CameraCaptureReader::IsBackendSupported(CameraCaptureBackend backend) { #if defined(__linux__) return backend == CAMERA_CAPTURE_V4L2 || backend == CAMERA_CAPTURE_AUTO; +#elif defined(_WIN32) + return backend == CAMERA_CAPTURE_WINDOWS_DSHOW || backend == CAMERA_CAPTURE_AUTO; #else (void) backend; return false; @@ -48,18 +54,78 @@ CameraCaptureBackend CameraCaptureReader::DefaultBackend() { #if defined(__linux__) return CAMERA_CAPTURE_V4L2; +#elif defined(_WIN32) + return CAMERA_CAPTURE_WINDOWS_DSHOW; #else return CAMERA_CAPTURE_AUTO; #endif } +AudioDeviceList CameraCaptureReader::GetDeviceNames(CameraCaptureBackend backend) +{ + if (backend == CAMERA_CAPTURE_AUTO) { + backend = DefaultBackend(); + } + + AudioDeviceList devices; + const char* input_format_name = nullptr; +#if defined(__linux__) + if (backend == CAMERA_CAPTURE_V4L2) { + input_format_name = "v4l2"; + } +#elif defined(_WIN32) + if (backend == CAMERA_CAPTURE_WINDOWS_DSHOW) { + input_format_name = "dshow"; + } +#endif + if (!input_format_name) { + return devices; + } + + avdevice_register_all(); + const AVInputFormat* input_format = av_find_input_format(input_format_name); + if (!input_format) { + return devices; + } + + AVDeviceInfoList* device_list = nullptr; + const int result = avdevice_list_input_sources(input_format, nullptr, nullptr, &device_list); + if (result >= 0 && device_list) { + for (int index = 0; index < device_list->nb_devices; ++index) { + const AVDeviceInfo* device = device_list->devices[index]; + if (!device || !device->device_name) { + continue; + } + if (device->nb_media_types > 0 && device->media_types) { + bool has_video = false; + for (int media_index = 0; media_index < device->nb_media_types; ++media_index) { + if (device->media_types[media_index] == AVMEDIA_TYPE_VIDEO) { + has_video = true; + break; + } + } + if (!has_video) { + continue; + } + } + const std::string name = device->device_name; + const std::string label = device->device_description + ? device->device_description + : device->device_name; + devices.emplace_back(label, name); + } + } + avdevice_free_list_devices(&device_list); + return devices; +} + void CameraCaptureReader::ValidateSettings() const { if (!IsBackendSupported(settings.backend)) { throw InvalidOptions("Camera capture backend is not supported on this OS."); } - if (settings.backend != CAMERA_CAPTURE_V4L2) { - throw InvalidOptions("Only the v4l2 camera capture backend is implemented in this build."); + if (settings.backend != CAMERA_CAPTURE_V4L2 && settings.backend != CAMERA_CAPTURE_WINDOWS_DSHOW) { + throw InvalidOptions("Camera capture backend is not implemented in this build."); } if (settings.device.empty()) { throw InvalidOptions("Camera capture requires a device path."); @@ -75,13 +141,19 @@ void CameraCaptureReader::ValidateSettings() const ScreenCaptureSettings CameraCaptureReader::ToDeviceSettings() const { ScreenCaptureSettings converted; - converted.backend = SCREEN_CAPTURE_X11; converted.display = settings.device; converted.width = settings.width; converted.height = settings.height; converted.fps = settings.fps; converted.options = settings.options; - converted.options["input_format_name"] = "v4l2"; + if (settings.backend == CAMERA_CAPTURE_WINDOWS_DSHOW) { + converted.backend = SCREEN_CAPTURE_WINDOWS_GDI; + converted.display = "video=" + settings.device; + converted.options["input_format_name"] = "dshow"; + } else { + converted.backend = SCREEN_CAPTURE_X11; + converted.options["input_format_name"] = "v4l2"; + } return converted; } diff --git a/src/CameraCaptureReader.h b/src/CameraCaptureReader.h index 638ec7fde..f9bea3ca3 100644 --- a/src/CameraCaptureReader.h +++ b/src/CameraCaptureReader.h @@ -13,6 +13,7 @@ #ifndef OPENSHOT_CAMERACAPTUREREADER_H #define OPENSHOT_CAMERACAPTUREREADER_H +#include "AudioDevices.h" #include "ScreenCaptureReader.h" #include @@ -58,6 +59,7 @@ namespace openshot CameraCaptureSettings GetSettings() const { return settings; }; static bool IsBackendSupported(CameraCaptureBackend backend); static CameraCaptureBackend DefaultBackend(); + static AudioDeviceList GetDeviceNames(CameraCaptureBackend backend = CAMERA_CAPTURE_AUTO); private: void ValidateSettings() const; diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 84a53bb9b..71b1a0071 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -100,6 +100,8 @@ bool ScreenCaptureReader::IsBackendSupported(ScreenCaptureBackend backend) } #endif return false; +#elif defined(_WIN32) + return backend == SCREEN_CAPTURE_WINDOWS_GDI || backend == SCREEN_CAPTURE_AUTO; #else (void) backend; return false; @@ -116,18 +118,19 @@ ScreenCaptureBackend ScreenCaptureReader::DefaultBackend() if (!session || std::string(session) == "x11") { return SCREEN_CAPTURE_X11; } +#elif defined(_WIN32) + return SCREEN_CAPTURE_WINDOWS_GDI; #endif return SCREEN_CAPTURE_AUTO; } void ScreenCaptureReader::ValidateSettings() const { - const bool using_v4l2_device = settings.options.count("input_format_name") && settings.options.at("input_format_name") == "v4l2"; if (!IsBackendSupported(settings.backend)) { throw InvalidOptions("Screen capture backend is not supported on this OS or session."); } - if (!UsesFFmpegDevice() && !UsesWaylandPortal() && !using_v4l2_device) { - throw InvalidOptions("Only the X11 screen capture backend is implemented in this build."); + if (!UsesFFmpegDevice() && !UsesWaylandPortal()) { + throw InvalidOptions("Screen capture backend is not implemented in this build."); } if (settings.width <= 0 || settings.height <= 0) { throw InvalidOptions("Screen capture requires a positive width and height."); @@ -139,8 +142,9 @@ void ScreenCaptureReader::ValidateSettings() const bool ScreenCaptureReader::UsesFFmpegDevice() const { - const bool using_v4l2_device = settings.options.count("input_format_name") && settings.options.at("input_format_name") == "v4l2"; - return settings.backend == SCREEN_CAPTURE_X11 || using_v4l2_device; + return settings.backend == SCREEN_CAPTURE_X11 + || settings.backend == SCREEN_CAPTURE_WINDOWS_GDI + || settings.options.count("input_format_name"); } bool ScreenCaptureReader::UsesWaylandPortal() const @@ -157,6 +161,9 @@ std::string ScreenCaptureReader::InputFormatName() const if (settings.backend == SCREEN_CAPTURE_X11) { return "x11grab"; } + if (settings.backend == SCREEN_CAPTURE_WINDOWS_GDI) { + return "gdigrab"; + } return ""; } @@ -165,6 +172,12 @@ std::string ScreenCaptureReader::InputName() const if (InputFormatName() == "v4l2") { return settings.display.empty() ? "/dev/video0" : settings.display; } + if (InputFormatName() == "dshow") { + return settings.display; + } + if (InputFormatName() == "gdigrab") { + return settings.display.empty() ? "desktop" : settings.display; + } std::string display = settings.display; if (display.empty()) { @@ -239,6 +252,11 @@ void ScreenCaptureReader::OpenDevice() if (InputFormatName() == "x11grab") { set_option(&options, "draw_mouse", settings.include_cursor ? "1" : "0"); set_option(&options, "show_region", settings.show_region ? "1" : "0"); + } else if (InputFormatName() == "gdigrab") { + set_option(&options, "draw_mouse", settings.include_cursor ? "1" : "0"); + set_option(&options, "show_region", settings.show_region ? "1" : "0"); + set_option(&options, "offset_x", std::to_string(settings.x)); + set_option(&options, "offset_y", std::to_string(settings.y)); } for (const auto& option : settings.options) { if (option.first == "input_format_name") { diff --git a/tests/CameraCaptureReader.cpp b/tests/CameraCaptureReader.cpp index 160647a5d..ba9a33cd2 100644 --- a/tests/CameraCaptureReader.cpp +++ b/tests/CameraCaptureReader.cpp @@ -19,10 +19,15 @@ using namespace openshot; TEST_CASE("Camera capture settings validation", "[libopenshot][cameracapturereader]") { -#if defined(__linux__) CameraCaptureSettings settings; +#if defined(__linux__) settings.backend = CAMERA_CAPTURE_V4L2; settings.device = "/dev/video0"; +#elif defined(_WIN32) + settings.backend = CAMERA_CAPTURE_WINDOWS_DSHOW; + settings.device = "Integrated Camera"; +#endif +#if defined(__linux__) || defined(_WIN32) settings.width = 640; settings.height = 480; settings.fps = Fraction(30, 1); @@ -42,10 +47,15 @@ TEST_CASE("Camera capture settings validation", "[libopenshot][cameracaptureread TEST_CASE("Camera capture reader reports configured video info", "[libopenshot][cameracapturereader]") { -#if defined(__linux__) CameraCaptureSettings settings; +#if defined(__linux__) settings.backend = CAMERA_CAPTURE_V4L2; settings.device = "/dev/video9"; +#elif defined(_WIN32) + settings.backend = CAMERA_CAPTURE_WINDOWS_DSHOW; + settings.device = "Integrated Camera"; +#endif +#if defined(__linux__) || defined(_WIN32) settings.width = 1280; settings.height = 720; settings.fps = Fraction(24, 1); @@ -67,3 +77,11 @@ TEST_CASE("Camera capture reader reports configured video info", "[libopenshot][ CHECK_FALSE(CameraCaptureReader::IsBackendSupported(CAMERA_CAPTURE_V4L2)); #endif } + +TEST_CASE("Camera capture default backend follows platform", "[libopenshot][cameracapturereader]") +{ +#if defined(_WIN32) + CHECK(CameraCaptureReader::IsBackendSupported(CAMERA_CAPTURE_WINDOWS_DSHOW)); + CHECK(CameraCaptureReader::DefaultBackend() == CAMERA_CAPTURE_WINDOWS_DSHOW); +#endif +} diff --git a/tests/ScreenCaptureReader.cpp b/tests/ScreenCaptureReader.cpp index 224fbea90..606b49ba4 100644 --- a/tests/ScreenCaptureReader.cpp +++ b/tests/ScreenCaptureReader.cpp @@ -22,10 +22,15 @@ using namespace openshot; TEST_CASE("Screen capture settings validation", "[libopenshot][screencapturereader]") { -#if defined(__linux__) ScreenCaptureSettings settings; +#if defined(__linux__) settings.backend = SCREEN_CAPTURE_X11; settings.display = ":0.0"; +#elif defined(_WIN32) + settings.backend = SCREEN_CAPTURE_WINDOWS_GDI; + settings.display = "desktop"; +#endif +#if defined(__linux__) || defined(_WIN32) settings.width = 640; settings.height = 360; settings.fps = Fraction(30, 1); @@ -45,10 +50,15 @@ TEST_CASE("Screen capture settings validation", "[libopenshot][screencaptureread TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][screencapturereader]") { -#if defined(__linux__) ScreenCaptureSettings settings; +#if defined(__linux__) settings.backend = SCREEN_CAPTURE_X11; settings.display = ":99.0"; +#elif defined(_WIN32) + settings.backend = SCREEN_CAPTURE_WINDOWS_GDI; + settings.display = "desktop"; +#endif +#if defined(__linux__) || defined(_WIN32) settings.x = 10; settings.y = 20; settings.width = 800; @@ -74,7 +84,9 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ CHECK(json["y"].asInt() == 20); CHECK(json["include_cursor"].asBool() == false); CHECK(json["show_region"].asBool() == true); +#if defined(__linux__) CHECK(json["options"]["window_id"].asString() == "12345"); +#endif #else CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); #endif @@ -96,13 +108,25 @@ TEST_CASE("Screen capture backend support follows platform build features", "[li } else { CHECK_THROWS_AS([&wayland_settings]() { ScreenCaptureReader reader(wayland_settings); }(), InvalidOptions); } +#else +#if defined(_WIN32) + CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); + CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_WINDOWS_GDI)); #else CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); +#endif CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_X11)); CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_WAYLAND)); #endif } +TEST_CASE("Screen capture default backend follows platform", "[libopenshot][screencapturereader]") +{ +#if defined(_WIN32) + CHECK(ScreenCaptureReader::DefaultBackend() == SCREEN_CAPTURE_WINDOWS_GDI); +#endif +} + TEST_CASE("Screen capture auto backend prefers Wayland only when supported", "[libopenshot][screencapturereader]") { #if defined(__linux__) From 4eebfe8c0449ebccb9249a2540cfa908e1a6ba06 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 19 Jun 2026 22:58:35 -0500 Subject: [PATCH 09/30] Drop invalid Wayland PipeWire capture frames --- src/WaylandScreenCaptureReader.cpp | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/WaylandScreenCaptureReader.cpp b/src/WaylandScreenCaptureReader.cpp index 36062eb2c..54a01108c 100644 --- a/src/WaylandScreenCaptureReader.cpp +++ b/src/WaylandScreenCaptureReader.cpp @@ -280,6 +280,12 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack , streaming(false) , stream_error(false) , crop_logged(false) + , have_last_header_sequence(false) + , last_header_sequence(0) + , header_sequence_ordering_active(false) + , have_last_header_pts(false) + , last_header_pts(0) + , header_drop_log_count(0) { } @@ -698,6 +704,13 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack return; } + auto* header = static_cast( + spa_buffer_find_meta_data(spa_buffer, SPA_META_Header, sizeof(spa_meta_header))); + if (header && !AcceptHeader(*header)) { + dropped_packets++; + return; + } + spa_data& data = spa_buffer->datas[0]; const spa_chunk* chunk = data.chunk; const uint8_t* src = static_cast(data.data) + (chunk ? chunk->offset : 0); @@ -796,6 +809,51 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack queue_condition.notify_one(); } + bool AcceptHeader(const spa_meta_header& header) + { + if (header.flags & (SPA_META_HEADER_FLAG_CORRUPTED | SPA_META_HEADER_FLAG_GAP)) { + LogDroppedHeader("flags", header); + return false; + } + + const bool discontinuity = (header.flags & SPA_META_HEADER_FLAG_DISCONT) != 0; + if (!discontinuity && have_last_header_sequence && header.seq > last_header_sequence) { + header_sequence_ordering_active = true; + } + if (!discontinuity && header_sequence_ordering_active && header.seq <= last_header_sequence) { + LogDroppedHeader("sequence", header); + return false; + } + if (!discontinuity && header.pts >= 0 && have_last_header_pts && header.pts < last_header_pts) { + LogDroppedHeader("pts", header); + return false; + } + + have_last_header_sequence = true; + last_header_sequence = header.seq; + if (header.pts >= 0) { + have_last_header_pts = true; + last_header_pts = header.pts; + } + return true; + } + + void LogDroppedHeader(const std::string& reason, const spa_meta_header& header) + { + if (header_drop_log_count >= 5) { + return; + } + ZmqLogger::Instance()->Log( + "Wayland PipeWire dropped non-monotonic frame: reason=" + reason + + " flags=" + std::to_string(header.flags) + + " seq=" + std::to_string(header.seq) + + " last_seq=" + (have_last_header_sequence ? std::to_string(last_header_sequence) : std::string("none")) + + " seq_active=" + std::to_string(header_sequence_ordering_active ? 1 : 0) + + " pts=" + std::to_string(header.pts) + + " last_pts=" + (have_last_header_pts ? std::to_string(last_header_pts) : std::string("none"))); + header_drop_log_count++; + } + ScreenCaptureSettings settings; ReaderInfo& info; GDBusConnection* connection; @@ -816,6 +874,12 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack bool streaming; bool stream_error; bool crop_logged; + bool have_last_header_sequence; + uint64_t last_header_sequence; + bool header_sequence_ordering_active; + bool have_last_header_pts; + int64_t last_header_pts; + int header_drop_log_count; std::mutex queue_mutex; std::condition_variable queue_condition; std::deque frame_queue; From a8191a07d60fe90cce4193a1a0cb69016daead83 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 19 Jun 2026 23:01:27 -0500 Subject: [PATCH 10/30] Fix recording build compatibility --- .gitlab-ci.yml | 2 +- src/CameraCaptureReader.cpp | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cd3d9e59a..4ba22c0be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ linux-builder: - export LD_LIBRARY_PATH="$OPENCV_ROOT/lib:$LD_LIBRARY_PATH" - export PATH="$OPENCV_ROOT/bin:$PATH" - mkdir -p build; cd build; - - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 + - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support" - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON ../ - cmake --build . --parallel $(nproc) - make install diff --git a/src/CameraCaptureReader.cpp b/src/CameraCaptureReader.cpp index f47a074cf..aa729d070 100644 --- a/src/CameraCaptureReader.cpp +++ b/src/CameraCaptureReader.cpp @@ -96,18 +96,6 @@ AudioDeviceList CameraCaptureReader::GetDeviceNames(CameraCaptureBackend backend if (!device || !device->device_name) { continue; } - if (device->nb_media_types > 0 && device->media_types) { - bool has_video = false; - for (int media_index = 0; media_index < device->nb_media_types; ++media_index) { - if (device->media_types[media_index] == AVMEDIA_TYPE_VIDEO) { - has_video = true; - break; - } - } - if (!has_video) { - continue; - } - } const std::string name = device->device_name; const std::string label = device->device_description ? device->device_description From 2034b34046680786dc27bd954d13af0c68af7c2b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 19 Jun 2026 23:04:58 -0500 Subject: [PATCH 11/30] Support older FFmpeg avdevice headers --- src/CameraCaptureReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CameraCaptureReader.cpp b/src/CameraCaptureReader.cpp index aa729d070..9fb1463df 100644 --- a/src/CameraCaptureReader.cpp +++ b/src/CameraCaptureReader.cpp @@ -83,7 +83,7 @@ AudioDeviceList CameraCaptureReader::GetDeviceNames(CameraCaptureBackend backend } avdevice_register_all(); - const AVInputFormat* input_format = av_find_input_format(input_format_name); + AVInputFormat* input_format = const_cast(av_find_input_format(input_format_name)); if (!input_format) { return devices; } From d1fbd835ad46d3e16ab8a3f5f0d8c7e01817de15 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 19 Jun 2026 23:09:14 -0500 Subject: [PATCH 12/30] Support older FFmpeg input format API --- src/ScreenCaptureReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 71b1a0071..b9f42c8ce 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -241,7 +241,7 @@ void ScreenCaptureReader::OpenDevice() { avdevice_register_all(); - const AVInputFormat* input_format = av_find_input_format(InputFormatName().c_str()); + AVInputFormat* input_format = const_cast(av_find_input_format(InputFormatName().c_str())); if (!input_format) { throw InvalidOptions("FFmpeg input device is not available: " + InputFormatName(), InputName()); } From 4c100e0a80e339fb37a10b59569e1565352b11f3 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 20 Jun 2026 15:41:46 -0500 Subject: [PATCH 13/30] Enumerate DirectShow cameras with COM --- src/CMakeLists.txt | 4 +- src/CameraCaptureReader.cpp | 98 ++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 503b72d80..0f995e51c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -616,8 +616,8 @@ if(DEFINED PROFILER) endif() if(WIN32) - # Required for exception handling on Windows - target_link_libraries(openshot PUBLIC "imagehlp" "dbghelp" ) + # Required for exception handling and DirectShow device enumeration on Windows + target_link_libraries(openshot PUBLIC "imagehlp" "dbghelp" "ole32" "oleaut32" "strmiids" ) endif() ### diff --git a/src/CameraCaptureReader.cpp b/src/CameraCaptureReader.cpp index 9fb1463df..8d3045a18 100644 --- a/src/CameraCaptureReader.cpp +++ b/src/CameraCaptureReader.cpp @@ -14,6 +14,13 @@ #include +#if defined(_WIN32) + #define WIN32_LEAN_AND_MEAN + #include + #include + #include +#endif + extern "C" { #include } @@ -22,6 +29,95 @@ extern "C" { using namespace openshot; +#if defined(_WIN32) +namespace +{ + std::string WideToUtf8(const wchar_t* value) + { + if (!value || !*value) { + return ""; + } + const int length = WideCharToMultiByte(CP_UTF8, 0, value, -1, nullptr, 0, nullptr, nullptr); + if (length <= 1) { + return ""; + } + std::string converted(static_cast(length - 1), '\0'); + WideCharToMultiByte(CP_UTF8, 0, value, -1, &converted[0], length, nullptr, nullptr); + return converted; + } + + bool ContainsDeviceName(const AudioDeviceList& devices, const std::string& name) + { + for (const auto& device : devices) { + if (device.second == name) { + return true; + } + } + return false; + } + + AudioDeviceList GetWindowsDirectShowVideoDevices() + { + AudioDeviceList devices; + const HRESULT init_result = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const bool should_uninitialize = SUCCEEDED(init_result); + if (FAILED(init_result) && init_result != RPC_E_CHANGED_MODE) { + return devices; + } + + ICreateDevEnum* device_enumerator = nullptr; + HRESULT result = CoCreateInstance( + CLSID_SystemDeviceEnum, + nullptr, + CLSCTX_INPROC_SERVER, + IID_ICreateDevEnum, + reinterpret_cast(&device_enumerator) + ); + if (SUCCEEDED(result) && device_enumerator) { + IEnumMoniker* moniker_enumerator = nullptr; + result = device_enumerator->CreateClassEnumerator( + CLSID_VideoInputDeviceCategory, + &moniker_enumerator, + 0 + ); + if (result == S_OK && moniker_enumerator) { + IMoniker* moniker = nullptr; + while (moniker_enumerator->Next(1, &moniker, nullptr) == S_OK) { + IPropertyBag* property_bag = nullptr; + result = moniker->BindToStorage( + nullptr, + nullptr, + IID_IPropertyBag, + reinterpret_cast(&property_bag) + ); + if (SUCCEEDED(result) && property_bag) { + VARIANT friendly_name; + VariantInit(&friendly_name); + result = property_bag->Read(L"FriendlyName", &friendly_name, nullptr); + if (SUCCEEDED(result) && friendly_name.vt == VT_BSTR) { + const std::string name = WideToUtf8(friendly_name.bstrVal); + if (!name.empty() && !ContainsDeviceName(devices, name)) { + devices.emplace_back(name, name); + } + } + VariantClear(&friendly_name); + property_bag->Release(); + } + moniker->Release(); + moniker = nullptr; + } + moniker_enumerator->Release(); + } + device_enumerator->Release(); + } + if (should_uninitialize) { + CoUninitialize(); + } + return devices; + } +} +#endif + CameraCaptureReader::CameraCaptureReader(const CameraCaptureSettings& new_settings) : settings(new_settings) , reader(nullptr) @@ -75,7 +171,7 @@ AudioDeviceList CameraCaptureReader::GetDeviceNames(CameraCaptureBackend backend } #elif defined(_WIN32) if (backend == CAMERA_CAPTURE_WINDOWS_DSHOW) { - input_format_name = "dshow"; + return GetWindowsDirectShowVideoDevices(); } #endif if (!input_format_name) { From ac3baf509f1fbbd03f7fafe83e90bc70c71220ed Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 20 Jun 2026 16:51:32 -0500 Subject: [PATCH 14/30] Fix Windows capture frame buffer cleanup --- src/ScreenCaptureReader.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index b9f42c8ce..1a6974b9b 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -23,6 +23,7 @@ extern "C" { #include "Exceptions.h" #include "Frame.h" +#include "QtUtilities.h" using namespace openshot; @@ -385,7 +386,8 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) } const int bytes_per_pixel = 4; - unsigned char* buffer = static_cast(malloc(static_cast(width) * height * bytes_per_pixel)); + const size_t buffer_size = static_cast(width) * height * bytes_per_pixel; + unsigned char* buffer = static_cast(aligned_malloc(buffer_size)); if (!buffer) { throw OutOfMemory("Unable to allocate capture frame buffer.", InputName()); } @@ -395,7 +397,7 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) av_frame_unref(source_frame); if (scaled_lines <= 0) { - free(buffer); + openshot::aligned_free(buffer); continue; } From 325beed42d9ac8beb97e52ac1e9ef6c63f0767b9 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 20 Jun 2026 20:35:16 -0500 Subject: [PATCH 15/30] Allow capture devices to use default modes --- src/ScreenCaptureReader.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 1a6974b9b..a70676185 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -248,8 +248,11 @@ void ScreenCaptureReader::OpenDevice() } AVDictionary* options = nullptr; - set_option(&options, "framerate", fraction_to_string(settings.fps)); - set_option(&options, "video_size", std::to_string(settings.width) + "x" + std::to_string(settings.height)); + const bool use_device_defaults = settings.options.count("use_device_defaults") > 0; + if (!use_device_defaults) { + set_option(&options, "framerate", fraction_to_string(settings.fps)); + set_option(&options, "video_size", std::to_string(settings.width) + "x" + std::to_string(settings.height)); + } if (InputFormatName() == "x11grab") { set_option(&options, "draw_mouse", settings.include_cursor ? "1" : "0"); set_option(&options, "show_region", settings.show_region ? "1" : "0"); @@ -260,7 +263,7 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, "offset_y", std::to_string(settings.y)); } for (const auto& option : settings.options) { - if (option.first == "input_format_name") { + if (option.first == "input_format_name" || option.first == "use_device_defaults") { continue; } set_option(&options, option.first, option.second); From 2c18616be5487623e57cdfa5d7952ca01faa3d61 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 21 Jun 2026 15:33:08 -0500 Subject: [PATCH 16/30] Enable macOS AVFoundation capture backends --- src/CameraCaptureReader.cpp | 24 +++++++++++++++++++++++- src/ScreenCaptureReader.cpp | 11 +++++++++++ tests/CameraCaptureReader.cpp | 19 +++++++++++++++++-- tests/ScreenCaptureReader.cpp | 21 +++++++++++++++++++-- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/CameraCaptureReader.cpp b/src/CameraCaptureReader.cpp index 8d3045a18..35f3b71c6 100644 --- a/src/CameraCaptureReader.cpp +++ b/src/CameraCaptureReader.cpp @@ -13,6 +13,7 @@ #include "CameraCaptureReader.h" #include +#include #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN @@ -140,6 +141,8 @@ bool CameraCaptureReader::IsBackendSupported(CameraCaptureBackend backend) return backend == CAMERA_CAPTURE_V4L2 || backend == CAMERA_CAPTURE_AUTO; #elif defined(_WIN32) return backend == CAMERA_CAPTURE_WINDOWS_DSHOW || backend == CAMERA_CAPTURE_AUTO; +#elif defined(__APPLE__) + return backend == CAMERA_CAPTURE_MAC_AVFOUNDATION || backend == CAMERA_CAPTURE_AUTO; #else (void) backend; return false; @@ -152,6 +155,8 @@ CameraCaptureBackend CameraCaptureReader::DefaultBackend() return CAMERA_CAPTURE_V4L2; #elif defined(_WIN32) return CAMERA_CAPTURE_WINDOWS_DSHOW; +#elif defined(__APPLE__) + return CAMERA_CAPTURE_MAC_AVFOUNDATION; #else return CAMERA_CAPTURE_AUTO; #endif @@ -173,6 +178,10 @@ AudioDeviceList CameraCaptureReader::GetDeviceNames(CameraCaptureBackend backend if (backend == CAMERA_CAPTURE_WINDOWS_DSHOW) { return GetWindowsDirectShowVideoDevices(); } +#elif defined(__APPLE__) + if (backend == CAMERA_CAPTURE_MAC_AVFOUNDATION) { + input_format_name = "avfoundation"; + } #endif if (!input_format_name) { return devices; @@ -196,6 +205,11 @@ AudioDeviceList CameraCaptureReader::GetDeviceNames(CameraCaptureBackend backend const std::string label = device->device_description ? device->device_description : device->device_name; + #if defined(__APPLE__) + if (backend == CAMERA_CAPTURE_MAC_AVFOUNDATION && label.find("Capture screen") != std::string::npos) { + continue; + } + #endif devices.emplace_back(label, name); } } @@ -208,7 +222,9 @@ void CameraCaptureReader::ValidateSettings() const if (!IsBackendSupported(settings.backend)) { throw InvalidOptions("Camera capture backend is not supported on this OS."); } - if (settings.backend != CAMERA_CAPTURE_V4L2 && settings.backend != CAMERA_CAPTURE_WINDOWS_DSHOW) { + if (settings.backend != CAMERA_CAPTURE_V4L2 + && settings.backend != CAMERA_CAPTURE_WINDOWS_DSHOW + && settings.backend != CAMERA_CAPTURE_MAC_AVFOUNDATION) { throw InvalidOptions("Camera capture backend is not implemented in this build."); } if (settings.device.empty()) { @@ -234,6 +250,12 @@ ScreenCaptureSettings CameraCaptureReader::ToDeviceSettings() const converted.backend = SCREEN_CAPTURE_WINDOWS_GDI; converted.display = "video=" + settings.device; converted.options["input_format_name"] = "dshow"; + } else if (settings.backend == CAMERA_CAPTURE_MAC_AVFOUNDATION) { + converted.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; + converted.display = settings.device.find(':') == std::string::npos + ? settings.device + ":none" + : settings.device; + converted.options["input_format_name"] = "avfoundation"; } else { converted.backend = SCREEN_CAPTURE_X11; converted.options["input_format_name"] = "v4l2"; diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index a70676185..bb52c0470 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -103,6 +103,8 @@ bool ScreenCaptureReader::IsBackendSupported(ScreenCaptureBackend backend) return false; #elif defined(_WIN32) return backend == SCREEN_CAPTURE_WINDOWS_GDI || backend == SCREEN_CAPTURE_AUTO; +#elif defined(__APPLE__) + return backend == SCREEN_CAPTURE_MAC_AVFOUNDATION || backend == SCREEN_CAPTURE_AUTO; #else (void) backend; return false; @@ -121,6 +123,8 @@ ScreenCaptureBackend ScreenCaptureReader::DefaultBackend() } #elif defined(_WIN32) return SCREEN_CAPTURE_WINDOWS_GDI; +#elif defined(__APPLE__) + return SCREEN_CAPTURE_MAC_AVFOUNDATION; #endif return SCREEN_CAPTURE_AUTO; } @@ -145,6 +149,7 @@ bool ScreenCaptureReader::UsesFFmpegDevice() const { return settings.backend == SCREEN_CAPTURE_X11 || settings.backend == SCREEN_CAPTURE_WINDOWS_GDI + || settings.backend == SCREEN_CAPTURE_MAC_AVFOUNDATION || settings.options.count("input_format_name"); } @@ -165,6 +170,9 @@ std::string ScreenCaptureReader::InputFormatName() const if (settings.backend == SCREEN_CAPTURE_WINDOWS_GDI) { return "gdigrab"; } + if (settings.backend == SCREEN_CAPTURE_MAC_AVFOUNDATION) { + return "avfoundation"; + } return ""; } @@ -179,6 +187,9 @@ std::string ScreenCaptureReader::InputName() const if (InputFormatName() == "gdigrab") { return settings.display.empty() ? "desktop" : settings.display; } + if (InputFormatName() == "avfoundation") { + return settings.display.empty() ? "1:none" : settings.display; + } std::string display = settings.display; if (display.empty()) { diff --git a/tests/CameraCaptureReader.cpp b/tests/CameraCaptureReader.cpp index ba9a33cd2..079ba1342 100644 --- a/tests/CameraCaptureReader.cpp +++ b/tests/CameraCaptureReader.cpp @@ -26,8 +26,11 @@ TEST_CASE("Camera capture settings validation", "[libopenshot][cameracaptureread #elif defined(_WIN32) settings.backend = CAMERA_CAPTURE_WINDOWS_DSHOW; settings.device = "Integrated Camera"; +#elif defined(__APPLE__) + settings.backend = CAMERA_CAPTURE_MAC_AVFOUNDATION; + settings.device = "0:none"; #endif -#if defined(__linux__) || defined(_WIN32) +#if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) settings.width = 640; settings.height = 480; settings.fps = Fraction(30, 1); @@ -54,8 +57,11 @@ TEST_CASE("Camera capture reader reports configured video info", "[libopenshot][ #elif defined(_WIN32) settings.backend = CAMERA_CAPTURE_WINDOWS_DSHOW; settings.device = "Integrated Camera"; +#elif defined(__APPLE__) + settings.backend = CAMERA_CAPTURE_MAC_AVFOUNDATION; + settings.device = "0:none"; #endif -#if defined(__linux__) || defined(_WIN32) +#if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) settings.width = 1280; settings.height = 720; settings.fps = Fraction(24, 1); @@ -70,7 +76,13 @@ TEST_CASE("Camera capture reader reports configured video info", "[libopenshot][ const Json::Value json = reader.JsonValue(); CHECK(json["type"].asString() == "CameraCaptureReader"); +#if defined(__linux__) CHECK(json["device"].asString() == "/dev/video9"); +#elif defined(_WIN32) + CHECK(json["device"].asString() == "Integrated Camera"); +#elif defined(__APPLE__) + CHECK(json["device"].asString() == "0:none"); +#endif CHECK(json["width"].asInt() == 1280); CHECK(json["height"].asInt() == 720); #else @@ -83,5 +95,8 @@ TEST_CASE("Camera capture default backend follows platform", "[libopenshot][came #if defined(_WIN32) CHECK(CameraCaptureReader::IsBackendSupported(CAMERA_CAPTURE_WINDOWS_DSHOW)); CHECK(CameraCaptureReader::DefaultBackend() == CAMERA_CAPTURE_WINDOWS_DSHOW); +#elif defined(__APPLE__) + CHECK(CameraCaptureReader::IsBackendSupported(CAMERA_CAPTURE_MAC_AVFOUNDATION)); + CHECK(CameraCaptureReader::DefaultBackend() == CAMERA_CAPTURE_MAC_AVFOUNDATION); #endif } diff --git a/tests/ScreenCaptureReader.cpp b/tests/ScreenCaptureReader.cpp index 606b49ba4..065dcdfc8 100644 --- a/tests/ScreenCaptureReader.cpp +++ b/tests/ScreenCaptureReader.cpp @@ -29,8 +29,11 @@ TEST_CASE("Screen capture settings validation", "[libopenshot][screencaptureread #elif defined(_WIN32) settings.backend = SCREEN_CAPTURE_WINDOWS_GDI; settings.display = "desktop"; +#elif defined(__APPLE__) + settings.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; + settings.display = "1:none"; #endif -#if defined(__linux__) || defined(_WIN32) +#if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) settings.width = 640; settings.height = 360; settings.fps = Fraction(30, 1); @@ -57,8 +60,11 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ #elif defined(_WIN32) settings.backend = SCREEN_CAPTURE_WINDOWS_GDI; settings.display = "desktop"; +#elif defined(__APPLE__) + settings.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; + settings.display = "1:none"; #endif -#if defined(__linux__) || defined(_WIN32) +#if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) settings.x = 10; settings.y = 20; settings.width = 800; @@ -79,7 +85,13 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ const Json::Value json = reader.JsonValue(); CHECK(json["type"].asString() == "ScreenCaptureReader"); +#if defined(__APPLE__) + CHECK(json["display"].asString() == "1:none"); +#elif defined(_WIN32) + CHECK(json["display"].asString() == "desktop"); +#else CHECK(json["display"].asString() == ":99.0"); +#endif CHECK(json["x"].asInt() == 10); CHECK(json["y"].asInt() == 20); CHECK(json["include_cursor"].asBool() == false); @@ -112,6 +124,9 @@ TEST_CASE("Screen capture backend support follows platform build features", "[li #if defined(_WIN32) CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_WINDOWS_GDI)); +#elif defined(__APPLE__) + CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); + CHECK(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_MAC_AVFOUNDATION)); #else CHECK_FALSE(ScreenCaptureReader::IsBackendSupported(SCREEN_CAPTURE_AUTO)); #endif @@ -124,6 +139,8 @@ TEST_CASE("Screen capture default backend follows platform", "[libopenshot][scre { #if defined(_WIN32) CHECK(ScreenCaptureReader::DefaultBackend() == SCREEN_CAPTURE_WINDOWS_GDI); +#elif defined(__APPLE__) + CHECK(ScreenCaptureReader::DefaultBackend() == SCREEN_CAPTURE_MAC_AVFOUNDATION); #endif } From 2abd35e06ffc821ab010a3d8637d4970a205eb7b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 21 Jun 2026 18:07:58 -0500 Subject: [PATCH 17/30] Harden macOS AVFoundation screen defaults --- src/ScreenCaptureReader.cpp | 4 +++- tests/ScreenCaptureReader.cpp | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index bb52c0470..9b66b41d3 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -188,7 +188,7 @@ std::string ScreenCaptureReader::InputName() const return settings.display.empty() ? "desktop" : settings.display; } if (InputFormatName() == "avfoundation") { - return settings.display.empty() ? "1:none" : settings.display; + return settings.display.empty() ? "Capture screen 0:none" : settings.display; } std::string display = settings.display; @@ -272,6 +272,8 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, "show_region", settings.show_region ? "1" : "0"); set_option(&options, "offset_x", std::to_string(settings.x)); set_option(&options, "offset_y", std::to_string(settings.y)); + } else if (InputFormatName() == "avfoundation") { + set_option(&options, "capture_cursor", settings.include_cursor ? "1" : "0"); } for (const auto& option : settings.options) { if (option.first == "input_format_name" || option.first == "use_device_defaults") { diff --git a/tests/ScreenCaptureReader.cpp b/tests/ScreenCaptureReader.cpp index 065dcdfc8..c12bc0890 100644 --- a/tests/ScreenCaptureReader.cpp +++ b/tests/ScreenCaptureReader.cpp @@ -31,7 +31,7 @@ TEST_CASE("Screen capture settings validation", "[libopenshot][screencaptureread settings.display = "desktop"; #elif defined(__APPLE__) settings.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; - settings.display = "1:none"; + settings.display = "Capture screen 0:none"; #endif #if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) settings.width = 640; @@ -62,7 +62,7 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ settings.display = "desktop"; #elif defined(__APPLE__) settings.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; - settings.display = "1:none"; + settings.display = "Capture screen 0:none"; #endif #if defined(__linux__) || defined(_WIN32) || defined(__APPLE__) settings.x = 10; @@ -86,7 +86,7 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ const Json::Value json = reader.JsonValue(); CHECK(json["type"].asString() == "ScreenCaptureReader"); #if defined(__APPLE__) - CHECK(json["display"].asString() == "1:none"); + CHECK(json["display"].asString() == "Capture screen 0:none"); #elif defined(_WIN32) CHECK(json["display"].asString() == "desktop"); #else From 9e8d226353cf91abf7986aadfc0a8add353caea2 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 22 Jun 2026 17:40:18 -0500 Subject: [PATCH 18/30] Preserve live capture timing gaps --- src/AudioRecorder.cpp | 20 +++++++++++++++++++ src/FFmpegWriter.cpp | 32 ++++++++++++++++++++++++++++++ src/FFmpegWriter.h | 8 ++++++++ src/WaylandScreenCaptureReader.cpp | 22 ++++++++++++++++++-- tests/FFmpegWriter.cpp | 26 ++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/AudioRecorder.cpp b/src/AudioRecorder.cpp index 3e6cabb9e..b901294c1 100644 --- a/src/AudioRecorder.cpp +++ b/src/AudioRecorder.cpp @@ -497,6 +497,7 @@ bool AudioRecorder::PopBlock(AudioRecorderBlock& block) void AudioRecorder::WriterLoop() { + int64_t expected_next_sample = -1; while (true) { AudioRecorderBlock block; if (!PopBlock(block)) { @@ -523,10 +524,29 @@ void AudioRecorder::WriterLoop() } if (writer) { + while (expected_next_sample >= 0 && block.first_sample > expected_next_sample) { + const int64_t missing_samples = block.first_sample - expected_next_sample; + const int silence_samples = static_cast(std::min( + missing_samples, + std::max(1, settings.buffer_size))); + AudioRecorderBlock silence_block; + silence_block.sample_rate = block.sample_rate; + silence_block.first_sample = expected_next_sample; + silence_block.channels.assign( + settings.channels, + std::vector(silence_samples, 0.0f)); + writer->WriteFrame(AudioRecordingFrameFactory::CreateFrame( + silence_block, + settings.channel_layout, + next_frame_number++)); + expected_next_sample += silence_samples; + } + writer->WriteFrame(AudioRecordingFrameFactory::CreateFrame( block, settings.channel_layout, next_frame_number++)); } + expected_next_sample = block.first_sample + block.Samples(); } } diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index a2667fb04..4ae40ec50 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -686,6 +686,38 @@ void FFmpegWriter::WriteFrame(std::shared_ptr frame) { last_frame = frame; } +void FFmpegWriter::WriteFrameAt(std::shared_ptr frame, int64_t frame_number) { + // Check for open reader (or throw exception) + if (!is_open) + throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path); + + if (frame_number < 1) { + frame_number = 1; + } + + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegWriter::WriteFrameAt", + "frame->number", frame->number, + "output_frame_number", frame_number, + "is_writing", is_writing); + + const int64_t previous_video_timestamp = video_timestamp; + if (info.has_video && video_st && video_codec_ctx) { + video_timestamp = av_rescale_q( + frame_number - 1, + av_make_q(info.fps.den, info.fps.num), + video_codec_ctx->time_base); + } + + write_frame(frame); + + if (!(info.has_video && video_st && video_codec_ctx)) { + video_timestamp = previous_video_timestamp; + } + + last_frame = frame; +} + // Write all frames in the queue to the video file. void FFmpegWriter::write_frame(std::shared_ptr frame) { // Flip writing flag diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index fdfd874d3..a3bc8923d 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -300,6 +300,14 @@ namespace openshot { /// \note This is an overloaded function. void WriteFrame(std::shared_ptr frame); + /// @brief Add a frame at a specific output video frame number. + /// @param frame The openshot::Frame object to write + /// @param frame_number The 1-based output video frame number used for video PTS + /// + /// \note Audio is still written sequentially. This overload is intended for + /// timestamped live video capture where missing frame numbers represent gaps. + void WriteFrameAt(std::shared_ptr frame, int64_t frame_number); + /// @brief Write a block of frames from a reader /// @param reader A openshot::ReaderBase object which will provide frames to be written /// @param start The starting frame number of the reader diff --git a/src/WaylandScreenCaptureReader.cpp b/src/WaylandScreenCaptureReader.cpp index 54a01108c..8147877f3 100644 --- a/src/WaylandScreenCaptureReader.cpp +++ b/src/WaylandScreenCaptureReader.cpp @@ -559,6 +559,10 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack uint8_t format_buffer[1024]; spa_pod_builder builder = SPA_POD_BUILDER_INIT(format_buffer, sizeof(format_buffer)); + const spa_fraction requested_framerate = SPA_FRACTION( + static_cast(std::max(1, settings.fps.num)), + static_cast(std::max(1, settings.fps.den))); + const spa_fraction variable_framerate = SPA_FRACTION(0, 1); const spa_pod* params[1]; params[0] = static_cast(spa_pod_builder_add_object( &builder, @@ -574,7 +578,11 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBA, SPA_VIDEO_FORMAT_BGRA, - SPA_VIDEO_FORMAT_RGBx))); + SPA_VIDEO_FORMAT_RGBx), + SPA_FORMAT_VIDEO_framerate, + SPA_POD_Fraction(&variable_framerate), + SPA_FORMAT_VIDEO_maxFramerate, + SPA_POD_Fraction(&requested_framerate))); pw_thread_loop_lock(thread_loop); const uint32_t target_id = stream_info.pipewire_serial > 0 ? PW_ID_ANY : stream_info.node_id; @@ -651,11 +659,21 @@ class WaylandScreenCaptureReader final : public ScreenCaptureReader::CaptureBack self->video_format = info.info.raw.format; self->stream_width = static_cast(info.info.raw.size.width); self->stream_height = static_cast(info.info.raw.size.height); + if (info.info.raw.framerate.num > 0 && info.info.raw.framerate.denom > 0) { + self->info.fps = Fraction( + static_cast(info.info.raw.framerate.num), + static_cast(info.info.raw.framerate.denom)); + self->info.video_timebase = self->info.fps.Reciprocal(); + } if (self->stream_width > 0 && self->stream_height > 0) { ZmqLogger::Instance()->Log( "Wayland PipeWire stream format: " + std::to_string(self->stream_width) + "x" + std::to_string(self->stream_height) + - " format=" + std::to_string(self->video_format)); + " format=" + std::to_string(self->video_format) + + " framerate=" + std::to_string(info.info.raw.framerate.num) + + "/" + std::to_string(info.info.raw.framerate.denom) + + " max_framerate=" + std::to_string(info.info.raw.max_framerate.num) + + "/" + std::to_string(info.info.raw.max_framerate.denom)); self->info.width = self->stream_width; self->info.height = self->stream_height; self->info.display_ratio = Fraction(self->stream_width, self->stream_height); diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 9d6b9db01..cb55a529a 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -280,6 +280,32 @@ TEST_CASE( "MP4_30fps_duration_exact_with_b_frames", "[libopenshot][ffmpegwriter avformat_close_input(&format_context); } +TEST_CASE( "WriteFrameAt_preserves_sparse_video_timestamps", "[libopenshot][ffmpegwriter][fps]" ) +{ + const std::string out_name = "WriteFrameAt_sparse_timestamps.mp4"; + DummyReader reader(Fraction(30, 1), 1280, 720, 0, 0, 1.0f); + reader.Open(); + + FFmpegWriter writer(out_name); + writer.SetVideoOptions(true, "libx264", Fraction(30, 1), 1280, 720, Fraction(1, 1), false, false, 15000000); + writer.Open(); + writer.WriteFrameAt(reader.GetFrame(1), 1); + writer.WriteFrameAt(reader.GetFrame(2), 2); + writer.WriteFrameAt(reader.GetFrame(3), 10); + writer.Close(); + reader.Close(); + + AVFormatContext* format_context = nullptr; + REQUIRE(avformat_open_input(&format_context, out_name.c_str(), nullptr, nullptr) == 0); + REQUIRE(avformat_find_stream_info(format_context, nullptr) >= 0); + AVStream* stream = first_video_stream(format_context); + REQUIRE(stream != nullptr); + + CHECK(stream->duration == av_rescale_q(10, av_make_q(1, 30), stream->time_base)); + + avformat_close_input(&format_context); +} + TEST_CASE( "SizeOrdering_x264_CRF", "[libopenshot][ffmpegwriter][filesize]" ) { std::stringstream path; From b575dbbc8e7e61246e5c31f569b7ab5f4110f987 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 23 Jun 2026 19:15:43 -0500 Subject: [PATCH 19/30] Fix recording waveform and mac screen crop capture --- src/AudioWaveformer.cpp | 3 ++ src/ScreenCaptureReader.cpp | 47 +++++++++++++++++++++++++--- tests/AudioRecorder.cpp | 62 +++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/AudioWaveformer.cpp b/src/AudioWaveformer.cpp index c50135a21..3c0a5eb10 100644 --- a/src/AudioWaveformer.cpp +++ b/src/AudioWaveformer.cpp @@ -329,6 +329,9 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames); f++) { std::shared_ptr frame = get_frame_with_retry(f); + if (!known_duration && frame && frame->number < f) { + break; + } int sample_count = frame->GetAudioSamplesCount(); if (sample_count <= 0) { if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) { diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 9b66b41d3..51f2cbd5f 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -40,6 +40,15 @@ namespace { av_dict_set(options, key.c_str(), value.c_str(), 0); } + + bool option_enabled(const std::map& options, const std::string& key) + { + const auto option = options.find(key); + if (option == options.end()) { + return false; + } + return option->second == "1" || option->second == "true" || option->second == "yes"; + } } #if defined(HAVE_WAYLAND_CAPTURE) @@ -260,9 +269,12 @@ void ScreenCaptureReader::OpenDevice() AVDictionary* options = nullptr; const bool use_device_defaults = settings.options.count("use_device_defaults") > 0; - if (!use_device_defaults) { + const bool post_capture_crop = option_enabled(settings.options, "crop_after_capture"); + if (!use_device_defaults && !post_capture_crop) { set_option(&options, "framerate", fraction_to_string(settings.fps)); set_option(&options, "video_size", std::to_string(settings.width) + "x" + std::to_string(settings.height)); + } else if (post_capture_crop) { + set_option(&options, "framerate", fraction_to_string(settings.fps)); } if (InputFormatName() == "x11grab") { set_option(&options, "draw_mouse", settings.include_cursor ? "1" : "0"); @@ -276,7 +288,7 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, "capture_cursor", settings.include_cursor ? "1" : "0"); } for (const auto& option : settings.options) { - if (option.first == "input_format_name" || option.first == "use_device_defaults") { + if (option.first == "input_format_name" || option.first == "use_device_defaults" || option.first == "crop_after_capture") { continue; } set_option(&options, option.first, option.second); @@ -318,6 +330,10 @@ void ScreenCaptureReader::OpenDecoder() info.width = codec_context->width > 0 ? codec_context->width : settings.width; info.height = codec_context->height > 0 ? codec_context->height : settings.height; + if (option_enabled(settings.options, "crop_after_capture")) { + info.width = settings.width; + info.height = settings.height; + } info.video_stream_index = video_stream; info.pixel_format = codec_context->pix_fmt; info.display_ratio = Fraction(info.width, info.height); @@ -417,8 +433,31 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) continue; } - auto frame = std::make_shared(number, width, height, "#000000"); - frame->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888, buffer); + int output_width = width; + int output_height = height; + unsigned char* output_buffer = buffer; + if (option_enabled(settings.options, "crop_after_capture")) { + const int crop_x = std::max(0, std::min(settings.x, width - 1)); + const int crop_y = std::max(0, std::min(settings.y, height - 1)); + output_width = std::max(1, std::min(settings.width, width - crop_x)); + output_height = std::max(1, std::min(settings.height, height - crop_y)); + const size_t output_buffer_size = static_cast(output_width) * output_height * bytes_per_pixel; + output_buffer = static_cast(aligned_malloc(output_buffer_size)); + if (!output_buffer) { + openshot::aligned_free(buffer); + throw OutOfMemory("Unable to allocate cropped capture frame buffer.", InputName()); + } + for (int row = 0; row < output_height; ++row) { + const unsigned char* source_row = buffer + + (static_cast(crop_y + row) * width + crop_x) * bytes_per_pixel; + unsigned char* dest_row = output_buffer + static_cast(row) * output_width * bytes_per_pixel; + std::copy(source_row, source_row + static_cast(output_width) * bytes_per_pixel, dest_row); + } + openshot::aligned_free(buffer); + } + + auto frame = std::make_shared(number, output_width, output_height, "#000000"); + frame->AddImage(output_width, output_height, bytes_per_pixel, QImage::Format_RGBA8888, output_buffer); frames_read++; return frame; } diff --git a/tests/AudioRecorder.cpp b/tests/AudioRecorder.cpp index 4cb254e9f..b8056aca1 100644 --- a/tests/AudioRecorder.cpp +++ b/tests/AudioRecorder.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include +#include #include #include @@ -148,3 +151,62 @@ TEST_CASE("Audio recorder frames write through FFmpegWriter", "[libopenshot][aud CHECK(frame->GetAudioSamplesCount() > 0); reader.Close(); } + +TEST_CASE("Audio recorder silent frames produce readable zero waveform", "[libopenshot][audiorecorder][ffmpegwriter][audiowaveformer]") +{ + const std::string path = "AudioRecorder-silent-output.wav"; + std::remove(path.c_str()); + + const int sample_rate = 8000; + const int samples = sample_rate / 10; + std::vector silence(samples, 0.0f); + + FFmpegWriter writer(path); + writer.SetAudioOptions(true, "pcm_s16le", sample_rate, 1, LAYOUT_MONO, 128000); + writer.Open(); + writer.WriteFrame(AudioRecordingFrameFactory::CreateFrame(MakeBlock(sample_rate, 0, silence), LAYOUT_MONO, 1)); + writer.Close(); + + FFmpegReader reader(path); + reader.Open(); + REQUIRE(reader.info.has_audio); + CHECK(reader.info.duration > 0.05f); + + AudioWaveformer waveformer(&reader); + AudioWaveformData waveform = waveformer.ExtractSamples(-1, 20, true); + + REQUIRE_FALSE(waveform.max_samples.empty()); + CHECK(waveform.max_samples.size() == waveform.rms_samples.size()); + CHECK(*std::max_element(waveform.max_samples.begin(), waveform.max_samples.end()) == Approx(0.0f)); + CHECK(*std::max_element(waveform.rms_samples.begin(), waveform.rms_samples.end()) == Approx(0.0f)); + + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE("Audio recorder empty writer output has no waveform samples", "[libopenshot][audiorecorder][ffmpegwriter][audiowaveformer]") +{ + const std::string path = "AudioRecorder-empty-output.wav"; + std::remove(path.c_str()); + + FFmpegWriter writer(path); + writer.SetAudioOptions(true, "pcm_s16le", 8000, 1, LAYOUT_MONO, 128000); + writer.Open(); + writer.Close(); + + FFmpegReader reader(path); + reader.Open(); + + AudioWaveformer waveformer(&reader); + auto future_waveform = std::async(std::launch::async, [&]() { + return waveformer.ExtractSamples(-1, 20, true); + }); + + REQUIRE(future_waveform.wait_for(std::chrono::seconds(2)) == std::future_status::ready); + AudioWaveformData waveform = future_waveform.get(); + CHECK(waveform.max_samples.empty()); + CHECK(waveform.rms_samples.empty()); + + reader.Close(); + std::remove(path.c_str()); +} From 7a71278da5b73691d5bcb13191bff6caca464894 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 23 Jun 2026 21:21:08 -0500 Subject: [PATCH 20/30] Use full display size for mac crop capture --- src/ScreenCaptureReader.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 51f2cbd5f..8fbb8acc4 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -49,6 +49,19 @@ namespace } return option->second == "1" || option->second == "true" || option->second == "yes"; } + + int option_int(const std::map& options, const std::string& key, int fallback) + { + const auto option = options.find(key); + if (option == options.end()) { + return fallback; + } + try { + return std::stoi(option->second); + } catch (...) { + return fallback; + } + } } #if defined(HAVE_WAYLAND_CAPTURE) @@ -274,7 +287,10 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, "framerate", fraction_to_string(settings.fps)); set_option(&options, "video_size", std::to_string(settings.width) + "x" + std::to_string(settings.height)); } else if (post_capture_crop) { + const int source_width = option_int(settings.options, "crop_source_width", settings.width); + const int source_height = option_int(settings.options, "crop_source_height", settings.height); set_option(&options, "framerate", fraction_to_string(settings.fps)); + set_option(&options, "video_size", std::to_string(source_width) + "x" + std::to_string(source_height)); } if (InputFormatName() == "x11grab") { set_option(&options, "draw_mouse", settings.include_cursor ? "1" : "0"); @@ -288,7 +304,11 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, "capture_cursor", settings.include_cursor ? "1" : "0"); } for (const auto& option : settings.options) { - if (option.first == "input_format_name" || option.first == "use_device_defaults" || option.first == "crop_after_capture") { + if (option.first == "input_format_name" + || option.first == "use_device_defaults" + || option.first == "crop_after_capture" + || option.first == "crop_source_width" + || option.first == "crop_source_height") { continue; } set_option(&options, option.first, option.second); From b18c147c69987a095a2f1de65baf712ef51b749b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 23 Jun 2026 22:28:55 -0500 Subject: [PATCH 21/30] Interrupt blocking screen capture reads on close --- src/ScreenCaptureReader.cpp | 16 ++++++++++++++++ src/ScreenCaptureReader.h | 2 ++ 2 files changed, 18 insertions(+) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 8fbb8acc4..4f157bebf 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -62,6 +62,12 @@ namespace return fallback; } } + + int capture_interrupt_callback(void* opaque) + { + const auto* reader = static_cast*>(opaque); + return reader && reader->load() ? 1 : 0; + } } #if defined(HAVE_WAYLAND_CAPTURE) @@ -83,6 +89,7 @@ ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settin , rgba_frame(nullptr) , packet(nullptr) , sws_context(nullptr) + , close_requested(false) { if (settings.backend == SCREEN_CAPTURE_AUTO) { settings.backend = DefaultBackend(); @@ -266,6 +273,7 @@ void ScreenCaptureReader::Open() if (is_open) { return; } + close_requested = false; OpenDevice(); OpenDecoder(); is_open = true; @@ -280,6 +288,13 @@ void ScreenCaptureReader::OpenDevice() throw InvalidOptions("FFmpeg input device is not available: " + InputFormatName(), InputName()); } + format_context = avformat_alloc_context(); + if (!format_context) { + throw OutOfMemory("Unable to allocate capture format context.", InputName()); + } + format_context->interrupt_callback.callback = capture_interrupt_callback; + format_context->interrupt_callback.opaque = &close_requested; + AVDictionary* options = nullptr; const bool use_device_defaults = settings.options.count("use_device_defaults") > 0; const bool post_capture_crop = option_enabled(settings.options, "crop_after_capture"); @@ -487,6 +502,7 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) void ScreenCaptureReader::Close() { + close_requested = true; if (backend_reader) { backend_reader->Close(); } diff --git a/src/ScreenCaptureReader.h b/src/ScreenCaptureReader.h index 0e04bcc3e..003328d3d 100644 --- a/src/ScreenCaptureReader.h +++ b/src/ScreenCaptureReader.h @@ -15,6 +15,7 @@ #include "ReaderBase.h" +#include #include #include #include @@ -114,6 +115,7 @@ namespace openshot AVFrame* rgba_frame; AVPacket* packet; SwsContext* sws_context; + std::atomic close_requested; }; } From 98f797d4050e51b4e3739c51e8f72a5e9a6e1314 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 25 Jun 2026 23:14:34 -0500 Subject: [PATCH 22/30] Add timer effect --- bindings/java/openshot.i | 1 + bindings/python/openshot.i | 1 + bindings/ruby/openshot.i | 1 + src/CMakeLists.txt | 1 + src/EffectInfo.cpp | 4 + src/Effects.h | 1 + src/effects/Timer.cpp | 555 +++++++++++++++++++++++++++++++++++++ src/effects/Timer.h | 100 +++++++ tests/Benchmark.cpp | 27 ++ tests/CMakeLists.txt | 3 +- tests/Timer.cpp | 303 ++++++++++++++++++++ 11 files changed, 996 insertions(+), 1 deletion(-) create mode 100644 src/effects/Timer.cpp create mode 100644 src/effects/Timer.h create mode 100644 tests/Timer.cpp diff --git a/bindings/java/openshot.i b/bindings/java/openshot.i index 660112d0e..9f8d1794f 100644 --- a/bindings/java/openshot.i +++ b/bindings/java/openshot.i @@ -263,6 +263,7 @@ typedef struct OpenShotByteBuffer { %include "effects/Sharpen.h" %include "effects/Shift.h" %include "effects/SphericalProjection.cpp" +%include "effects/Timer.h" %include "effects/DenoiseImage.h" %include "effects/Wave.h" #ifdef USE_OPENCV diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index f90ec070c..73d3da46e 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -557,6 +557,7 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "effects/Sharpen.h" %include "effects/Shift.h" %include "effects/SphericalProjection.cpp" +%include "effects/Timer.h" %include "effects/DenoiseImage.h" %include "effects/Wave.h" #ifdef USE_OPENCV diff --git a/bindings/ruby/openshot.i b/bindings/ruby/openshot.i index 0d4c4846c..566a84b13 100644 --- a/bindings/ruby/openshot.i +++ b/bindings/ruby/openshot.i @@ -297,5 +297,6 @@ typedef struct OpenShotByteBuffer { %include "effects/Pixelate.h" %include "effects/Saturation.h" %include "effects/Shift.h" +%include "effects/Timer.h" %include "effects/DenoiseImage.h" %include "effects/Wave.h" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f995e51c..82387dc3f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -156,6 +156,7 @@ set(EFFECTS_SOURCES effects/Shadow.cpp effects/Shift.cpp effects/SphericalProjection.cpp + effects/Timer.cpp effects/DenoiseImage.cpp effects/Wave.cpp audio_effects/STFT.cpp diff --git a/src/EffectInfo.cpp b/src/EffectInfo.cpp index 87e417dca..76f389a97 100644 --- a/src/EffectInfo.cpp +++ b/src/EffectInfo.cpp @@ -104,6 +104,9 @@ EffectBase* EffectInfo::CreateEffect(std::string effect_type) { else if (effect_type == "SphericalProjection") return new SphericalProjection(); + else if (effect_type == "Timer") + return new Timer(); + else if (effect_type == "DenoiseImage") return new DenoiseImage(); @@ -190,6 +193,7 @@ Json::Value EffectInfo::JsonValue() { root.append(Shadow().JsonInfo()); root.append(Shift().JsonInfo()); root.append(SphericalProjection().JsonInfo()); + root.append(Timer().JsonInfo()); root.append(DenoiseImage().JsonInfo()); root.append(Wave().JsonInfo()); /* Audio */ diff --git a/src/Effects.h b/src/Effects.h index af19ed7ea..f5b097083 100644 --- a/src/Effects.h +++ b/src/Effects.h @@ -40,6 +40,7 @@ #include "effects/Shadow.h" #include "effects/SphericalProjection.h" #include "effects/Shift.h" +#include "effects/Timer.h" #include "effects/DenoiseImage.h" #include "effects/Wave.h" diff --git a/src/effects/Timer.cpp b/src/effects/Timer.cpp new file mode 100644 index 000000000..483d270d9 --- /dev/null +++ b/src/effects/Timer.cpp @@ -0,0 +1,555 @@ +/** + * @file + * @brief Source file for Timer effect class + * @author OpenShot Studios, LLC + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "Timer.h" +#include "Exceptions.h" +#include "../Clip.h" +#include "../Timeline.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace openshot; + +namespace { + int ClampGravity(int gravity) + { + if (gravity < GRAVITY_TOP_LEFT || gravity > GRAVITY_BOTTOM_RIGHT) + return GRAVITY_BOTTOM; + return gravity; + } +} + +Timer::Timer() : + mode(TIMER_MODE_COUNT_UP), + time_source(TIMER_TIME_SOURCE), + format(TIMER_FORMAT_MM_SS), + clamp(1), + gravity(GRAVITY_BOTTOM), + show_background(1), + font_name("sans"), + prefix(""), + suffix(""), + color("#ffffff"), + stroke("#000000"), + background("#000000"), + start_time(0.0), + end_time(0.0), + font_size(48.0), + font_alpha(1.0), + stroke_width(2.0), + x_offset(0.0), + y_offset(0.0), + background_alpha(0.45), + background_padding(14.0), + background_corner(6.0) +{ + init_effect_details(); +} + +void Timer::init_effect_details() +{ + InitEffectInfo(); + info.class_name = "Timer"; + info.name = "Timer"; + info.description = "Render a styled count up, count down, clock, timecode, or frame number overlay."; + info.has_audio = false; + info.has_video = true; +} + +double Timer::ResolveFps() const +{ + Clip* clip = (Clip*) const_cast(this)->ParentClip(); + Timeline* timeline = NULL; + + if (clip && clip->ParentTimeline() != NULL) { + timeline = (Timeline*) clip->ParentTimeline(); + } else if (const_cast(this)->ParentTimeline() != NULL) { + timeline = (Timeline*) const_cast(this)->ParentTimeline(); + } + + if (time_source == TIMER_TIME_SOURCE && clip != NULL && clip->Reader() != NULL && clip->Reader()->info.fps.ToDouble() > 0.0) + return clip->Reader()->info.fps.ToDouble(); + if (timeline != NULL && timeline->info.fps.ToDouble() > 0.0) + return timeline->info.fps.ToDouble(); + if (clip != NULL && clip->Reader() != NULL && clip->Reader()->info.fps.ToDouble() > 0.0) + return clip->Reader()->info.fps.ToDouble(); + return 30.0; +} + +int64_t Timer::EffectiveFrameNumber(int64_t frame_number) const +{ + if (time_source != TIMER_TIME_SOURCE) + return std::max(1, frame_number); + + Clip* clip = (Clip*) const_cast(this)->ParentClip(); + if (clip != NULL && clip->time.GetLength() > 1) + return std::max(1, clip->time.GetLong(frame_number)); + + return std::max(1, frame_number); +} + +double Timer::CountdownDuration(int64_t frame_number) const +{ + const double configured_duration = end_time.GetValue(frame_number); + if (configured_duration > 0.0) + return configured_duration; + + Clip* clip = (Clip*) const_cast(this)->ParentClip(); + if (clip != NULL) + return std::max(0.0f, clip->End() - clip->Start()); + + return 0.0; +} + +std::string Timer::FormatSeconds(double seconds, double fps, bool duration_style) const +{ + if (duration_style && clamp && seconds < 0.0) + seconds = 0.0; + + const bool negative = seconds < 0.0; + const int64_t total_milliseconds = std::llround(std::fabs(seconds) * 1000.0); + const int milliseconds = static_cast(total_milliseconds % 1000); + const int64_t total_seconds = total_milliseconds / 1000; + const int hours = static_cast(total_seconds / 3600); + const int minutes = static_cast((total_seconds / 60) % 60); + const int whole_seconds = static_cast(total_seconds % 60); + + std::ostringstream text; + if (negative) + text << "-"; + + if (format == TIMER_FORMAT_HH_MM_SS_MILLISECONDS) { + text << std::setfill('0') << std::setw(2) << hours << ":" + << std::setw(2) << minutes << ":" + << std::setw(2) << whole_seconds << "." + << std::setw(3) << milliseconds; + } else if (format == TIMER_FORMAT_HH_MM_SS) { + text << std::setfill('0') << std::setw(2) << hours << ":" + << std::setw(2) << minutes << ":" + << std::setw(2) << whole_seconds; + } else if (format == TIMER_FORMAT_TIMECODE) { + text << FormatTimecode(seconds, fps); + } else if (format == TIMER_FORMAT_FRAMES) { + text << static_cast(std::llround(seconds * fps)); + } else { + const int total_minutes = hours * 60 + minutes; + text << std::setfill('0') << std::setw(2) << total_minutes << ":" + << std::setw(2) << whole_seconds; + } + + return text.str(); +} + +std::string Timer::FormatTimecode(double seconds, double fps) const +{ + const bool negative = seconds < 0.0; + const int64_t total_frames = std::llround(std::fabs(seconds) * fps); + const int64_t rounded_fps = std::max(1, std::llround(fps)); + const int64_t frames = total_frames % rounded_fps; + const int64_t total_seconds = total_frames / rounded_fps; + const int64_t secs = total_seconds % 60; + const int64_t mins = (total_seconds / 60) % 60; + const int64_t hours = total_seconds / 3600; + + std::ostringstream text; + if (negative) + text << "-"; + text << std::setfill('0') << std::setw(2) << hours << ":" + << std::setw(2) << mins << ":" + << std::setw(2) << secs << ":" + << std::setw(2) << frames; + return text.str(); +} + +double Timer::TimerSeconds(int64_t frame_number) const +{ + const double fps = ResolveFps(); + const int64_t effective_frame = EffectiveFrameNumber(frame_number); + const double elapsed = static_cast(effective_frame - 1) / fps; + + if (mode == TIMER_MODE_COUNT_DOWN) + return CountdownDuration(frame_number) - start_time.GetValue(frame_number) - elapsed; + + return start_time.GetValue(frame_number) + elapsed; +} + +std::string Timer::TimerText(int64_t frame_number) const +{ + const double fps = ResolveFps(); + const int64_t effective_frame = EffectiveFrameNumber(frame_number); + std::ostringstream text; + text << prefix; + + if (mode == TIMER_MODE_FRAME_NUMBER || format == TIMER_FORMAT_FRAMES) { + const int64_t frame_offset = std::llround(start_time.GetValue(frame_number) * fps); + text << (effective_frame + frame_offset); + } else if (mode == TIMER_MODE_TIMECODE || format == TIMER_FORMAT_TIMECODE) { + text << FormatTimecode(TimerSeconds(frame_number), fps); + } else if (mode == TIMER_MODE_CLOCK) { + double seconds = std::fmod(TimerSeconds(frame_number), 24.0 * 60.0 * 60.0); + if (seconds < 0.0) + seconds += 24.0 * 60.0 * 60.0; + text << FormatSeconds(seconds, fps, false); + } else { + text << FormatSeconds(TimerSeconds(frame_number), fps, true); + } + + text << suffix; + return text.str(); +} + +std::string Timer::TimerLayoutText(int64_t frame_number) const +{ + if (mode == TIMER_MODE_FRAME_NUMBER || format == TIMER_FORMAT_FRAMES) + return TimerText(frame_number); + + std::string timer_template; + if (mode == TIMER_MODE_TIMECODE || format == TIMER_FORMAT_TIMECODE) + timer_template = "88:88:88:88"; + else if (format == TIMER_FORMAT_HH_MM_SS_MILLISECONDS) + timer_template = "88:88:88.888"; + else if (format == TIMER_FORMAT_HH_MM_SS) + timer_template = "88:88:88"; + else + timer_template = "88:88"; + + return prefix + timer_template + suffix; +} + +std::shared_ptr Timer::GetFrame(std::shared_ptr frame, int64_t frame_number) +{ + Clip* clip = (Clip*) ParentClip(); + Timeline* timeline = NULL; + QSize image_size(1280, 720); + + if (clip && clip->ParentTimeline() != NULL) { + timeline = (Timeline*) clip->ParentTimeline(); + } else if (this->ParentTimeline() != NULL) { + timeline = (Timeline*) this->ParentTimeline(); + } + + if (timeline != NULL) { + image_size = QSize(timeline->info.width, timeline->info.height); + } else if (clip != NULL && clip->Reader() != NULL) { + image_size = QSize(clip->Reader()->info.width, clip->Reader()->info.height); + } + + if (!frame->has_image_data) + frame->AddColor(image_size.width(), image_size.height(), "#00000000"); + + std::shared_ptr frame_image = frame->GetImage(); + if (!frame_image || frame_image->isNull()) + return frame; + + const double scale_factor = frame_image->width() / 600.0; + const double font_size_value = std::max(1.0, font_size.GetValue(frame_number) * scale_factor); + const double stroke_width_value = std::max(0.0, stroke_width.GetValue(frame_number) * scale_factor); + const double padding_value = std::max(0.0, background_padding.GetValue(frame_number) * scale_factor); + const double corner_value = std::max(0.0, background_corner.GetValue(frame_number) * scale_factor); + const QString timer_text = QString::fromStdString(TimerText(frame_number)); + const QString layout_text = QString::fromStdString(TimerLayoutText(frame_number)); + + QFont font(QString::fromStdString(font_name), int(font_size_value)); + font.setPixelSize(font_size_value); + QFontMetricsF metrics(font); + QRectF text_bounds = metrics.tightBoundingRect(timer_text); + if (text_bounds.isEmpty()) + text_bounds = metrics.boundingRect(timer_text); + QRectF layout_bounds = metrics.tightBoundingRect(layout_text); + if (layout_bounds.isEmpty()) + layout_bounds = metrics.boundingRect(layout_text); + + const double text_width = std::max(1.0, text_bounds.width()); + const double text_height = std::max(1.0, text_bounds.height()); + double digit_slot_width = 0.0; + for (int digit = 0; digit <= 9; ++digit) + digit_slot_width = std::max(digit_slot_width, metrics.horizontalAdvance(QString::number(digit))); + + const bool use_slot_layout = timer_text.size() == layout_text.size() && + !(mode == TIMER_MODE_FRAME_NUMBER || format == TIMER_FORMAT_FRAMES); + std::vector slot_widths; + double layout_width = 0.0; + if (use_slot_layout) { + slot_widths.reserve(layout_text.size()); + for (int index = 0; index < layout_text.size(); ++index) { + const QString layout_char(layout_text[index]); + const double slot_width = layout_text[index].isDigit() ? digit_slot_width : metrics.horizontalAdvance(layout_char); + slot_widths.push_back(slot_width); + layout_width += slot_width; + } + } else { + layout_width = std::max(text_width, layout_bounds.width()); + } + layout_width = std::max(1.0, layout_width); + const double rendered_box_width = layout_width + (padding_value * 2.0); + const double box_height = std::max(text_height, layout_bounds.height()) + (padding_value * 2.0); + + double x = padding_value; + double y = 0.0; + const int resolved_gravity = ClampGravity(gravity); + if (resolved_gravity == GRAVITY_TOP || resolved_gravity == GRAVITY_CENTER || resolved_gravity == GRAVITY_BOTTOM) + x = (frame_image->width() - rendered_box_width) / 2.0; + else if (resolved_gravity == GRAVITY_TOP_RIGHT || resolved_gravity == GRAVITY_RIGHT || resolved_gravity == GRAVITY_BOTTOM_RIGHT) + x = frame_image->width() - rendered_box_width - padding_value; + + switch (resolved_gravity) { + case GRAVITY_TOP_LEFT: + case GRAVITY_TOP: + case GRAVITY_TOP_RIGHT: + break; + case GRAVITY_LEFT: + case GRAVITY_CENTER: + case GRAVITY_RIGHT: + y = (frame_image->height() - box_height) / 2.0; + break; + case GRAVITY_BOTTOM_LEFT: + case GRAVITY_BOTTOM: + case GRAVITY_BOTTOM_RIGHT: + y = frame_image->height() - box_height; + break; + default: + break; + } + x += frame_image->width() * (x_offset.GetValue(frame_number) / 100.0); + y += frame_image->height() * (y_offset.GetValue(frame_number) / 100.0); + x = std::round(x); + y = std::round(y); + + double text_x = x + padding_value - text_bounds.left(); + if (resolved_gravity == GRAVITY_TOP || resolved_gravity == GRAVITY_CENTER || resolved_gravity == GRAVITY_BOTTOM) + text_x = x + padding_value + ((layout_width - text_width) / 2.0) - text_bounds.left(); + else if (resolved_gravity == GRAVITY_TOP_RIGHT || resolved_gravity == GRAVITY_RIGHT || resolved_gravity == GRAVITY_BOTTOM_RIGHT) + text_x = x + padding_value + layout_width - text_width - text_bounds.left(); + text_x = std::round(text_x); + const double text_y = std::round(y + padding_value - text_bounds.top()); + + QPainter painter(frame_image.get()); + painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true); + painter.setCompositionMode(QPainter::CompositionMode_SourceOver); + + if (show_background) { + QColor background_qcolor(QString::fromStdString(background.GetColorHex(frame_number))); + background_qcolor.setAlphaF(std::max(0.0, std::min(1.0, background_alpha.GetValue(frame_number)))); + painter.setPen(Qt::NoPen); + painter.setBrush(QBrush(background_qcolor)); + painter.drawRoundedRect(QRectF(x, y, rendered_box_width, box_height), corner_value, corner_value); + } + + QPainterPath path; + if (use_slot_layout) { + double cursor_x = x + padding_value; + for (int index = 0; index < timer_text.size(); ++index) { + const QString text_char(timer_text[index]); + QRectF char_bounds = metrics.tightBoundingRect(text_char); + if (char_bounds.isEmpty()) + char_bounds = metrics.boundingRect(text_char); + + const double char_x = std::round(cursor_x); + path.addText(QPointF(char_x - char_bounds.left(), text_y), font, text_char); + cursor_x += slot_widths[index]; + } + } else { + path.addText(QPointF(text_x, text_y), font, timer_text); + } + + QColor stroke_qcolor(QString::fromStdString(stroke.GetColorHex(frame_number))); + stroke_qcolor.setAlphaF(std::max(0.0, std::min(1.0, font_alpha.GetValue(frame_number)))); + QPen pen(stroke_qcolor); + pen.setWidthF(stroke_width_value); + painter.setPen(stroke_width_value <= 0.0 ? Qt::NoPen : pen); + + QColor font_qcolor(QString::fromStdString(color.GetColorHex(frame_number))); + font_qcolor.setAlphaF(std::max(0.0, std::min(1.0, font_alpha.GetValue(frame_number)))); + painter.setBrush(QBrush(font_qcolor)); + painter.drawPath(path); + painter.end(); + + return frame; +} + +std::string Timer::Json() const +{ + return JsonValue().toStyledString(); +} + +Json::Value Timer::JsonValue() const +{ + Json::Value root = EffectBase::JsonValue(); + root["type"] = info.class_name; + root["mode"] = mode; + root["time_source"] = time_source; + root["format"] = format; + root["clamp"] = clamp; + root["gravity"] = ClampGravity(gravity); + root["show_background"] = show_background; + root["font_name"] = font_name; + root["prefix"] = prefix; + root["suffix"] = suffix; + root["color"] = color.JsonValue(); + root["stroke"] = stroke.JsonValue(); + root["background"] = background.JsonValue(); + root["start_time"] = start_time.JsonValue(); + root["end_time"] = end_time.JsonValue(); + root["font_size"] = font_size.JsonValue(); + root["font_alpha"] = font_alpha.JsonValue(); + root["stroke_width"] = stroke_width.JsonValue(); + root["x_offset"] = x_offset.JsonValue(); + root["y_offset"] = y_offset.JsonValue(); + root["background_alpha"] = background_alpha.JsonValue(); + root["background_padding"] = background_padding.JsonValue(); + root["background_corner"] = background_corner.JsonValue(); + return root; +} + +void Timer::SetJson(const std::string value) +{ + try + { + const Json::Value root = openshot::stringToJson(value); + SetJsonValue(root); + } + catch (const std::exception& e) + { + throw InvalidJSON("JSON is invalid (missing keys or invalid data types)"); + } +} + +void Timer::SetJsonValue(const Json::Value root) +{ + EffectBase::SetJsonValue(root); + + if (!root["mode"].isNull()) + mode = root["mode"].asInt(); + if (!root["time_source"].isNull()) + time_source = root["time_source"].asInt(); + if (!root["format"].isNull()) + format = root["format"].asInt(); + if (!root["clamp"].isNull()) + clamp = root["clamp"].asInt(); + if (!root["gravity"].isNull()) + gravity = ClampGravity(root["gravity"].asInt()); + if (!root["show_background"].isNull()) + show_background = root["show_background"].asInt(); + if (!root["font_name"].isNull()) + font_name = root["font_name"].asString(); + if (!root["prefix"].isNull()) + prefix = root["prefix"].asString(); + if (!root["suffix"].isNull()) + suffix = root["suffix"].asString(); + if (!root["color"].isNull()) + color.SetJsonValue(root["color"]); + if (!root["stroke"].isNull()) + stroke.SetJsonValue(root["stroke"]); + if (!root["background"].isNull()) + background.SetJsonValue(root["background"]); + if (!root["start_time"].isNull()) + start_time.SetJsonValue(root["start_time"]); + if (!root["end_time"].isNull()) + end_time.SetJsonValue(root["end_time"]); + if (!root["font_size"].isNull()) + font_size.SetJsonValue(root["font_size"]); + if (!root["font_alpha"].isNull()) + font_alpha.SetJsonValue(root["font_alpha"]); + if (!root["stroke_width"].isNull()) + stroke_width.SetJsonValue(root["stroke_width"]); + if (!root["x_offset"].isNull()) + x_offset.SetJsonValue(root["x_offset"]); + if (!root["y_offset"].isNull()) + y_offset.SetJsonValue(root["y_offset"]); + if (!root["background_alpha"].isNull()) + background_alpha.SetJsonValue(root["background_alpha"]); + if (!root["background_padding"].isNull()) + background_padding.SetJsonValue(root["background_padding"]); + if (!root["background_corner"].isNull()) + background_corner.SetJsonValue(root["background_corner"]); +} + +std::string Timer::PropertiesJSON(int64_t requested_frame) const +{ + Json::Value root = BasePropertiesJSON(requested_frame); + + root["mode"] = add_property_json("Mode", mode, "int", "", NULL, TIMER_MODE_COUNT_UP, TIMER_MODE_FRAME_NUMBER, false, requested_frame); + root["mode"]["choices"].append(add_property_choice_json("Count Up", TIMER_MODE_COUNT_UP, mode)); + root["mode"]["choices"].append(add_property_choice_json("Count Down", TIMER_MODE_COUNT_DOWN, mode)); + root["mode"]["choices"].append(add_property_choice_json("Clock", TIMER_MODE_CLOCK, mode)); + root["mode"]["choices"].append(add_property_choice_json("Timecode", TIMER_MODE_TIMECODE, mode)); + root["mode"]["choices"].append(add_property_choice_json("Frame Number", TIMER_MODE_FRAME_NUMBER, mode)); + root["time_source"] = add_property_json("Time Source", time_source, "int", "", NULL, TIMER_TIME_CLIP, TIMER_TIME_SOURCE, false, requested_frame); + root["time_source"]["choices"].append(add_property_choice_json("Clip Time", TIMER_TIME_CLIP, time_source)); + root["time_source"]["choices"].append(add_property_choice_json("Source Time", TIMER_TIME_SOURCE, time_source)); + root["format"] = add_property_json("Format", format, "int", "", NULL, TIMER_FORMAT_MM_SS, TIMER_FORMAT_FRAMES, false, requested_frame); + root["format"]["choices"].append(add_property_choice_json("MM:SS", TIMER_FORMAT_MM_SS, format)); + root["format"]["choices"].append(add_property_choice_json("HH:MM:SS", TIMER_FORMAT_HH_MM_SS, format)); + root["format"]["choices"].append(add_property_choice_json("HH:MM:SS.mmm", TIMER_FORMAT_HH_MM_SS_MILLISECONDS, format)); + root["format"]["choices"].append(add_property_choice_json("Timecode", TIMER_FORMAT_TIMECODE, format)); + root["format"]["choices"].append(add_property_choice_json("Frames", TIMER_FORMAT_FRAMES, format)); + root["start_time"] = add_property_json("Start Time", start_time.GetValue(requested_frame), "float", "", &start_time, -86400.0, 86400.0, false, requested_frame); + root["end_time"] = add_property_json("Countdown Duration", end_time.GetValue(requested_frame), "float", "", &end_time, 0.0, 86400.0, false, requested_frame); + root["clamp"] = add_property_json("Clamp at Zero", clamp, "int", "", NULL, 0, 1, false, requested_frame); + root["clamp"]["choices"].append(add_property_choice_json("Yes", 1, clamp)); + root["clamp"]["choices"].append(add_property_choice_json("No", 0, clamp)); + root["prefix"] = add_property_json("Prefix", 0.0, "string", prefix, NULL, -1, -1, false, requested_frame); + root["suffix"] = add_property_json("Suffix", 0.0, "string", suffix, NULL, -1, -1, false, requested_frame); + root["font_name"] = add_property_json("Font", 0.0, "font", font_name, NULL, -1, -1, false, requested_frame); + root["font_size"] = add_property_json("Font Size", font_size.GetValue(requested_frame), "float", "", &font_size, 1.0, 300.0, false, requested_frame); + root["font_alpha"] = add_property_json("Font Alpha", font_alpha.GetValue(requested_frame), "float", "", &font_alpha, 0.0, 1.0, false, requested_frame); + root["color"] = add_property_json("Text Color", 0.0, "color", "", &color.red, 0, 255, false, requested_frame); + root["color"]["red"] = add_property_json("Red", color.red.GetValue(requested_frame), "float", "", &color.red, 0, 255, false, requested_frame); + root["color"]["blue"] = add_property_json("Blue", color.blue.GetValue(requested_frame), "float", "", &color.blue, 0, 255, false, requested_frame); + root["color"]["green"] = add_property_json("Green", color.green.GetValue(requested_frame), "float", "", &color.green, 0, 255, false, requested_frame); + root["stroke"] = add_property_json("Stroke Color", 0.0, "color", "", &stroke.red, 0, 255, false, requested_frame); + root["stroke"]["red"] = add_property_json("Red", stroke.red.GetValue(requested_frame), "float", "", &stroke.red, 0, 255, false, requested_frame); + root["stroke"]["blue"] = add_property_json("Blue", stroke.blue.GetValue(requested_frame), "float", "", &stroke.blue, 0, 255, false, requested_frame); + root["stroke"]["green"] = add_property_json("Green", stroke.green.GetValue(requested_frame), "float", "", &stroke.green, 0, 255, false, requested_frame); + root["stroke_width"] = add_property_json("Stroke Width", stroke_width.GetValue(requested_frame), "float", "", &stroke_width, 0.0, 20.0, false, requested_frame); + const int resolved_gravity = ClampGravity(gravity); + root["gravity"] = add_property_json("Gravity", resolved_gravity, "int", "", NULL, GRAVITY_TOP_LEFT, GRAVITY_BOTTOM_RIGHT, false, requested_frame); + root["gravity"]["choices"].append(add_property_choice_json("Top Left", GRAVITY_TOP_LEFT, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Top Center", GRAVITY_TOP, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Top Right", GRAVITY_TOP_RIGHT, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Left", GRAVITY_LEFT, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Center", GRAVITY_CENTER, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Right", GRAVITY_RIGHT, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Bottom Left", GRAVITY_BOTTOM_LEFT, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Bottom Center", GRAVITY_BOTTOM, resolved_gravity)); + root["gravity"]["choices"].append(add_property_choice_json("Bottom Right", GRAVITY_BOTTOM_RIGHT, resolved_gravity)); + root["x_offset"] = add_property_json("X Offset (%)", x_offset.GetValue(requested_frame), "float", "", &x_offset, -100.0, 100.0, false, requested_frame); + root["y_offset"] = add_property_json("Y Offset (%)", y_offset.GetValue(requested_frame), "float", "", &y_offset, -100.0, 100.0, false, requested_frame); + root["show_background"] = add_property_json("Show Background", show_background, "int", "", NULL, 0, 1, false, requested_frame); + root["show_background"]["choices"].append(add_property_choice_json("Yes", 1, show_background)); + root["show_background"]["choices"].append(add_property_choice_json("No", 0, show_background)); + root["background"] = add_property_json("Background Color", 0.0, "color", "", &background.red, 0, 255, false, requested_frame); + root["background"]["red"] = add_property_json("Red", background.red.GetValue(requested_frame), "float", "", &background.red, 0, 255, false, requested_frame); + root["background"]["blue"] = add_property_json("Blue", background.blue.GetValue(requested_frame), "float", "", &background.blue, 0, 255, false, requested_frame); + root["background"]["green"] = add_property_json("Green", background.green.GetValue(requested_frame), "float", "", &background.green, 0, 255, false, requested_frame); + root["background_alpha"] = add_property_json("Background Alpha", background_alpha.GetValue(requested_frame), "float", "", &background_alpha, 0.0, 1.0, false, requested_frame); + root["background_padding"] = add_property_json("Background Padding", background_padding.GetValue(requested_frame), "float", "", &background_padding, 0.0, 100.0, false, requested_frame); + root["background_corner"] = add_property_json("Background Corner Radius", background_corner.GetValue(requested_frame), "float", "", &background_corner, 0.0, 100.0, false, requested_frame); + + return root.toStyledString(); +} diff --git a/src/effects/Timer.h b/src/effects/Timer.h new file mode 100644 index 000000000..1d96dd0a7 --- /dev/null +++ b/src/effects/Timer.h @@ -0,0 +1,100 @@ +/** + * @file + * @brief Header file for Timer effect class + * @author OpenShot Studios, LLC + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#ifndef OPENSHOT_TIMER_EFFECT_H +#define OPENSHOT_TIMER_EFFECT_H + +#include "../Color.h" +#include "../EffectBase.h" +#include "../Enums.h" +#include "../Frame.h" +#include "../Json.h" +#include "../KeyFrame.h" + +#include +#include + +namespace openshot +{ + enum TimerMode { + TIMER_MODE_COUNT_UP = 0, + TIMER_MODE_COUNT_DOWN = 1, + TIMER_MODE_CLOCK = 2, + TIMER_MODE_TIMECODE = 3, + TIMER_MODE_FRAME_NUMBER = 4 + }; + + enum TimerTimeSource { + TIMER_TIME_CLIP = 0, + TIMER_TIME_SOURCE = 1 + }; + + enum TimerFormat { + TIMER_FORMAT_MM_SS = 0, + TIMER_FORMAT_HH_MM_SS = 1, + TIMER_FORMAT_HH_MM_SS_MILLISECONDS = 2, + TIMER_FORMAT_TIMECODE = 3, + TIMER_FORMAT_FRAMES = 4 + }; + + class Timer : public EffectBase + { + private: + void init_effect_details(); + double ResolveFps() const; + int64_t EffectiveFrameNumber(int64_t frame_number) const; + double CountdownDuration(int64_t frame_number) const; + std::string FormatSeconds(double seconds, double fps, bool duration_style) const; + std::string FormatTimecode(double seconds, double fps) const; + std::string TimerLayoutText(int64_t frame_number) const; + + public: + int mode; + int time_source; + int format; + int clamp; + int gravity; + int show_background; + std::string font_name; + std::string prefix; + std::string suffix; + Color color; + Color stroke; + Color background; + Keyframe start_time; + Keyframe end_time; + Keyframe font_size; + Keyframe font_alpha; + Keyframe stroke_width; + Keyframe x_offset; + Keyframe y_offset; + Keyframe background_alpha; + Keyframe background_padding; + Keyframe background_corner; + + Timer(); + + std::shared_ptr GetFrame(int64_t frame_number) override { return GetFrame(std::make_shared(), frame_number); } + std::shared_ptr GetFrame(std::shared_ptr frame, int64_t frame_number) override; + + double TimerSeconds(int64_t frame_number) const; + std::string TimerText(int64_t frame_number) const; + + std::string Json() const override; + void SetJson(const std::string value) override; + Json::Value JsonValue() const override; + void SetJsonValue(const Json::Value root) override; + std::string PropertiesJSON(int64_t requested_frame) const override; + }; +} + +#endif diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp index c8b1df781..3805995ff 100644 --- a/tests/Benchmark.cpp +++ b/tests/Benchmark.cpp @@ -116,6 +116,10 @@ extern "C" { # include "effects/Caption.h" # define OPENSHOT_HAS_CAPTION #endif +#if __has_include("effects/Timer.h") +# include "effects/Timer.h" +# define OPENSHOT_HAS_TIMER +#endif #if __has_include("effects/Displace.h") # include "effects/Displace.h" # define OPENSHOT_HAS_DISPLACE @@ -704,6 +708,29 @@ int main(int argc, char* argv[]) { } // headless guard #endif +#ifdef OPENSHOT_HAS_TIMER + { + const char* qt_platform = std::getenv("QT_QPA_PLATFORM"); + const bool headless = qt_platform && std::string(qt_platform) == "offscreen"; + if (!headless) + trials.emplace_back("Effect_Timer", [&]() -> TrialResult { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + Timer timer; + timer.mode = TIMER_MODE_COUNT_UP; + timer.format = TIMER_FORMAT_HH_MM_SS_MILLISECONDS; + timer.prefix = "T "; + clip.AddEffect(&timer); + TrialResult result = timed_read(clip); + clip.Close(); + r.Close(); + return result; + }); + } // headless guard +#endif + #ifdef OPENSHOT_HAS_DISPLACE trials.emplace_back("Effect_Displace", [&]() -> TrialResult { FFmpegReader r(video); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index afe0ebe4c..6815c9246 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -76,6 +76,7 @@ set(OPENSHOT_TESTS Sharpen Shadow SphericalEffect + Timer DenoiseImage WaveEffect ) @@ -150,7 +151,7 @@ foreach(tname ${OPENSHOT_TESTS}) set(test_properties LABELS ${tname} ) - if(tname STREQUAL "Caption") + if(tname STREQUAL "Caption" OR tname STREQUAL "Timer") list(APPEND test_properties ENVIRONMENT "QT_QPA_PLATFORM=minimal" ) diff --git a/tests/Timer.cpp b/tests/Timer.cpp new file mode 100644 index 000000000..c340d8e0b --- /dev/null +++ b/tests/Timer.cpp @@ -0,0 +1,303 @@ +/** + * @file + * @brief Unit tests for openshot::Timer + * @author OpenShot Studios, LLC + * + * @ref License + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "openshot_catch.h" + +#include +#include + +#include "Clip.h" +#include "DummyReader.h" +#include "EffectInfo.h" +#include "Frame.h" +#include "Json.h" +#include "effects/Timer.h" + +static bool HasNonBlackPixels(const std::shared_ptr& frame) { + for (int row = 0; row < frame->GetHeight(); ++row) { + const unsigned char* pixels = frame->GetPixels(row); + for (int col = 0; col < frame->GetWidth(); ++col) { + const int index = col * 4; + if (pixels[index] != 0 || pixels[index + 1] != 0 || pixels[index + 2] != 0) + return true; + } + } + return false; +} + +static int FirstNonBlackColumn(const std::shared_ptr& frame) { + for (int col = 0; col < frame->GetWidth(); ++col) { + for (int row = 0; row < frame->GetHeight(); ++row) { + const unsigned char* pixels = frame->GetPixels(row); + const int index = col * 4; + if (pixels[index] != 0 || pixels[index + 1] != 0 || pixels[index + 2] != 0) + return col; + } + } + return -1; +} + +static int LastNonBlackColumn(const std::shared_ptr& frame) { + for (int col = frame->GetWidth() - 1; col >= 0; --col) { + for (int row = 0; row < frame->GetHeight(); ++row) { + const unsigned char* pixels = frame->GetPixels(row); + const int index = col * 4; + if (pixels[index] != 0 || pixels[index + 1] != 0 || pixels[index + 2] != 0) + return col; + } + } + return -1; +} + +static int FirstNonBlackRow(const std::shared_ptr& frame) { + for (int row = 0; row < frame->GetHeight(); ++row) { + const unsigned char* pixels = frame->GetPixels(row); + for (int col = 0; col < frame->GetWidth(); ++col) { + const int index = col * 4; + if (pixels[index] != 0 || pixels[index + 1] != 0 || pixels[index + 2] != 0) + return row; + } + } + return -1; +} + +TEST_CASE("timer effect", "[libopenshot][timer]") { + int argc = 1; + char* argv[1] = {(char*)""}; +#if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) + QGuiApplication::setAttribute(Qt::AA_EnableHighDpiScaling); +#endif + QApplication app(argc, argv); + + SECTION("defaults, json, and properties") { + openshot::Timer timer; + + CHECK(timer.mode == openshot::TIMER_MODE_COUNT_UP); + CHECK(timer.time_source == openshot::TIMER_TIME_SOURCE); + CHECK(timer.format == openshot::TIMER_FORMAT_MM_SS); + CHECK(timer.clamp == 1); + CHECK(timer.gravity == openshot::GRAVITY_BOTTOM); + CHECK(timer.font_name == "sans"); + CHECK(timer.start_time.GetValue(1) == Approx(0.0).margin(0.00001)); + CHECK(timer.end_time.GetValue(1) == Approx(0.0).margin(0.00001)); + CHECK(timer.x_offset.GetValue(1) == Approx(0.0).margin(0.00001)); + CHECK(timer.y_offset.GetValue(1) == Approx(0.0).margin(0.00001)); + CHECK(timer.TimerText(1) == "00:00"); + + Json::Value json = timer.JsonValue(); + CHECK(json["type"].asString() == "Timer"); + CHECK(json["gravity"].asInt() == openshot::GRAVITY_BOTTOM); + CHECK(json["anchor"].isNull()); + CHECK(json["text_align"].isNull()); + json["mode"] = openshot::TIMER_MODE_COUNT_DOWN; + json["format"] = openshot::TIMER_FORMAT_HH_MM_SS; + json["prefix"] = "Left "; + openshot::Timer loaded; + loaded.SetJsonValue(json); + CHECK(loaded.mode == openshot::TIMER_MODE_COUNT_DOWN); + CHECK(loaded.format == openshot::TIMER_FORMAT_HH_MM_SS); + CHECK(loaded.prefix == "Left "); + + Json::Value properties = openshot::stringToJson(timer.PropertiesJSON(1)); + CHECK(properties["mode"]["name"].asString() == "Mode"); + CHECK(properties["mode"]["choices"].size() == 5); + CHECK(properties["time_source"]["name"].asString() == "Time Source"); + CHECK(properties["time_source"]["choices"].size() == 2); + CHECK(properties["gravity"]["name"].asString() == "Gravity"); + CHECK(properties["gravity"]["choices"].size() == 9); + CHECK(properties["anchor"].isNull()); + CHECK(properties["text_align"].isNull()); + CHECK(properties["x_offset"]["name"].asString() == "X Offset (%)"); + CHECK(properties["x_offset"]["min"].asFloat() == Approx(-100.0).margin(0.00001)); + CHECK(properties["x_offset"]["max"].asFloat() == Approx(100.0).margin(0.00001)); + CHECK(properties["y_offset"]["name"].asString() == "Y Offset (%)"); + CHECK(properties["y_offset"]["min"].asFloat() == Approx(-100.0).margin(0.00001)); + CHECK(properties["y_offset"]["max"].asFloat() == Approx(100.0).margin(0.00001)); + CHECK(properties["show_background"]["choices"].size() == 2); + CHECK(properties["font_name"]["type"].asString() == "font"); + } + + SECTION("registered in effect catalog and factory") { + std::unique_ptr effect(openshot::EffectInfo().CreateEffect("Timer")); + REQUIRE(effect); + CHECK(effect->info.class_name == "Timer"); + + Json::Value effects = openshot::EffectInfo().JsonValue(); + bool found_timer = false; + for (Json::ArrayIndex index = 0; index < effects.size(); ++index) { + if (effects[index]["class_name"].asString() == "Timer") { + found_timer = true; + CHECK(effects[index]["name"].asString() == "Timer"); + break; + } + } + CHECK(found_timer); + } + + SECTION("formats count up and down") { + openshot::Timer timer; + timer.format = openshot::TIMER_FORMAT_HH_MM_SS_MILLISECONDS; + timer.start_time = openshot::Keyframe(10.0); + CHECK(timer.TimerText(31) == "00:00:11.000"); + + timer.mode = openshot::TIMER_MODE_COUNT_DOWN; + timer.end_time = openshot::Keyframe(5.0); + timer.format = openshot::TIMER_FORMAT_MM_SS; + CHECK(timer.TimerText(151) == "00:00"); + + timer.mode = openshot::TIMER_MODE_TIMECODE; + timer.format = openshot::TIMER_FORMAT_TIMECODE; + CHECK(timer.TimerText(31) == "00:00:11:00"); + + timer.mode = openshot::TIMER_MODE_FRAME_NUMBER; + timer.format = openshot::TIMER_FORMAT_FRAMES; + timer.start_time = openshot::Keyframe(1.0); + CHECK(timer.TimerText(1) == "31"); + } + + SECTION("count down defaults to clip duration") { + openshot::DummyReader reader(openshot::Fraction(30, 1), 640, 360, 44100, 2, 37.0); + reader.Open(); + openshot::Clip clip(&reader); + clip.Open(); + + openshot::Timer timer; + timer.mode = openshot::TIMER_MODE_COUNT_DOWN; + clip.AddEffect(&timer); + + CHECK(timer.TimerText(1) == "00:37"); + CHECK(timer.TimerText(31) == "00:36"); + + timer.start_time = openshot::Keyframe(5.0); + CHECK(timer.TimerText(1) == "00:32"); + + timer.end_time = openshot::Keyframe(60.0); + CHECK(timer.TimerText(1) == "00:55"); + + clip.Close(); + reader.Close(); + } + + SECTION("renders styled timer text") { + openshot::Timer timer; + timer.gravity = openshot::GRAVITY_CENTER; + timer.x_offset = openshot::Keyframe(0.0); + timer.y_offset = openshot::Keyframe(0.0); + timer.font_size = openshot::Keyframe(36.0); + + auto frame = std::make_shared(1, 640, 360, "#000000", 0, 2); + timer.GetFrame(frame, 1); + CHECK(HasNonBlackPixels(frame)); + } + + SECTION("gravity affects rendered placement") { + openshot::Timer left_timer; + left_timer.gravity = openshot::GRAVITY_LEFT; + left_timer.x_offset = openshot::Keyframe(0.0); + left_timer.y_offset = openshot::Keyframe(0.0); + left_timer.font_size = openshot::Keyframe(36.0); + left_timer.stroke_width = openshot::Keyframe(0.0); + left_timer.show_background = 0; + + openshot::Timer right_timer = left_timer; + right_timer.gravity = openshot::GRAVITY_RIGHT; + + auto left_frame = std::make_shared(1, 640, 360, "#000000", 0, 2); + auto right_frame = std::make_shared(1, 640, 360, "#000000", 0, 2); + left_timer.GetFrame(left_frame, 1); + right_timer.GetFrame(right_frame, 1); + + const int left_column = FirstNonBlackColumn(left_frame); + const int right_column = FirstNonBlackColumn(right_frame); + REQUIRE(left_column >= 0); + REQUIRE(right_column >= 0); + CHECK(right_column > left_column + 300); + } + + SECTION("centered countdown uses stable layout width") { + openshot::Timer timer; + timer.mode = openshot::TIMER_MODE_COUNT_DOWN; + timer.time_source = openshot::TIMER_TIME_CLIP; + timer.end_time = openshot::Keyframe(11.0); + timer.gravity = openshot::GRAVITY_BOTTOM; + timer.font_size = openshot::Keyframe(48.0); + timer.stroke_width = openshot::Keyframe(0.0); + timer.show_background = 0; + + auto wide_frame = std::make_shared(1, 640, 360, "#000000", 0, 2); + auto narrow_frame = std::make_shared(331, 640, 360, "#000000", 0, 2); + timer.GetFrame(wide_frame, 1); + timer.GetFrame(narrow_frame, 331); + + const int wide_left = FirstNonBlackColumn(wide_frame); + const int wide_right = LastNonBlackColumn(wide_frame); + const int narrow_left = FirstNonBlackColumn(narrow_frame); + const int narrow_right = LastNonBlackColumn(narrow_frame); + REQUIRE(wide_left >= 0); + REQUIRE(wide_right >= 0); + REQUIRE(narrow_left >= 0); + REQUIRE(narrow_right >= 0); + + const double wide_center = (wide_left + wide_right) / 2.0; + const double narrow_center = (narrow_left + narrow_right) / 2.0; + CHECK(narrow_center == Approx(wide_center).margin(3.0)); + } + + SECTION("offsets are percentages of the frame") { + openshot::Timer base_timer; + base_timer.gravity = openshot::GRAVITY_CENTER; + base_timer.font_size = openshot::Keyframe(36.0); + base_timer.stroke_width = openshot::Keyframe(0.0); + base_timer.show_background = 0; + + openshot::Timer shifted_timer = base_timer; + shifted_timer.x_offset = openshot::Keyframe(10.0); + shifted_timer.y_offset = openshot::Keyframe(10.0); + + auto base_frame = std::make_shared(1, 640, 360, "#000000", 0, 2); + auto shifted_frame = std::make_shared(1, 640, 360, "#000000", 0, 2); + base_timer.GetFrame(base_frame, 1); + shifted_timer.GetFrame(shifted_frame, 1); + + const int base_column = FirstNonBlackColumn(base_frame); + const int shifted_column = FirstNonBlackColumn(shifted_frame); + const int base_row = FirstNonBlackRow(base_frame); + const int shifted_row = FirstNonBlackRow(shifted_frame); + REQUIRE(base_column >= 0); + REQUIRE(shifted_column >= 0); + REQUIRE(base_row >= 0); + REQUIRE(shifted_row >= 0); + CHECK(shifted_column > base_column + 50); + CHECK(shifted_row > base_row + 25); + } + + SECTION("source time follows clip time keyframe") { + openshot::DummyReader reader(openshot::Fraction(30, 1), 640, 360, 44100, 2, 10.0); + reader.Open(); + openshot::Clip clip(&reader); + clip.Open(); + clip.time.AddPoint(1, 1.0); + clip.time.AddPoint(31, 16.0); + + openshot::Timer timer; + timer.time_source = openshot::TIMER_TIME_SOURCE; + clip.AddEffect(&timer); + + CHECK(timer.TimerSeconds(31) == Approx(0.5).margin(0.00001)); + CHECK(timer.TimerText(31) == "00:00"); + + clip.Close(); + reader.Close(); + } + + app.quit(); +} From d3e3c6b146fbfe5295a4baece0f23dd21ae2de4d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 26 Jun 2026 16:15:36 -0500 Subject: [PATCH 23/30] Fix live screen capture startup on macOS --- src/ScreenCaptureReader.cpp | 89 ++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 4f157bebf..3350a9448 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -13,8 +13,11 @@ #include "ScreenCaptureReader.h" #include +#include +#include #include #include +#include extern "C" { #include @@ -68,6 +71,62 @@ namespace const auto* reader = static_cast*>(opaque); return reader && reader->load() ? 1 : 0; } + + std::string avfoundation_video_name(const std::string& input_name) + { + const size_t separator = input_name.find(':'); + return separator == std::string::npos ? input_name : input_name.substr(0, separator); + } + + std::string avfoundation_audio_name(const std::string& input_name) + { + const size_t separator = input_name.find(':'); + return separator == std::string::npos ? "none" : input_name.substr(separator + 1); + } + + bool resolve_avfoundation_video_index( + AVInputFormat* input_format, + const std::string& video_name, + std::string& video_index) + { + const auto is_digit = [](unsigned char value) { + return std::isdigit(value) != 0; + }; + if (video_name.empty() + || video_name == "default" + || video_name == "none" + || std::all_of(video_name.begin(), video_name.end(), is_digit)) { + return false; + } + + AVDeviceInfoList* device_list = nullptr; + const int result = avdevice_list_input_sources(input_format, nullptr, nullptr, &device_list); + if (result < 0 || !device_list) { + avdevice_free_list_devices(&device_list); + return false; + } + + bool found = false; + for (int index = 0; index < device_list->nb_devices; ++index) { + const AVDeviceInfo* device = device_list->devices[index]; + if (!device || !device->device_name) { + continue; + } + const std::string name = device->device_name; + const std::string description = device->device_description + ? device->device_description + : device->device_name; + if (video_name == name || video_name == description) { + video_index = std::all_of(name.begin(), name.end(), is_digit) + ? name + : std::to_string(index); + found = true; + break; + } + } + avdevice_free_list_devices(&device_list); + return found; + } } #if defined(HAVE_WAYLAND_CAPTURE) @@ -284,13 +343,14 @@ void ScreenCaptureReader::OpenDevice() avdevice_register_all(); AVInputFormat* input_format = const_cast(av_find_input_format(InputFormatName().c_str())); + std::string input_name = InputName(); if (!input_format) { - throw InvalidOptions("FFmpeg input device is not available: " + InputFormatName(), InputName()); + throw InvalidOptions("FFmpeg input device is not available: " + InputFormatName(), input_name); } format_context = avformat_alloc_context(); if (!format_context) { - throw OutOfMemory("Unable to allocate capture format context.", InputName()); + throw OutOfMemory("Unable to allocate capture format context.", input_name); } format_context->interrupt_callback.callback = capture_interrupt_callback; format_context->interrupt_callback.opaque = &close_requested; @@ -317,6 +377,11 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, "offset_y", std::to_string(settings.y)); } else if (InputFormatName() == "avfoundation") { set_option(&options, "capture_cursor", settings.include_cursor ? "1" : "0"); + std::string video_index; + if (resolve_avfoundation_video_index(input_format, avfoundation_video_name(input_name), video_index)) { + set_option(&options, "video_device_index", video_index); + input_name = ":" + avfoundation_audio_name(input_name); + } } for (const auto& option : settings.options) { if (option.first == "input_format_name" @@ -329,10 +394,10 @@ void ScreenCaptureReader::OpenDevice() set_option(&options, option.first, option.second); } - const int result = avformat_open_input(&format_context, InputName().c_str(), input_format, &options); + const int result = avformat_open_input(&format_context, input_name.c_str(), input_format, &options); av_dict_free(&options); if (result < 0) { - throw InvalidFile("Unable to open screen capture input: " + std::string(av_err2str(result)), InputName()); + throw InvalidFile("Unable to open screen capture input: " + std::string(av_err2str(result)), input_name); } } @@ -411,7 +476,19 @@ std::shared_ptr ScreenCaptureReader::GetFrame(int64_t number) std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) { - while (av_read_frame(format_context, packet) >= 0) { + while (!close_requested) { + const int read_result = av_read_frame(format_context, packet); + if (read_result == AVERROR(EAGAIN)) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + continue; + } + if (read_result < 0) { + const std::string message = frames_read > 0 + ? "Capture input ended after " + std::to_string(frames_read) + " frame(s): " + : "Capture input ended before a frame could be read: "; + throw InvalidFile(message + std::string(av_err2str(read_result)), InputName()); + } + if (packet->stream_index != video_stream) { av_packet_unref(packet); dropped_packets++; @@ -497,7 +574,7 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) return frame; } - throw InvalidFile("Capture input ended before a frame could be read.", InputName()); + throw ReaderClosed("The ScreenCaptureReader was closed."); } void ScreenCaptureReader::Close() From 60c4359a232fea7738024a626bea28403fb59311 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 27 Jun 2026 23:24:40 -0500 Subject: [PATCH 24/30] Add clip margin and corner radius support --- src/Clip.cpp | 68 ++++++++++++++++++------ src/Clip.h | 2 + tests/Benchmark.cpp | 2 + tests/Clip.cpp | 123 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 15 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index c62039756..75ba6f006 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #ifdef USE_IMAGEMAGICK #include "MagickUtilities.h" @@ -96,6 +97,8 @@ void Clip::init_settings() // Init alpha alpha = Keyframe(1.0); + margin = Keyframe(0.0); + corner_radius = Keyframe(0.0); // Init time & volume time = Keyframe(1.0); @@ -928,6 +931,8 @@ std::string Clip::PropertiesJSON(int64_t requested_frame) const { // Keyframes root["alpha"] = add_property_json("Alpha", alpha.GetValue(requested_frame), "float", "", &alpha, 0.0, 1.0, false, requested_frame); + root["corner_radius"] = add_property_json("Corner Radius", corner_radius.GetValue(requested_frame), "float", "", &corner_radius, 0.0, 0.5, false, requested_frame); + root["margin"] = add_property_json("Margin", margin.GetValue(requested_frame), "float", "", &margin, 0.0, 0.5, false, requested_frame); root["origin_x"] = add_property_json("Origin X", origin_x.GetValue(requested_frame), "float", "", &origin_x, 0.0, 1.0, false, requested_frame); root["origin_y"] = add_property_json("Origin Y", origin_y.GetValue(requested_frame), "float", "", &origin_y, 0.0, 1.0, false, requested_frame); root["volume"] = add_property_json("Volume", volume.GetValue(requested_frame), "float", "", &volume, 0.0, 1.0, false, requested_frame); @@ -983,6 +988,8 @@ Json::Value Clip::JsonValue() const { root["location_x"] = location_x.JsonValue(); root["location_y"] = location_y.JsonValue(); root["alpha"] = alpha.JsonValue(); + root["corner_radius"] = corner_radius.JsonValue(); + root["margin"] = margin.JsonValue(); root["rotation"] = rotation.JsonValue(); root["time"] = time.JsonValue(); root["volume"] = volume.JsonValue(); @@ -1099,6 +1106,10 @@ void Clip::SetJsonValue(const Json::Value root) { location_y.SetJsonValue(root["location_y"]); if (!root["alpha"].isNull()) alpha.SetJsonValue(root["alpha"]); + if (!root["corner_radius"].isNull()) + corner_radius.SetJsonValue(root["corner_radius"]); + if (!root["margin"].isNull()) + margin.SetJsonValue(root["margin"]); if (!root["rotation"].isNull()) rotation.SetJsonValue(root["rotation"]); if (!root["time"].isNull()) @@ -1149,6 +1160,8 @@ void Clip::SetJsonValue(const Json::Value root) { ensure_default_keyframe(origin_x, 0.5); ensure_default_keyframe(origin_y, 0.5); ensure_default_keyframe(rotation, 0.0); + ensure_default_keyframe(corner_radius, 0.0); + ensure_default_keyframe(margin, 0.0); if (!root["effects"].isNull()) { // Clear existing effects @@ -1399,6 +1412,18 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { // Apply opacity via painter instead of per-pixel alpha manipulation const float alpha_value = alpha.GetValue(frame->number); + const double corner_radius_value = std::max(0.0, std::min(0.5, corner_radius.GetValue(frame->number))); + if (corner_radius_value > 0.0f) { + painter.setRenderHint(QPainter::Antialiasing, true); + const double radius_pixels = corner_radius_value + * std::min(source_image->width(), source_image->height()); + QPainterPath clip_path; + clip_path.addRoundedRect( + QRectF(0, 0, source_image->width(), source_image->height()), + radius_pixels, + radius_pixels); + painter.setClipPath(clip_path); + } if (alpha_value != 1.0f) { painter.setOpacity(alpha_value); painter.drawImage(0, 0, *source_image); @@ -1521,8 +1546,15 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig // Get image from clip std::shared_ptr source_image = frame->GetImage(); + const double margin_value = std::max(0.0, std::min(0.5, margin.GetValue(frame->number))); + const double margin_pixels = margin_value * std::min(width, height); + const double layout_x = margin_pixels; + const double layout_y = margin_pixels; + const double layout_width = std::max(1.0, width - (margin_pixels * 2.0)); + const double layout_height = std::max(1.0, height - (margin_pixels * 2.0)); + /* RESIZE SOURCE IMAGE - based on scale type */ - QSize source_size = scale_size(source_image->size(), scale, width, height); + QSize source_size = scale_size(source_image->size(), scale, layout_width, layout_height); // Initialize parent object's properties (Clip or Tracked Object) float parentObject_location_x = 0.0; @@ -1602,35 +1634,41 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig switch (gravity) { case (GRAVITY_TOP_LEFT): + x = layout_x; + y = layout_y; // This is only here to prevent unused-enum warnings break; case (GRAVITY_TOP): - x = (width - scaled_source_width) / 2.0; // center + x = layout_x + ((layout_width - scaled_source_width) / 2.0); // center + y = layout_y; break; case (GRAVITY_TOP_RIGHT): - x = width - scaled_source_width; // right + x = layout_x + layout_width - scaled_source_width; // right + y = layout_y; break; case (GRAVITY_LEFT): - y = (height - scaled_source_height) / 2.0; // center + x = layout_x; + y = layout_y + ((layout_height - scaled_source_height) / 2.0); // center break; case (GRAVITY_CENTER): - x = (width - scaled_source_width) / 2.0; // center - y = (height - scaled_source_height) / 2.0; // center + x = layout_x + ((layout_width - scaled_source_width) / 2.0); // center + y = layout_y + ((layout_height - scaled_source_height) / 2.0); // center break; case (GRAVITY_RIGHT): - x = width - scaled_source_width; // right - y = (height - scaled_source_height) / 2.0; // center + x = layout_x + layout_width - scaled_source_width; // right + y = layout_y + ((layout_height - scaled_source_height) / 2.0); // center break; case (GRAVITY_BOTTOM_LEFT): - y = (height - scaled_source_height); // bottom + x = layout_x; + y = layout_y + (layout_height - scaled_source_height); // bottom break; case (GRAVITY_BOTTOM): - x = (width - scaled_source_width) / 2.0; // center - y = (height - scaled_source_height); // bottom + x = layout_x + ((layout_width - scaled_source_width) / 2.0); // center + y = layout_y + (layout_height - scaled_source_height); // bottom break; case (GRAVITY_BOTTOM_RIGHT): - x = width - scaled_source_width; // right - y = (height - scaled_source_height); // bottom + x = layout_x + layout_width - scaled_source_width; // right + y = layout_y + (layout_height - scaled_source_height); // bottom break; } @@ -1654,8 +1692,8 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig } return location * (canvas_size - anchored_position); }; - x += location_offset(location_x_value, x, width, scaled_source_width); - y += location_offset(location_y_value, y, height, scaled_source_height); + x += location_offset(location_x_value, x - layout_x, layout_width, scaled_source_width); + y += location_offset(location_y_value, y - layout_y, layout_height, scaled_source_height); float shear_x_value = shear_x.GetValue(frame->number) + parentObject_shear_x; float shear_y_value = shear_y.GetValue(frame->number) + parentObject_shear_y; float origin_x_value = origin_x.GetValue(frame->number); diff --git a/src/Clip.h b/src/Clip.h index e13c3d89a..8901141e9 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -333,6 +333,8 @@ namespace openshot { openshot::Keyframe location_x; ///< Curve representing the relative X position in percent based on the gravity (-1 to 1) openshot::Keyframe location_y; ///< Curve representing the relative Y position in percent based on the gravity (-1 to 1) openshot::Keyframe alpha; ///< Curve representing the alpha (1 to 0) + openshot::Keyframe margin; ///< Curve representing edge margin as a percent of the canvas' shortest side + openshot::Keyframe corner_radius; ///< Curve representing corner radius as a percent of the clip's shortest side // Rotation and Shear curves (origin point (x,y) is adjustable for both rotation and shear) openshot::Keyframe rotation; ///< Curve representing the rotation (0 to 360) diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp index 3805995ff..f15593a10 100644 --- a/tests/Benchmark.cpp +++ b/tests/Benchmark.cpp @@ -413,6 +413,8 @@ int main(int argc, char* argv[]) { overlay2.End(video_clip.Reader()->info.duration); overlay2.Open(); overlay2.rotation.AddPoint(1, 90.0); + overlay2.margin = Keyframe(0.03); + overlay2.corner_radius = Keyframe(0.2); t.AddClip(&video_clip); t.AddClip(&overlay1); t.AddClip(&overlay2); diff --git a/tests/Clip.cpp b/tests/Clip.cpp index bf0d6b6a7..4d1ef9c40 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -193,6 +193,8 @@ TEST_CASE( "default constructor", "[libopenshot][clip]" ) CHECK(c1.Position() == Approx(0.0f).margin(0.00001)); CHECK(c1.Start() == Approx(0.0f).margin(0.00001)); CHECK(c1.End() == Approx(0.0f).margin(0.00001)); + CHECK(c1.margin.GetValue(1) == Approx(0.0f).margin(0.00001)); + CHECK(c1.corner_radius.GetValue(1) == Approx(0.0f).margin(0.00001)); } TEST_CASE( "path string constructor", "[libopenshot][clip]" ) @@ -295,6 +297,8 @@ TEST_CASE( "properties", "[libopenshot][clip]" ) // Check for specific things CHECK(root["alpha"]["value"].asDouble() == Approx(1.0f).margin(0.01)); CHECK(root["alpha"]["keyframe"].asBool() == true); + CHECK(root["margin"]["value"].asDouble() == Approx(0.0f).margin(0.00001)); + CHECK(root["corner_radius"]["value"].asDouble() == Approx(0.0f).margin(0.00001)); // Get properties JSON string at frame 250 properties = c1.PropertiesJSON(250); @@ -419,6 +423,8 @@ TEST_CASE( "SetJsonValue restores defaults for empty core transform keyframes", clip.origin_x = Keyframe(0.2); clip.origin_y = Keyframe(0.8); clip.rotation = Keyframe(45.0); + clip.margin = Keyframe(0.2); + clip.corner_radius = Keyframe(0.3); Json::Value root = clip.JsonValue(); root["scale_x"]["Points"] = Json::Value(Json::arrayValue); @@ -428,6 +434,8 @@ TEST_CASE( "SetJsonValue restores defaults for empty core transform keyframes", root["origin_x"]["Points"] = Json::Value(Json::arrayValue); root["origin_y"]["Points"] = Json::Value(Json::arrayValue); root["rotation"]["Points"] = Json::Value(Json::arrayValue); + root["margin"]["Points"] = Json::Value(Json::arrayValue); + root["corner_radius"]["Points"] = Json::Value(Json::arrayValue); clip.SetJsonValue(root); @@ -438,6 +446,8 @@ TEST_CASE( "SetJsonValue restores defaults for empty core transform keyframes", REQUIRE(clip.origin_x.GetCount() == 1); REQUIRE(clip.origin_y.GetCount() == 1); REQUIRE(clip.rotation.GetCount() == 1); + REQUIRE(clip.margin.GetCount() == 1); + REQUIRE(clip.corner_radius.GetCount() == 1); CHECK(clip.scale_x.GetValue(1) == Approx(1.0).margin(0.00001)); CHECK(clip.scale_y.GetValue(1) == Approx(1.0).margin(0.00001)); @@ -446,6 +456,24 @@ TEST_CASE( "SetJsonValue restores defaults for empty core transform keyframes", CHECK(clip.origin_x.GetValue(1) == Approx(0.5).margin(0.00001)); CHECK(clip.origin_y.GetValue(1) == Approx(0.5).margin(0.00001)); CHECK(clip.rotation.GetValue(1) == Approx(0.0).margin(0.00001)); + CHECK(clip.margin.GetValue(1) == Approx(0.0).margin(0.00001)); + CHECK(clip.corner_radius.GetValue(1) == Approx(0.0).margin(0.00001)); +} + +TEST_CASE("Clip JSON stores margin and corner radius", "[libopenshot][clip][json]") +{ + Clip clip; + clip.margin = Keyframe(0.04); + clip.corner_radius = Keyframe(0.25); + + Json::Value json = clip.JsonValue(); + REQUIRE_FALSE(json["margin"].isNull()); + REQUIRE_FALSE(json["corner_radius"].isNull()); + + Clip loaded; + loaded.SetJsonValue(json); + CHECK(loaded.margin.GetValue(1) == Approx(0.04).margin(0.00001)); + CHECK(loaded.corner_radius.GetValue(1) == Approx(0.25).margin(0.00001)); } TEST_CASE( "waveform mode serializes and exposes visualization choices", "[libopenshot][clip][json]" ) @@ -1477,6 +1505,101 @@ TEST_CASE("clip_location_uses_distance_from_gravity_anchor_to_offscreen_edge", " CHECK(negative.height() == Approx((anchor_y + expected_h) * 0.5).margin(2.0)); } +TEST_CASE("clip_margin_insets_edge_gravity", "[libopenshot][clip][transform]") +{ + const int source_w = 40; + const int source_h = 30; + const int canvas_w = 160; + const int canvas_h = 90; + + openshot::CacheMemory cache; + auto src = std::make_shared(1, source_w, source_h, "#00000000", 0, 2); + src->AddColor(QColor(Qt::red)); + cache.Add(src); + + openshot::DummyReader dummy(openshot::Fraction(30, 1), source_w, source_h, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + clip.display = openshot::FRAME_DISPLAY_NONE; + clip.scale = openshot::SCALE_NONE; + clip.gravity = openshot::GRAVITY_BOTTOM_RIGHT; + clip.margin = openshot::Keyframe(0.1); + + QRect bounds = render_clip_bounds(clip, canvas_w, canvas_h); + REQUIRE_FALSE(bounds.isNull()); + const int expected_margin = 9; + CHECK(bounds.left() == Approx(canvas_w - source_w - expected_margin).margin(1.0)); + CHECK(bounds.top() == Approx(canvas_h - source_h - expected_margin).margin(1.0)); + CHECK(bounds.width() == Approx(source_w).margin(1.0)); + CHECK(bounds.height() == Approx(source_h).margin(1.0)); +} + +TEST_CASE("clip_margin_reduces_scale_layout_area", "[libopenshot][clip][transform]") +{ + const int source_w = 160; + const int source_h = 90; + const int canvas_w = 160; + const int canvas_h = 90; + + openshot::CacheMemory cache; + auto src = std::make_shared(1, source_w, source_h, "#00000000", 0, 2); + src->AddColor(QColor(Qt::red)); + cache.Add(src); + + openshot::DummyReader dummy(openshot::Fraction(30, 1), source_w, source_h, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + clip.display = openshot::FRAME_DISPLAY_NONE; + clip.scale = openshot::SCALE_FIT; + clip.gravity = openshot::GRAVITY_CENTER; + clip.margin = openshot::Keyframe(0.1); + + QRect bounds = render_clip_bounds(clip, canvas_w, canvas_h); + REQUIRE_FALSE(bounds.isNull()); + CHECK(bounds.left() == Approx(16).margin(1.0)); + CHECK(bounds.top() == Approx(9).margin(1.0)); + CHECK(bounds.width() == Approx(128).margin(3.0)); + CHECK(bounds.height() == Approx(72).margin(3.0)); +} + +TEST_CASE("clip_corner_radius_clips_source_alpha", "[libopenshot][clip][transform]") +{ + const int source_w = 40; + const int source_h = 40; + + openshot::CacheMemory cache; + auto src = std::make_shared(1, source_w, source_h, "#00000000", 0, 2); + src->AddColor(QColor(Qt::red)); + cache.Add(src); + + openshot::DummyReader dummy(openshot::Fraction(30, 1), source_w, source_h, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + clip.display = openshot::FRAME_DISPLAY_NONE; + clip.scale = openshot::SCALE_NONE; + clip.gravity = openshot::GRAVITY_TOP_LEFT; + clip.corner_radius = openshot::Keyframe(0.5); + + auto frame = clip.GetFrame(1); + auto image = frame->GetImage(); + REQUIRE(image); + + CHECK(image->pixelColor(0, 0).alpha() == 0); + CHECK(image->pixelColor(source_w - 1, 0).alpha() == 0); + CHECK(image->pixelColor(0, source_h - 1).alpha() == 0); + CHECK(image->pixelColor(source_w / 2, source_h / 2).red() > 200); + CHECK(image->pixelColor(source_w / 2, source_h / 2).alpha() > 200); +} + TEST_CASE( "transform_path_identity_vs_scaled", "[libopenshot][clip][pr]" ) { // Create a small checker-ish image to make scaling detectable From 48980736622177372542ad0ad1aaf5fdafb14a95 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 27 Jun 2026 23:48:23 -0500 Subject: [PATCH 25/30] Preserve live capture timestamps --- src/Frame.cpp | 4 +++- src/Frame.h | 1 + src/ScreenCaptureReader.cpp | 16 ++++++++++++++++ tests/Frame.cpp | 15 +++++++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/Frame.cpp b/src/Frame.cpp index a1da44fca..1862a82d0 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -13,6 +13,7 @@ #include // for std::this_thread::sleep_for #include // for std::chrono::milliseconds #include +#include #include "Frame.h" #include "AudioBufferSource.h" @@ -43,7 +44,7 @@ using namespace openshot; // Constructor - image & audio Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels) : audio(std::make_shared>(channels, samples)), - number(number), width(width), height(height), + number(number), capture_timestamp(std::numeric_limits::quiet_NaN()), width(width), height(height), pixel_ratio(1,1), color(color), channels(channels), channel_layout(LAYOUT_STEREO), sample_rate(44100), @@ -86,6 +87,7 @@ Frame& Frame::operator= (const Frame& other) void Frame::DeepCopy(const Frame& other) { number = other.number; + capture_timestamp = other.capture_timestamp; channels = other.channels; width = other.width; height = other.height; diff --git a/src/Frame.h b/src/Frame.h index 817a18c39..82106f685 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -115,6 +115,7 @@ namespace openshot public: std::shared_ptr> audio; int64_t number; ///< This is the frame number (starting at 1) + double capture_timestamp; ///< Optional source capture timestamp in seconds for live capture frames bool has_audio_data; ///< This frame has been loaded with audio data bool has_image_data; ///< This frame has been loaded with pixel data diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 3350a9448..6f6d01991 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -495,6 +496,7 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) continue; } + const int64_t packet_timestamp = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; const int send_result = avcodec_send_packet(codec_context, packet); av_packet_unref(packet); if (send_result < 0) { @@ -509,6 +511,19 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) throw InvalidFile("Unable to decode capture frame: " + std::string(av_err2str(receive_result)), InputName()); } + const AVStream* stream = format_context->streams[video_stream]; + int64_t frame_timestamp = source_frame->best_effort_timestamp; + if (frame_timestamp == AV_NOPTS_VALUE) { + frame_timestamp = source_frame->pts; + } + if (frame_timestamp == AV_NOPTS_VALUE) { + frame_timestamp = packet_timestamp; + } + double capture_timestamp = std::numeric_limits::quiet_NaN(); + if (frame_timestamp != AV_NOPTS_VALUE && stream) { + capture_timestamp = static_cast(frame_timestamp) * av_q2d(stream->time_base); + } + const int width = source_frame->width > 0 ? source_frame->width : info.width; const int height = source_frame->height > 0 ? source_frame->height : info.height; const PixelFormat src_fmt = static_cast(source_frame->format); @@ -569,6 +584,7 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) } auto frame = std::make_shared(number, output_width, output_height, "#000000"); + frame->capture_timestamp = capture_timestamp; frame->AddImage(output_width, output_height, bytes_per_pixel, QImage::Format_RGBA8888, output_buffer); frames_read++; return frame; diff --git a/tests/Frame.cpp b/tests/Frame.cpp index 9bbc79eb7..cd3cd523d 100644 --- a/tests/Frame.cpp +++ b/tests/Frame.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -90,6 +91,7 @@ TEST_CASE( "Default_Constructor", "[libopenshot][frame]" ) // Should be false until we load or create contents CHECK(f1->has_image_data == false); CHECK(f1->has_audio_data == false); + CHECK(std::isnan(f1->capture_timestamp)); // Calling GetImage() paints a blank frame, by default std::shared_ptr i1 = f1->GetImage(); @@ -100,6 +102,19 @@ TEST_CASE( "Default_Constructor", "[libopenshot][frame]" ) CHECK(f1->has_audio_data == false); } +TEST_CASE( "Capture_Timestamp_Copies", "[libopenshot][frame]" ) +{ + openshot::Frame f1(1, 800, 600, "#000000"); + f1.capture_timestamp = 12.345; + + openshot::Frame f2(f1); + CHECK(f2.capture_timestamp == Catch::Approx(12.345)); + + openshot::Frame f3; + f3.DeepCopy(f1); + CHECK(f3.capture_timestamp == Catch::Approx(12.345)); +} + TEST_CASE( "Data_Access", "[libopenshot][frame]" ) { From 3b6291f5afb0396aaa10ae05b214942444282610 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 28 Jun 2026 14:17:53 -0500 Subject: [PATCH 26/30] Fix frame timestamp test compatibility --- tests/Frame.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Frame.cpp b/tests/Frame.cpp index cd3cd523d..e352293f6 100644 --- a/tests/Frame.cpp +++ b/tests/Frame.cpp @@ -108,11 +108,11 @@ TEST_CASE( "Capture_Timestamp_Copies", "[libopenshot][frame]" ) f1.capture_timestamp = 12.345; openshot::Frame f2(f1); - CHECK(f2.capture_timestamp == Catch::Approx(12.345)); + CHECK(f2.capture_timestamp == Approx(12.345)); openshot::Frame f3; f3.DeepCopy(f1); - CHECK(f3.capture_timestamp == Catch::Approx(12.345)); + CHECK(f3.capture_timestamp == Approx(12.345)); } From 46a5c2c4b143ac97af4674091eb6572f498c6a98 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 18 Jul 2026 21:25:03 -0500 Subject: [PATCH 27/30] Add system audio to screen capture --- src/CMakeLists.txt | 2 +- src/ScreenCaptureReader.cpp | 520 +++++++++++++++++++++++++++++++++- src/ScreenCaptureReader.h | 12 + tests/ScreenCaptureReader.cpp | 60 ++++ 4 files changed, 583 insertions(+), 11 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 82387dc3f..331ae405c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -618,7 +618,7 @@ endif() if(WIN32) # Required for exception handling and DirectShow device enumeration on Windows - target_link_libraries(openshot PUBLIC "imagehlp" "dbghelp" "ole32" "oleaut32" "strmiids" ) + target_link_libraries(openshot PUBLIC "imagehlp" "dbghelp" "ole32" "oleaut32" "strmiids" "uuid" ) endif() ### diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 6f6d01991..9d532e19e 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -15,22 +15,425 @@ #include #include #include +#include #include +#include #include +#include #include #include +#include extern "C" { #include #include } +#if defined(_WIN32) + #include + #include +#endif + #include "Exceptions.h" #include "Frame.h" #include "QtUtilities.h" using namespace openshot; +#if defined(__linux__) +class ScreenCaptureReader::SystemAudioCapture +{ +public: + explicit SystemAudioCapture(const ScreenCaptureSettings& new_settings) + : settings(new_settings) {} + + ~SystemAudioCapture() { Close(); } + + void Open() + { + if (format_context) return; + avdevice_register_all(); + AVInputFormat* input_format = const_cast(av_find_input_format("pulse")); + if (!input_format) { + throw InvalidOptions("FFmpeg PulseAudio input is unavailable for system audio capture."); + } + + format_context = avformat_alloc_context(); + if (!format_context) { + throw OutOfMemory("Unable to allocate system audio capture context."); + } + format_context->interrupt_callback.callback = InterruptCallback; + format_context->interrupt_callback.opaque = &close_requested; + + AVDictionary* options = nullptr; + av_dict_set(&options, "sample_rate", std::to_string(settings.audio_sample_rate).c_str(), 0); + av_dict_set(&options, "channels", std::to_string(settings.audio_channels).c_str(), 0); + const std::string device = settings.audio_device.empty() ? "@DEFAULT_MONITOR@" : settings.audio_device; + const int open_result = avformat_open_input(&format_context, device.c_str(), input_format, &options); + av_dict_free(&options); + if (open_result < 0) { + throw InvalidFile("Unable to open system audio capture input: " + std::string(av_err2str(open_result)), device); + } + if (avformat_find_stream_info(format_context, nullptr) < 0) { + throw InvalidFile("Unable to read system audio capture stream information.", device); + } + for (unsigned int index = 0; index < format_context->nb_streams; ++index) { + if (format_context->streams[index]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) { + audio_stream = static_cast(index); + break; + } + } + if (audio_stream < 0) { + throw InvalidFile("No audio stream found in system audio capture input.", device); + } + AVStream* stream = format_context->streams[audio_stream]; + const AVCodec* codec = avcodec_find_decoder(stream->codecpar->codec_id); + codec_context = codec ? ffmpeg_get_codec_context(stream, codec) : nullptr; + if (!codec_context || avcodec_open2(codec_context, codec, nullptr) < 0) { + throw InvalidCodec("Unable to open system audio capture decoder.", device); + } + packet = av_packet_alloc(); + decoded_frame = av_frame_alloc(); + if (!packet || !decoded_frame) { + throw OutOfMemory("Unable to allocate system audio capture buffers."); + } + settings.audio_sample_rate = codec_context->sample_rate > 0 + ? codec_context->sample_rate : settings.audio_sample_rate; + channels.assign(settings.audio_channels, {}); + close_requested = false; + worker = std::thread(&SystemAudioCapture::CaptureLoop, this); + } + + void Close() + { + close_requested = true; + ready.notify_all(); + if (worker.joinable()) { + worker.join(); + } + if (decoded_frame) av_frame_free(&decoded_frame); + if (packet) av_packet_free(&packet); + if (codec_context) avcodec_free_context(&codec_context); + if (format_context) avformat_close_input(&format_context); + } + + void AddFrameAudio(const std::shared_ptr& frame, int64_t number, const Fraction& fps) + { + int sample_count = 0; + for (int64_t frame_number = last_output_frame + 1; frame_number <= number; ++frame_number) { + sample_count += Frame::GetSamplesPerFrame( + frame_number, fps, settings.audio_sample_rate, settings.audio_channels); + } + last_output_frame = std::max(last_output_frame, number); + if (!frame || sample_count <= 0) return; + + std::unique_lock lock(queue_mutex); + ready.wait_for(lock, std::chrono::milliseconds(100), [this, sample_count]() { + return close_requested || (!channels.empty() && static_cast(channels[0].size()) >= sample_count); + }); + frame->SampleRate(settings.audio_sample_rate); + frame->ChannelsLayout(settings.audio_channels == 1 ? LAYOUT_MONO : LAYOUT_STEREO); + for (int channel = 0; channel < settings.audio_channels; ++channel) { + std::vector samples(sample_count, 0.0f); + if (channel < static_cast(channels.size())) { + const int available = std::min(sample_count, static_cast(channels[channel].size())); + for (int index = 0; index < available; ++index) { + samples[index] = channels[channel].front(); + channels[channel].pop_front(); + } + } + frame->AddAudio(true, channel, 0, samples.data(), sample_count, 1.0f); + } + } + + void Reset() + { + std::lock_guard lock(queue_mutex); + for (auto& channel : channels) channel.clear(); + last_output_frame = 0; + } + + int SampleRate() const { return settings.audio_sample_rate; } + int Channels() const { return settings.audio_channels; } + +private: + static int InterruptCallback(void* opaque) + { + const auto* requested = static_cast*>(opaque); + return requested && requested->load() ? 1 : 0; + } + + float SampleAt(const AVFrame* frame, int channel, int sample) const + { + const AVSampleFormat format = static_cast(frame->format); + const bool planar = av_sample_fmt_is_planar(format) != 0; + const int source_channels = std::max(1, +#if HAVE_CH_LAYOUT + codec_context->ch_layout.nb_channels +#else + codec_context->channels +#endif + ); + const int source_channel = std::min(channel, source_channels - 1); + const uint8_t* data = planar ? frame->extended_data[source_channel] : frame->extended_data[0]; + const int offset = planar ? sample : sample * source_channels + source_channel; + switch (format) { + case AV_SAMPLE_FMT_U8: case AV_SAMPLE_FMT_U8P: + return (static_cast(static_cast(data))[offset] - 128) / 128.0f; + case AV_SAMPLE_FMT_S16: case AV_SAMPLE_FMT_S16P: + return static_cast(static_cast(data))[offset] / 32768.0f; + case AV_SAMPLE_FMT_S32: case AV_SAMPLE_FMT_S32P: + return static_cast(static_cast(static_cast(data))[offset] / 2147483648.0); + case AV_SAMPLE_FMT_FLT: case AV_SAMPLE_FMT_FLTP: + return static_cast(static_cast(data))[offset]; + case AV_SAMPLE_FMT_DBL: case AV_SAMPLE_FMT_DBLP: + return static_cast(static_cast(static_cast(data))[offset]); + default: + return 0.0f; + } + } + + void CaptureLoop() + { + while (!close_requested) { + const int read_result = av_read_frame(format_context, packet); + if (read_result == AVERROR(EAGAIN)) continue; + if (read_result < 0) break; + if (packet->stream_index != audio_stream) { + av_packet_unref(packet); + continue; + } + const int send_result = avcodec_send_packet(codec_context, packet); + av_packet_unref(packet); + if (send_result < 0) continue; + while (avcodec_receive_frame(codec_context, decoded_frame) == 0) { + std::lock_guard lock(queue_mutex); + for (int channel = 0; channel < settings.audio_channels; ++channel) { + const size_t max_samples = static_cast(settings.audio_sample_rate) * 10; + for (int sample = 0; sample < decoded_frame->nb_samples; ++sample) { + if (channels[channel].size() >= max_samples) channels[channel].pop_front(); + channels[channel].push_back(SampleAt(decoded_frame, channel, sample)); + } + } + av_frame_unref(decoded_frame); + ready.notify_all(); + } + } + } + + ScreenCaptureSettings settings; + AVFormatContext* format_context = nullptr; + AVCodecContext* codec_context = nullptr; + AVPacket* packet = nullptr; + AVFrame* decoded_frame = nullptr; + int audio_stream = -1; + std::atomic close_requested { false }; + std::thread worker; + std::mutex queue_mutex; + std::condition_variable ready; + std::vector> channels; + int64_t last_output_frame = 0; +}; +#elif defined(_WIN32) +class ScreenCaptureReader::SystemAudioCapture +{ +public: + explicit SystemAudioCapture(const ScreenCaptureSettings& new_settings) + : settings(new_settings) {} + + ~SystemAudioCapture() { Close(); } + + void Open() + { + if (worker.joinable()) return; + close_requested = false; + worker = std::thread(&SystemAudioCapture::CaptureLoop, this); + std::unique_lock lock(state_mutex); + state_ready.wait_for(lock, std::chrono::seconds(5), [this]() { return opened || failed; }); + if (!opened) { + close_requested = true; + lock.unlock(); + if (worker.joinable()) worker.join(); + throw InvalidOptions(error.empty() ? "Unable to open WASAPI system audio loopback capture." : error); + } + } + + void Close() + { + close_requested = true; + ready.notify_all(); + if (worker.joinable()) worker.join(); + opened = false; + } + + void AddFrameAudio(const std::shared_ptr& frame, int64_t number, const Fraction& fps) + { + int sample_count = 0; + for (int64_t frame_number = last_output_frame + 1; frame_number <= number; ++frame_number) { + sample_count += Frame::GetSamplesPerFrame(frame_number, fps, sample_rate, channel_count); + } + last_output_frame = std::max(last_output_frame, number); + if (!frame || sample_count <= 0) return; + std::unique_lock lock(queue_mutex); + ready.wait_for(lock, std::chrono::milliseconds(100), [this, sample_count]() { + return close_requested || (!channels.empty() && static_cast(channels[0].size()) >= sample_count); + }); + frame->SampleRate(sample_rate); + frame->ChannelsLayout(channel_count == 1 ? LAYOUT_MONO : LAYOUT_STEREO); + for (int channel = 0; channel < channel_count; ++channel) { + std::vector samples(sample_count, 0.0f); + const int available = std::min(sample_count, static_cast(channels[channel].size())); + for (int index = 0; index < available; ++index) { + samples[index] = channels[channel].front(); + channels[channel].pop_front(); + } + frame->AddAudio(true, channel, 0, samples.data(), sample_count, 1.0f); + } + } + + void Reset() + { + std::lock_guard lock(queue_mutex); + for (auto& channel : channels) channel.clear(); + last_output_frame = 0; + } + + int SampleRate() const { return sample_rate; } + int Channels() const { return channel_count; } + +private: + template static void Release(T*& value) + { + if (value) { value->Release(); value = nullptr; } + } + + void Fail(const std::string& message) + { + std::lock_guard lock(state_mutex); + error = message; + failed = true; + state_ready.notify_all(); + } + + void CaptureLoop() + { + IMMDeviceEnumerator* enumerator = nullptr; + IMMDevice* device = nullptr; + IAudioClient* client = nullptr; + IAudioCaptureClient* capture = nullptr; + WAVEFORMATEX* format = nullptr; + const HRESULT com_result = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + const bool uninitialize_com = SUCCEEDED(com_result); + HRESULT result = CoCreateInstance(CLSID_MMDeviceEnumerator, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator, reinterpret_cast(&enumerator)); + if (SUCCEEDED(result)) { + result = enumerator->GetDefaultAudioEndpoint(eRender, eConsole, &device); + } + if (SUCCEEDED(result)) { + result = device->Activate(IID_IAudioClient, CLSCTX_ALL, nullptr, reinterpret_cast(&client)); + } + if (SUCCEEDED(result)) result = client->GetMixFormat(&format); + if (SUCCEEDED(result)) { + result = client->Initialize(AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_LOOPBACK, + 0, 0, format, nullptr); + } + if (SUCCEEDED(result)) { + result = client->GetService(IID_IAudioCaptureClient, reinterpret_cast(&capture)); + } + if (SUCCEEDED(result)) result = client->Start(); + if (FAILED(result) || !format || !capture) { + Fail("Unable to initialize WASAPI system audio loopback capture (HRESULT " + std::to_string(result) + ")."); + if (format) CoTaskMemFree(format); + Release(capture); Release(client); Release(device); Release(enumerator); + if (uninitialize_com) CoUninitialize(); + return; + } + + sample_rate = static_cast(format->nSamplesPerSec); + channel_count = std::max(1, std::min(2, static_cast(format->nChannels))); + channels.assign(channel_count, {}); + bool floating_point = format->wFormatTag == WAVE_FORMAT_IEEE_FLOAT; + if (format->wFormatTag == WAVE_FORMAT_EXTENSIBLE && format->cbSize >= 22) { + const auto* extensible = reinterpret_cast(format); + floating_point = extensible->SubFormat.Data1 == WAVE_FORMAT_IEEE_FLOAT; + } + { + std::lock_guard lock(state_mutex); + opened = true; + state_ready.notify_all(); + } + + while (!close_requested) { + UINT32 packet_frames = 0; + if (FAILED(capture->GetNextPacketSize(&packet_frames))) break; + if (packet_frames == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + continue; + } + BYTE* data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + if (FAILED(capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr))) break; + { + std::lock_guard lock(queue_mutex); + for (UINT32 frame_index = 0; frame_index < frames; ++frame_index) { + for (int channel = 0; channel < channel_count; ++channel) { + float sample = 0.0f; + if (!(flags & AUDCLNT_BUFFERFLAGS_SILENT) && data) { + const int offset = static_cast(frame_index) * format->nChannels + channel; + if (floating_point && format->wBitsPerSample == 32) { + sample = reinterpret_cast(data)[offset]; + } else if (format->wBitsPerSample == 16) { + sample = reinterpret_cast(data)[offset] / 32768.0f; + } else if (format->wBitsPerSample == 32) { + sample = static_cast(reinterpret_cast(data)[offset] / 2147483648.0); + } + } + const size_t max_samples = static_cast(sample_rate) * 10; + if (channels[channel].size() >= max_samples) channels[channel].pop_front(); + channels[channel].push_back(sample); + } + } + } + capture->ReleaseBuffer(frames); + ready.notify_all(); + } + + client->Stop(); + CoTaskMemFree(format); + Release(capture); Release(client); Release(device); Release(enumerator); + if (uninitialize_com) CoUninitialize(); + } + + ScreenCaptureSettings settings; + std::atomic close_requested { false }; + std::thread worker; + std::mutex queue_mutex; + std::condition_variable ready; + std::vector> channels; + int64_t last_output_frame = 0; + int sample_rate = 48000; + int channel_count = 2; + std::mutex state_mutex; + std::condition_variable state_ready; + bool opened = false; + bool failed = false; + std::string error; +}; +#else +class ScreenCaptureReader::SystemAudioCapture +{ +public: + explicit SystemAudioCapture(const ScreenCaptureSettings&) {} + void Open() {} + void Close() {} + void AddFrameAudio(const std::shared_ptr&, int64_t, const Fraction&) {} + void Reset() {} + int SampleRate() const { return 0; } + int Channels() const { return 0; } +}; +#endif + namespace { std::string fraction_to_string(const Fraction& value) @@ -150,12 +553,18 @@ ScreenCaptureReader::ScreenCaptureReader(const ScreenCaptureSettings& new_settin , packet(nullptr) , sws_context(nullptr) , close_requested(false) + , system_audio(nullptr) { if (settings.backend == SCREEN_CAPTURE_AUTO) { settings.backend = DefaultBackend(); } ValidateSettings(); PopulateInfo(); +#if defined(__linux__) || defined(_WIN32) + if (settings.capture_audio) { + system_audio = std::make_unique(settings); + } +#endif if (UsesWaylandPortal()) { #if defined(HAVE_WAYLAND_CAPTURE) backend_reader = CreateWaylandScreenCaptureReader(settings, info); @@ -200,6 +609,23 @@ bool ScreenCaptureReader::IsBackendSupported(ScreenCaptureBackend backend) #endif } +bool ScreenCaptureReader::IsSystemAudioSupported(ScreenCaptureBackend backend) +{ +#if defined(__linux__) + if (backend == SCREEN_CAPTURE_AUTO) { + backend = DefaultBackend(); + } + avdevice_register_all(); + return (backend == SCREEN_CAPTURE_X11 || backend == SCREEN_CAPTURE_WAYLAND) + && av_find_input_format("pulse") != nullptr; +#elif defined(_WIN32) + return backend == SCREEN_CAPTURE_AUTO || backend == SCREEN_CAPTURE_WINDOWS_GDI; +#else + (void) backend; + return false; +#endif +} + ScreenCaptureBackend ScreenCaptureReader::DefaultBackend() { #if defined(__linux__) @@ -232,6 +658,15 @@ void ScreenCaptureReader::ValidateSettings() const if (settings.fps.num <= 0 || settings.fps.den <= 0) { throw InvalidOptions("Screen capture requires a positive frame rate."); } + if (settings.capture_audio && !IsSystemAudioSupported(settings.backend)) { + throw InvalidOptions("System audio capture is not supported by this screen capture backend."); + } + if (settings.audio_sample_rate < 8000) { + throw InvalidSampleRate("System audio capture requires a sample rate of at least 8000 Hz."); + } + if (settings.audio_channels < 1 || settings.audio_channels > 2) { + throw InvalidChannels("System audio capture requires one or two channels."); + } } bool ScreenCaptureReader::UsesFFmpegDevice() const @@ -297,7 +732,7 @@ std::string ScreenCaptureReader::InputName() const void ScreenCaptureReader::PopulateInfo() { info.has_video = true; - info.has_audio = false; + info.has_audio = settings.capture_audio; info.has_single_image = false; info.duration = 60.0f * 60.0f; info.file_size = 0; @@ -315,11 +750,11 @@ void ScreenCaptureReader::PopulateInfo() info.video_timebase = settings.fps.Reciprocal(); info.interlaced_frame = false; info.top_field_first = false; - info.acodec = ""; + info.acodec = settings.capture_audio ? "pcm_f32le" : ""; info.audio_bit_rate = 0; - info.sample_rate = 0; - info.channels = 0; - info.channel_layout = LAYOUT_MONO; + info.sample_rate = settings.capture_audio ? settings.audio_sample_rate : 0; + info.channels = settings.capture_audio ? settings.audio_channels : 0; + info.channel_layout = settings.audio_channels == 2 ? LAYOUT_STEREO : LAYOUT_MONO; info.audio_stream_index = -1; info.audio_timebase = Fraction(1, 1); } @@ -327,15 +762,40 @@ void ScreenCaptureReader::PopulateInfo() void ScreenCaptureReader::Open() { if (backend_reader) { - backend_reader->Open(); + if (backend_reader->IsOpen()) return; + manual_system_audio = false; + if (system_audio) { + system_audio->Open(); + info.sample_rate = system_audio->SampleRate(); + info.channels = system_audio->Channels(); + info.channel_layout = info.channels == 1 ? LAYOUT_MONO : LAYOUT_STEREO; + } + try { + backend_reader->Open(); + } catch (...) { + if (system_audio) system_audio->Close(); + throw; + } return; } if (is_open) { return; } + manual_system_audio = false; + if (system_audio) { + system_audio->Open(); + info.sample_rate = system_audio->SampleRate(); + info.channels = system_audio->Channels(); + info.channel_layout = info.channels == 1 ? LAYOUT_MONO : LAYOUT_STEREO; + } close_requested = false; - OpenDevice(); - OpenDecoder(); + try { + OpenDevice(); + OpenDecoder(); + } catch (...) { + if (system_audio) system_audio->Close(); + throw; + } is_open = true; } @@ -465,14 +925,33 @@ std::shared_ptr ScreenCaptureReader::GetFrame(int64_t number) throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame()."); } const std::lock_guard lock(getFrameMutex); - return backend_reader->GetFrame(number); + auto frame = backend_reader->GetFrame(number); + if (system_audio && !manual_system_audio) system_audio->AddFrameAudio(frame, number, info.fps); + return frame; } if (!is_open) { throw ReaderClosed("The ScreenCaptureReader is closed. Call Open() before GetFrame()."); } const std::lock_guard lock(getFrameMutex); - return DecodeNextFrame(number); + auto frame = DecodeNextFrame(number); + if (system_audio && !manual_system_audio) system_audio->AddFrameAudio(frame, number, info.fps); + return frame; +} + +void ScreenCaptureReader::AddSystemAudio(std::shared_ptr frame, int64_t output_frame_number) +{ + if (system_audio) { + system_audio->AddFrameAudio(frame, output_frame_number, info.fps); + } +} + +void ScreenCaptureReader::ResetSystemAudio() +{ + manual_system_audio = true; + if (system_audio) { + system_audio->Reset(); + } } std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) @@ -596,6 +1075,9 @@ std::shared_ptr ScreenCaptureReader::DecodeNextFrame(int64_t number) void ScreenCaptureReader::Close() { close_requested = true; + if (system_audio) { + system_audio->Close(); + } if (backend_reader) { backend_reader->Close(); } @@ -655,6 +1137,10 @@ Json::Value ScreenCaptureReader::JsonValue() const root["fps"]["den"] = settings.fps.den; root["include_cursor"] = settings.include_cursor; root["show_region"] = settings.show_region; + root["capture_audio"] = settings.capture_audio; + root["audio_device"] = settings.audio_device; + root["audio_sample_rate"] = settings.audio_sample_rate; + root["audio_channels"] = settings.audio_channels; root["options"] = Json::Value(Json::objectValue); for (const auto& option : settings.options) { root["options"][option.first] = option.second; @@ -695,6 +1181,14 @@ void ScreenCaptureReader::SetJsonValue(const Json::Value root) settings.include_cursor = root["include_cursor"].asBool(); if (!root["show_region"].isNull()) settings.show_region = root["show_region"].asBool(); + if (!root["capture_audio"].isNull()) + settings.capture_audio = root["capture_audio"].asBool(); + if (!root["audio_device"].isNull()) + settings.audio_device = root["audio_device"].asString(); + if (!root["audio_sample_rate"].isNull()) + settings.audio_sample_rate = root["audio_sample_rate"].asInt(); + if (!root["audio_channels"].isNull()) + settings.audio_channels = root["audio_channels"].asInt(); if (!root["options"].isNull() && root["options"].isObject()) { settings.options.clear(); for (const auto& key : root["options"].getMemberNames()) { @@ -706,4 +1200,10 @@ void ScreenCaptureReader::SetJsonValue(const Json::Value root) } ValidateSettings(); PopulateInfo(); +#if defined(__linux__) || defined(_WIN32) + system_audio.reset(); + if (settings.capture_audio) { + system_audio = std::make_unique(settings); + } +#endif } diff --git a/src/ScreenCaptureReader.h b/src/ScreenCaptureReader.h index 003328d3d..5b37c6758 100644 --- a/src/ScreenCaptureReader.h +++ b/src/ScreenCaptureReader.h @@ -44,6 +44,10 @@ namespace openshot openshot::Fraction fps = openshot::Fraction(30, 1); bool include_cursor = true; bool show_region = false; + bool capture_audio = false; + std::string audio_device; + int audio_sample_rate = 48000; + int audio_channels = 2; std::map options; }; @@ -86,8 +90,11 @@ namespace openshot void Open() override; openshot::CaptureReaderStats GetStats() const; + void AddSystemAudio(std::shared_ptr frame, int64_t output_frame_number); + void ResetSystemAudio(); ScreenCaptureSettings GetSettings() const { return settings; }; static bool IsBackendSupported(ScreenCaptureBackend backend); + static bool IsSystemAudioSupported(ScreenCaptureBackend backend); static ScreenCaptureBackend DefaultBackend(); private: @@ -116,6 +123,11 @@ namespace openshot AVPacket* packet; SwsContext* sws_context; std::atomic close_requested; + bool manual_system_audio = false; +#ifndef SWIG + class SystemAudioCapture; + std::unique_ptr system_audio; +#endif }; } diff --git a/tests/ScreenCaptureReader.cpp b/tests/ScreenCaptureReader.cpp index c12bc0890..bb5a024b5 100644 --- a/tests/ScreenCaptureReader.cpp +++ b/tests/ScreenCaptureReader.cpp @@ -104,6 +104,66 @@ TEST_CASE("Screen capture reader reports configured video info", "[libopenshot][ #endif } +TEST_CASE("Screen capture system audio settings follow backend capability", "[libopenshot][screencapturereader][audio]") +{ + ScreenCaptureSettings settings; +#if defined(__linux__) + settings.backend = SCREEN_CAPTURE_X11; + settings.display = ":99.0"; +#elif defined(_WIN32) + settings.backend = SCREEN_CAPTURE_WINDOWS_GDI; + settings.display = "desktop"; +#elif defined(__APPLE__) + settings.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; + settings.display = "Capture screen 0:none"; +#else + return; +#endif + settings.width = 640; + settings.height = 360; + settings.capture_audio = true; + settings.audio_device = "test-output"; + settings.audio_sample_rate = 48000; + settings.audio_channels = 2; + + if (ScreenCaptureReader::IsSystemAudioSupported(settings.backend)) { + ScreenCaptureReader reader(settings); + CHECK(reader.info.has_audio); + CHECK(reader.info.sample_rate == 48000); + CHECK(reader.info.channels == 2); + CHECK(reader.info.channel_layout == LAYOUT_STEREO); + const Json::Value json = reader.JsonValue(); + CHECK(json["capture_audio"].asBool()); + CHECK(json["audio_device"].asString() == "test-output"); + CHECK(json["audio_sample_rate"].asInt() == 48000); + CHECK(json["audio_channels"].asInt() == 2); + } else { + CHECK_THROWS_AS([&settings]() { ScreenCaptureReader reader(settings); }(), InvalidOptions); + } +} + +TEST_CASE("Screen capture rejects invalid system audio formats", "[libopenshot][screencapturereader][audio]") +{ + ScreenCaptureSettings settings; +#if defined(__linux__) + settings.backend = SCREEN_CAPTURE_X11; + settings.display = ":99.0"; +#elif defined(_WIN32) + settings.backend = SCREEN_CAPTURE_WINDOWS_GDI; + settings.display = "desktop"; +#elif defined(__APPLE__) + settings.backend = SCREEN_CAPTURE_MAC_AVFOUNDATION; + settings.display = "Capture screen 0:none"; +#else + return; +#endif + settings.audio_sample_rate = 7999; + CHECK_THROWS_AS([&settings]() { ScreenCaptureReader reader(settings); }(), InvalidSampleRate); + settings.audio_sample_rate = 48000; + settings.audio_channels = 3; + CHECK_THROWS_AS([&settings]() { ScreenCaptureReader reader(settings); }(), InvalidChannels); +} + TEST_CASE("Screen capture backend support follows platform build features", "[libopenshot][screencapturereader]") { #if defined(__linux__) From 2547c82e985b8f1b1cbe68353c0e45aeb5d76d4e Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 19 Jul 2026 14:44:36 -0500 Subject: [PATCH 28/30] Fix MinGW Core Audio GUID linking --- src/ScreenCaptureReader.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 9d532e19e..30ac8bd87 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -30,6 +30,9 @@ extern "C" { } #if defined(_WIN32) + // Instantiate the Core Audio COM GUIDs in this translation unit. Older + // MinGW-w64 uuid import libraries do not provide all WASAPI identifiers. + #include #include #include #endif From 114f25a08f633d1fabe6117543f41d5636a21789 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 19 Jul 2026 16:11:39 -0500 Subject: [PATCH 29/30] Synchronize initial system audio capture --- src/ScreenCaptureReader.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/ScreenCaptureReader.cpp b/src/ScreenCaptureReader.cpp index 30ac8bd87..10359421b 100644 --- a/src/ScreenCaptureReader.cpp +++ b/src/ScreenCaptureReader.cpp @@ -131,9 +131,20 @@ class ScreenCaptureReader::SystemAudioCapture if (!frame || sample_count <= 0) return; std::unique_lock lock(queue_mutex); - ready.wait_for(lock, std::chrono::milliseconds(100), [this, sample_count]() { + // The capture backend commonly delivers its first buffer later than the + // first video frame. Writing silence after the old 100 ms timeout moved + // every subsequently-delivered sample later on the recording timeline. + // Allow the first frame to establish the audio epoch before falling back + // to the short steady-state wait used for later frames. + const auto wait_time = timeline_started + ? std::chrono::milliseconds(100) + : std::chrono::milliseconds(3000); + ready.wait_for(lock, wait_time, [this, sample_count]() { return close_requested || (!channels.empty() && static_cast(channels[0].size()) >= sample_count); }); + if (!channels.empty() && static_cast(channels[0].size()) >= sample_count) { + timeline_started = true; + } frame->SampleRate(settings.audio_sample_rate); frame->ChannelsLayout(settings.audio_channels == 1 ? LAYOUT_MONO : LAYOUT_STEREO); for (int channel = 0; channel < settings.audio_channels; ++channel) { @@ -154,6 +165,7 @@ class ScreenCaptureReader::SystemAudioCapture std::lock_guard lock(queue_mutex); for (auto& channel : channels) channel.clear(); last_output_frame = 0; + timeline_started = false; } int SampleRate() const { return settings.audio_sample_rate; } @@ -236,6 +248,7 @@ class ScreenCaptureReader::SystemAudioCapture std::condition_variable ready; std::vector> channels; int64_t last_output_frame = 0; + bool timeline_started = false; }; #elif defined(_WIN32) class ScreenCaptureReader::SystemAudioCapture @@ -278,9 +291,18 @@ class ScreenCaptureReader::SystemAudioCapture last_output_frame = std::max(last_output_frame, number); if (!frame || sample_count <= 0) return; std::unique_lock lock(queue_mutex); - ready.wait_for(lock, std::chrono::milliseconds(100), [this, sample_count]() { + // WASAPI can take longer than one video-frame interval to make its first + // loopback packet available. Keep that startup latency out of the encoded + // media timeline by waiting for the first complete frame of audio. + const auto wait_time = timeline_started + ? std::chrono::milliseconds(100) + : std::chrono::milliseconds(3000); + ready.wait_for(lock, wait_time, [this, sample_count]() { return close_requested || (!channels.empty() && static_cast(channels[0].size()) >= sample_count); }); + if (!channels.empty() && static_cast(channels[0].size()) >= sample_count) { + timeline_started = true; + } frame->SampleRate(sample_rate); frame->ChannelsLayout(channel_count == 1 ? LAYOUT_MONO : LAYOUT_STEREO); for (int channel = 0; channel < channel_count; ++channel) { @@ -299,6 +321,7 @@ class ScreenCaptureReader::SystemAudioCapture std::lock_guard lock(queue_mutex); for (auto& channel : channels) channel.clear(); last_output_frame = 0; + timeline_started = false; } int SampleRate() const { return sample_rate; } @@ -415,6 +438,7 @@ class ScreenCaptureReader::SystemAudioCapture std::condition_variable ready; std::vector> channels; int64_t last_output_frame = 0; + bool timeline_started = false; int sample_rate = 48000; int channel_count = 2; std::mutex state_mutex; From a2709d50cbde4059bb5cff018dd9c4ee42d16b88 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 22 Jul 2026 12:43:43 -0500 Subject: [PATCH 30/30] Install FFmpeg avdevice in GitHub CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ceb028a56..eec88f957 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: cmake swig doxygen graphviz curl lcov \ libasound2-dev \ qtbase5-dev qtbase5-dev-tools libqt5svg5-dev \ - libfdk-aac-dev libavcodec-dev libavformat-dev \ + libfdk-aac-dev libavcodec-dev libavdevice-dev libavformat-dev \ libavutil-dev libswscale-dev libswresample-dev \ libzmq3-dev libbabl-dev \ libopencv-dev libprotobuf-dev protobuf-compiler \