From 2dc9d243236ca7126ceeb870f80eb21baf8f3831 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Thu, 31 Aug 2017 15:34:00 +0200 Subject: [PATCH 01/18] FilterGraph: push input frames into the filter buffer Let the filter managing the output frame size, using its internal frame buffer queue. --- src/AvTranscoder/filter/FilterGraph.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 11b7346e..ed91bbad 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -44,7 +44,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) // setup input frames for(size_t index = 0; index < inputs.size(); ++index) { - const int ret = av_buffersrc_write_frame(_filters.at(index)->getAVFilterContext(), &inputs.at(index)->getAVFrame()); + const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputs.at(index)->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); if(ret < 0) { throw std::runtime_error("Error when adding a frame to the source buffer used to start to process filters: " + From 80017d638271dc84a41a1bb74cfdc8c86b191f07 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Tue, 5 Sep 2017 16:43:22 +0200 Subject: [PATCH 02/18] FilterGraph: add new internal FrameBuffer class Used as frame buffers on the filter graph inputs. For the moment, this first version only handles audio frames. --- src/AvTranscoder/filter/FilterGraph.cpp | 98 ++++++++++++++++++++++++- src/AvTranscoder/filter/FilterGraph.hpp | 45 ++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index ed91bbad..86362585 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -1,7 +1,6 @@ #include "FilterGraph.hpp" #include -#include #include extern "C" { @@ -16,6 +15,103 @@ extern "C" { namespace avtranscoder { + +/****************** + + FrameBuffer + + ******************/ + +FrameBuffer::FrameBuffer(const AudioFrameDesc& audioFrameDesc) + : _audioFrameDesc(audioFrameDesc) + , _frameQueue() + , _totalDataSize(0) + , _positionInFrontFrame(0) +{ +} + +FrameBuffer::~FrameBuffer() +{ + for (int i = 0; i < _frameQueue.size(); ++i) + popFrame(); +} + +void FrameBuffer::addFrame(IFrame* frame) +{ + AudioFrame* newAudioFrame = new AudioFrame(_audioFrameDesc, false); + const size_t expectedNbSamples = frame->getDataSize() / (newAudioFrame->getNbChannels() * newAudioFrame->getBytesPerSample()); + newAudioFrame->setNbSamplesPerChannel(expectedNbSamples); + newAudioFrame->allocateData(); + newAudioFrame->copyData(*frame); + + _totalDataSize += newAudioFrame->getDataSize(); + _frameQueue.push(newAudioFrame); +} + +void FrameBuffer::popFrame() +{ + _frameQueue.pop(); +} + +IFrame* FrameBuffer::getFrame(const size_t size) +{ + IFrame* next = _frameQueue.front(); + const size_t nextFrameSize = next->getDataSize(); + + // If no expected size, or if the expected size equals the front frame of the queue (with no offset) + if(size == 0 || (size == nextFrameSize && _positionInFrontFrame == 0)) + { + _totalDataSize -= nextFrameSize; + popFrame(); + return next; + } + + // Create a new frame + AudioFrame* newAudioFrame = new AudioFrame(_audioFrameDesc, false); + const size_t expectedNbSamples = size / (newAudioFrame->getNbChannels() * newAudioFrame->getBytesPerSample()); + newAudioFrame->setNbSamplesPerChannel(expectedNbSamples); + newAudioFrame->allocateData(); + + // Concatenate frames data + size_t extractedDataSize = 0; + unsigned char* outputData = new unsigned char[size]; + while(extractedDataSize != size && _frameQueue.size() != 0) + { + next = _frameQueue.front(); + size_t dataToGet = size - extractedDataSize; + size_t remainingDataInNextFrame = next->getDataSize() - _positionInFrontFrame; + + if(dataToGet > remainingDataInNextFrame) + dataToGet = remainingDataInNextFrame; + + for(size_t i = 0; i < dataToGet; i++) + outputData[extractedDataSize++] = next->getData()[0][_positionInFrontFrame + i]; + + if(dataToGet < remainingDataInNextFrame) + { + _positionInFrontFrame += dataToGet; + } + else + { + popFrame(); + _positionInFrontFrame = 0; + } + } + + _totalDataSize -= extractedDataSize; + newAudioFrame->assignBuffer(outputData); + + return newAudioFrame; +} + + + +/****************** + + FilterGraph + + ******************/ + FilterGraph::FilterGraph(const ICodec& codec) : _graph(avfilter_graph_alloc()) , _filters() diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 287f6e9f..6822942e 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -5,14 +5,59 @@ #include #include #include +#include #include +#include struct AVFilterGraph; namespace avtranscoder { +/** + * @brief Filter graph input frame buffer. + * This FIFO buffer contains IFrame pointers and can deliver specific size frames. + * + * @todo Only for audio frame, for the moment. Make it usable with video frames. + **/ +class FrameBuffer +{ +public: + FrameBuffer(const AudioFrameDesc& audioFrameDesc); + ~FrameBuffer(); + + /** + * @brief Return whether the buffer is empty or not. + */ + bool isEmpty() { return _frameQueue.empty() && _totalDataSize == 0; } + /** + * @brief Return the total amount of data contained in the frames of the buffer. + */ + size_t getDataSize() { return _totalDataSize; } + + /** + * @brief Push a frame at the end of the buffer. + */ + void addFrame(IFrame* frame); + + /** + * @brief Retrieve a IFrame pointer of the specified size, from the beginning of the buffer. + * If no size is specified, the whole first IFrame pointer is returned. + */ + IFrame* getFrame(const size_t size = 0); + +private: + void popFrame(); + + const AudioFrameDesc _audioFrameDesc; + + std::queue _frameQueue; + size_t _totalDataSize; + size_t _positionInFrontFrame; + +}; + /** * @brief Manager of filters. **/ From cb8ae5d079999ead16b7237efdb9c9716e76dd24 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Tue, 5 Sep 2017 16:49:15 +0200 Subject: [PATCH 03/18] FilterGraph: use FrameBuffers for each filter graph input Returning frames with the same size as inputs of the graph, avoiding to overflow the internal filter buffers. --- src/AvTranscoder/filter/FilterGraph.cpp | 37 +++++++++++++++++++++++-- src/AvTranscoder/filter/FilterGraph.hpp | 4 +++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 86362585..0bef2d24 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -124,6 +124,7 @@ FilterGraph::FilterGraph(const ICodec& codec) FilterGraph::~FilterGraph() { + _inputFramesBuffer.clear(); for(std::vector::iterator it = _filters.begin(); it < _filters.end(); ++it) { delete(*it); @@ -131,6 +132,20 @@ FilterGraph::~FilterGraph() avfilter_graph_free(&_graph); } +size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) +{ + if(!inputs.size()) + return 0; + + int minFrameSize = inputs.at(0)->getDataSize(); + for(size_t index = 1; index < inputs.size(); ++index) + { + if(minFrameSize > inputs.at(index)->getDataSize()) + minFrameSize = inputs.at(index)->getDataSize(); + } + return minFrameSize; +} + void FilterGraph::process(const std::vector& inputs, IFrame& output) { // init filter graph @@ -138,9 +153,22 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) init(inputs, output); // setup input frames + + // Fill the frame buffer with inputs for(size_t index = 0; index < inputs.size(); ++index) { - const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputs.at(index)->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); + _inputFramesBuffer.at(index).addFrame(inputs.at(index)); + } + + // Get the minimum input frames size + const size_t minInputFrameSize = getMinInputFrameSize(inputs); + + // Setup input frames into the filter graph + for(size_t index = 0; index < inputs.size(); ++index) + { + IFrame* inputBufferedFrame = _inputFramesBuffer.at(index).getFrame(minInputFrameSize); + const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputBufferedFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); + if(ret < 0) { throw std::runtime_error("Error when adding a frame to the source buffer used to start to process filters: " + @@ -148,7 +176,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) } } - // pull filtered data from the filter graph + // Pull filtered data from the filter graph for(;;) { const int ret = av_buffersink_get_frame(_filters.at(_filters.size() - 1)->getAVFilterContext(), &output.getAVFrame()); @@ -246,6 +274,11 @@ void FilterGraph::addInBuffer(const std::vector& inputs) filterOptions << "sample_rate=" << audioFrame->getSampleRate() << ":"; filterOptions << "sample_fmt=" << getSampleFormatName(audioFrame->getSampleFormat()) << ":"; filterOptions << "channel_layout=0x" << std::hex << audioFrame->getChannelLayout(); + + const AudioFrameDesc audioFrameDesc(audioFrame->getSampleRate(), + audioFrame->getNbChannels(), + getSampleFormatName(audioFrame->getSampleFormat())); + _inputFramesBuffer.push_back(FrameBuffer(audioFrameDesc)); } // video frame else if((*it)->isVideoFrame()) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 6822942e..e1e62418 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -127,11 +127,15 @@ class AvExport FilterGraph void addOutBuffer(const IFrame& output); //@} + size_t getMinInputFrameSize(const std::vector& inputs); + private: AVFilterGraph* _graph; ///< The graph which holds the filters. std::vector _filters; ///< List of filters to process. const ICodec& _codec; ///< Codec of the stream on which the filters will be applied. + std::vector _inputFramesBuffer; + /** * @brief Is the FilterGraph initialized. * @see init From 3864d7e5ceedc6e4d0d8c1ba6fe7c36a5e5a944c Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Tue, 5 Sep 2017 16:58:37 +0200 Subject: [PATCH 04/18] FilterGraph: improve input frame size computing --- src/AvTranscoder/filter/FilterGraph.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 0bef2d24..2fbcb632 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -140,7 +140,8 @@ size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) int minFrameSize = inputs.at(0)->getDataSize(); for(size_t index = 1; index < inputs.size(); ++index) { - if(minFrameSize > inputs.at(index)->getDataSize()) + // if the input frame is shorter, and if there is no data enough into the corresponding frame buffer + if(minFrameSize > inputs.at(index)->getDataSize() && minFrameSize > _inputFramesBuffer.at(index).getDataSize()) minFrameSize = inputs.at(index)->getDataSize(); } return minFrameSize; From 053f9bebb06625c903c81776353a6652aa7659fe Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Tue, 5 Sep 2017 17:21:39 +0200 Subject: [PATCH 05/18] FilterGraph: frame buffers can be bypassed if not necessary If the input frames have the same size and the buffers are empty, bypass to avoid useless data copies. --- src/AvTranscoder/filter/FilterGraph.cpp | 54 ++++++++++++++++++++----- src/AvTranscoder/filter/FilterGraph.hpp | 2 + 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 2fbcb632..d6eb71a0 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -147,28 +147,64 @@ size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) return minFrameSize; } +bool FilterGraph::areInputFrameSizeEqual(const std::vector& inputs) +{ + if(!inputs.size() || inputs.size() == 1) + return true; + + const int frameSize = inputs.at(0)->getDataSize(); + for(size_t index = 1; index < inputs.size(); ++index) + { + if(frameSize != inputs.at(index)->getDataSize()) + return false; + } + return true; +} + +bool FilterGraph::areFrameBuffersEmpty() +{ + if(!_inputFramesBuffer.size()) + return true; + + for(std::vector::iterator it = _inputFramesBuffer.begin(); it != _inputFramesBuffer.end(); ++it) + { + if(!it->isEmpty()) + return false; + } + return true; +} + void FilterGraph::process(const std::vector& inputs, IFrame& output) { - // init filter graph + // Init the filter graph if(!_isInit) init(inputs, output); - // setup input frames + // Check whether we can bypass the input buffers + const bool bypassBuffers = areInputFrameSizeEqual(inputs) && areFrameBuffersEmpty(); + size_t minInputFrameSize = 0; - // Fill the frame buffer with inputs - for(size_t index = 0; index < inputs.size(); ++index) + if(!bypassBuffers) { - _inputFramesBuffer.at(index).addFrame(inputs.at(index)); + // Fill the frame buffer with inputs + for(size_t index = 0; index < inputs.size(); ++index) + _inputFramesBuffer.at(index).addFrame(inputs.at(index)); + + // Get the minimum input frames size + minInputFrameSize = getMinInputFrameSize(inputs); } - // Get the minimum input frames size - const size_t minInputFrameSize = getMinInputFrameSize(inputs); // Setup input frames into the filter graph for(size_t index = 0; index < inputs.size(); ++index) { - IFrame* inputBufferedFrame = _inputFramesBuffer.at(index).getFrame(minInputFrameSize); - const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputBufferedFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); + IFrame* inputFrame = NULL; + if(bypassBuffers) + inputFrame = inputs.at(index); + else + inputFrame = _inputFramesBuffer.at(index).getFrame(minInputFrameSize); + + const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); if(ret < 0) { diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index e1e62418..02a4d65d 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -128,6 +128,8 @@ class AvExport FilterGraph //@} size_t getMinInputFrameSize(const std::vector& inputs); + bool areInputFrameSizeEqual(const std::vector& inputs); + bool areFrameBuffersEmpty(); private: AVFilterGraph* _graph; ///< The graph which holds the filters. From 380f2382f2e0c2073aa2a00b2971f32abbe89d1e Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Wed, 6 Sep 2017 12:03:59 +0200 Subject: [PATCH 06/18] FilterGraph: minor clean into FrameBuffer class --- src/AvTranscoder/filter/FilterGraph.cpp | 2 +- src/AvTranscoder/filter/FilterGraph.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index d6eb71a0..f0d450c0 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -32,7 +32,7 @@ FrameBuffer::FrameBuffer(const AudioFrameDesc& audioFrameDesc) FrameBuffer::~FrameBuffer() { - for (int i = 0; i < _frameQueue.size(); ++i) + for(size_t i = 0; i < _frameQueue.size(); ++i) popFrame(); } diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 02a4d65d..9fbb4968 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -30,11 +30,11 @@ class FrameBuffer /** * @brief Return whether the buffer is empty or not. */ - bool isEmpty() { return _frameQueue.empty() && _totalDataSize == 0; } + bool isEmpty() const { return _frameQueue.empty() && _totalDataSize == 0; } /** * @brief Return the total amount of data contained in the frames of the buffer. */ - size_t getDataSize() { return _totalDataSize; } + size_t getDataSize() const { return _totalDataSize; } /** * @brief Push a frame at the end of the buffer. @@ -50,7 +50,7 @@ class FrameBuffer private: void popFrame(); - const AudioFrameDesc _audioFrameDesc; + AudioFrameDesc _audioFrameDesc; std::queue _frameQueue; size_t _totalDataSize; From 27cf448a274f9a131c5c944b7b047ae098753286 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Wed, 6 Sep 2017 12:11:23 +0200 Subject: [PATCH 07/18] FilterGraph: refactoring of getMinInputFrameSize() private method Add a new getAvailableFrameSize() private method --- src/AvTranscoder/filter/FilterGraph.cpp | 16 ++++++++++++---- src/AvTranscoder/filter/FilterGraph.hpp | 8 ++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index f0d450c0..c2fc0698 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -132,17 +132,25 @@ FilterGraph::~FilterGraph() avfilter_graph_free(&_graph); } +size_t FilterGraph::getAvailableFrameSize(const std::vector& inputs, const size_t& index) +{ + size_t frameSize = inputs.at(index)->getDataSize(); + if(frameSize == 0) + frameSize = _inputFramesBuffer.at(index).getDataSize(); + return frameSize; +} + size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) { if(!inputs.size()) return 0; - int minFrameSize = inputs.at(0)->getDataSize(); + size_t minFrameSize = getAvailableFrameSize(inputs, 0); for(size_t index = 1; index < inputs.size(); ++index) { - // if the input frame is shorter, and if there is no data enough into the corresponding frame buffer - if(minFrameSize > inputs.at(index)->getDataSize() && minFrameSize > _inputFramesBuffer.at(index).getDataSize()) - minFrameSize = inputs.at(index)->getDataSize(); + const size_t availableFrameSize = getAvailableFrameSize(inputs, index); + if(minFrameSize > availableFrameSize) + minFrameSize = availableFrameSize; } return minFrameSize; } diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 9fbb4968..edcb3245 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -127,7 +127,15 @@ class AvExport FilterGraph void addOutBuffer(const IFrame& output); //@} + /** + * @brief Return the input frame size if not null, or the available size into the corresponding frame buffer + */ + size_t getAvailableFrameSize(const std::vector& inputs, const size_t& index); + /** + * @brief Get the minimum size between input frames, or available frame buffers + */ size_t getMinInputFrameSize(const std::vector& inputs); + bool areInputFrameSizeEqual(const std::vector& inputs); bool areFrameBuffersEmpty(); From 82382fc08192029aa56265270cae19b2b079394c Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Wed, 6 Sep 2017 14:53:06 +0200 Subject: [PATCH 08/18] StreamTranscoder: flush filter graph buffers after decoding process Add utility methods, documentation and logs into FilterGraph and FrameBuffer classes --- src/AvTranscoder/filter/FilterGraph.cpp | 57 +++++++++++++++---- src/AvTranscoder/filter/FilterGraph.hpp | 7 +++ .../transcoder/StreamTranscoder.cpp | 24 ++++++-- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index c2fc0698..1c94c324 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -38,6 +38,8 @@ FrameBuffer::~FrameBuffer() void FrameBuffer::addFrame(IFrame* frame) { + LOG_DEBUG("Add a new frame to frame buffer. New buffer size: " << _frameQueue.size() + 1); + // Copy the input frame to store it into the queue AudioFrame* newAudioFrame = new AudioFrame(_audioFrameDesc, false); const size_t expectedNbSamples = frame->getDataSize() / (newAudioFrame->getNbChannels() * newAudioFrame->getBytesPerSample()); newAudioFrame->setNbSamplesPerChannel(expectedNbSamples); @@ -51,16 +53,19 @@ void FrameBuffer::addFrame(IFrame* frame) void FrameBuffer::popFrame() { _frameQueue.pop(); + LOG_DEBUG("Pop frame from buffer. Remaining frames in buffer: " << _frameQueue.size()); } IFrame* FrameBuffer::getFrame(const size_t size) { + LOG_DEBUG("Get a " << size << " bytes frame from a " << _totalDataSize << " bytes frame buffer"); IFrame* next = _frameQueue.front(); const size_t nextFrameSize = next->getDataSize(); // If no expected size, or if the expected size equals the front frame of the queue (with no offset) if(size == 0 || (size == nextFrameSize && _positionInFrontFrame == 0)) { + // Directly return the front frame of the queue _totalDataSize -= nextFrameSize; popFrame(); return next; @@ -77,22 +82,27 @@ IFrame* FrameBuffer::getFrame(const size_t size) unsigned char* outputData = new unsigned char[size]; while(extractedDataSize != size && _frameQueue.size() != 0) { + // Get the front frame from queue next = _frameQueue.front(); - size_t dataToGet = size - extractedDataSize; - size_t remainingDataInNextFrame = next->getDataSize() - _positionInFrontFrame; + size_t remainingDataInFrontFrame = next->getDataSize() - _positionInFrontFrame; - if(dataToGet > remainingDataInNextFrame) - dataToGet = remainingDataInNextFrame; + // Compute the data size to get from the frame + size_t dataToGet = size - extractedDataSize; + if(dataToGet > remainingDataInFrontFrame) + dataToGet = remainingDataInFrontFrame; + // Copy the data from the frame to temporal buffer for(size_t i = 0; i < dataToGet; i++) outputData[extractedDataSize++] = next->getData()[0][_positionInFrontFrame + i]; - if(dataToGet < remainingDataInNextFrame) + if(dataToGet < remainingDataInFrontFrame) { + // Set new position into front frame _positionInFrontFrame += dataToGet; } else { + // The whole front frame has been read, so pop it from queue popFrame(); _positionInFrontFrame = 0; } @@ -100,7 +110,6 @@ IFrame* FrameBuffer::getFrame(const size_t size) _totalDataSize -= extractedDataSize; newAudioFrame->assignBuffer(outputData); - return newAudioFrame; } @@ -155,6 +164,27 @@ size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) return minFrameSize; } +bool FilterGraph::hasBufferedFrames() +{ + if(!_inputFramesBuffer.size()) + return false; + + for(std::vector::iterator it = _inputFramesBuffer.begin(); it != _inputFramesBuffer.end(); ++it) + { + if(it->isEmpty()) + return false; + } + return true; +} + +bool FilterGraph::hasBufferedFrames(const size_t index) +{ + if(index >= _inputFramesBuffer.size()) + return false; + + return !_inputFramesBuffer.at(index).isEmpty(); +} + bool FilterGraph::areInputFrameSizeEqual(const std::vector& inputs) { if(!inputs.size() || inputs.size() == 1) @@ -196,7 +226,14 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) { // Fill the frame buffer with inputs for(size_t index = 0; index < inputs.size(); ++index) + { + if(!inputs.at(index)->getDataSize()) + { + LOG_DEBUG("Empty frame from filter graph input " << index << ". Remaining frames in buffer: " << _inputFramesBuffer.at(index).getBufferSize()); + continue; + } _inputFramesBuffer.at(index).addFrame(inputs.at(index)); + } // Get the minimum input frames size minInputFrameSize = getMinInputFrameSize(inputs); @@ -206,12 +243,8 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) // Setup input frames into the filter graph for(size_t index = 0; index < inputs.size(); ++index) { - IFrame* inputFrame = NULL; - if(bypassBuffers) - inputFrame = inputs.at(index); - else - inputFrame = _inputFramesBuffer.at(index).getFrame(minInputFrameSize); - + // Retrieve frame from buffer or directly from input + IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputFramesBuffer.at(index).getFrame(minInputFrameSize); const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); if(ret < 0) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index edcb3245..862ce4dd 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -35,6 +35,10 @@ class FrameBuffer * @brief Return the total amount of data contained in the frames of the buffer. */ size_t getDataSize() const { return _totalDataSize; } + /** + * @brief Return the number of frames contained in the buffer. + */ + size_t getBufferSize() const { return _frameQueue.size(); } /** * @brief Push a frame at the end of the buffer. @@ -107,6 +111,9 @@ class AvExport FilterGraph */ bool hasFilters() const { return !_filters.empty(); } + bool hasBufferedFrames(); + bool hasBufferedFrames(const size_t index); + private: /** * @brief Initialize the graph of filters to process. diff --git a/src/AvTranscoder/transcoder/StreamTranscoder.cpp b/src/AvTranscoder/transcoder/StreamTranscoder.cpp index e3133342..f22d3c84 100644 --- a/src/AvTranscoder/transcoder/StreamTranscoder.cpp +++ b/src/AvTranscoder/transcoder/StreamTranscoder.cpp @@ -533,7 +533,7 @@ bool StreamTranscoder::processTranscode() // Decode LOG_DEBUG("Decode next frame") - bool decodingStatus = true; + std::vector decodingStatus(_generators.size(), true); for(size_t index = 0; index < _generators.size(); ++index) { if(getProcessCase() == eProcessCaseTranscode) @@ -542,15 +542,17 @@ bool StreamTranscoder::processTranscode() _currentDecoder = _generators.at(index); if(! _inputStreamDesc.empty() && _inputStreamDesc.at(index).demultiplexing()) - decodingStatus = decodingStatus && _currentDecoder->decodeNextFrame(*_decodedData.at(index), _inputStreamDesc.at(index)._channelIndexArray); + decodingStatus.at(index) = decodingStatus.at(index) && _currentDecoder->decodeNextFrame(*_decodedData.at(index), _inputStreamDesc.at(index)._channelIndexArray); else - decodingStatus = decodingStatus && _currentDecoder->decodeNextFrame(*_decodedData.at(index)); + decodingStatus.at(index) = decodingStatus.at(index) && _currentDecoder->decodeNextFrame(*_decodedData.at(index)); } // check the next data buffers in case of audio frames if(_decodedData.at(0)->isAudioFrame()) { const int nbInputSamplesPerChannel = _decodedData.at(0)->getAVFrame().nb_samples; + + // Reallocate output frame if(nbInputSamplesPerChannel > _filteredData->getAVFrame().nb_samples) { LOG_WARN("The buffer of filtered data corresponds to a frame of " << _filteredData->getAVFrame().nb_samples << " samples. The decoded buffer contains " << nbInputSamplesPerChannel << " samples. Reallocate it.") @@ -569,7 +571,21 @@ bool StreamTranscoder::processTranscode() // Transform CodedData data; - if(decodingStatus) + bool continueProcess = true; + for(size_t index = 0; index < decodingStatus.size(); ++index) + { + if(!decodingStatus.at(index)) + { + if(!_filterGraph->hasFilters() || !_filterGraph->hasBufferedFrames(index)) + { + continueProcess = false; + continue; + } + LOG_DEBUG("Some frames remain into filter graph buffer " << index); + } + } + + if(continueProcess) { IFrame* dataToTransform = NULL; if(_filterGraph->hasFilters()) From cbd47000f2731a3e302bcbeef3c6960ea0c1b6eb Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Wed, 6 Sep 2017 17:56:28 +0200 Subject: [PATCH 09/18] StreamTranscoder: reset frame that has not been decoded Avoid filling the filter graph buffers with wrong data frames. Improve documentation into FilterGraph::FrameBuffer class. --- src/AvTranscoder/filter/FilterGraph.cpp | 2 +- src/AvTranscoder/filter/FilterGraph.hpp | 2 +- src/AvTranscoder/transcoder/StreamTranscoder.cpp | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 1c94c324..0041a0eb 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -38,7 +38,7 @@ FrameBuffer::~FrameBuffer() void FrameBuffer::addFrame(IFrame* frame) { - LOG_DEBUG("Add a new frame to frame buffer. New buffer size: " << _frameQueue.size() + 1); + LOG_DEBUG("Add a new " << frame->getDataSize() << " bytes frame to frame buffer. New buffer size: " << _frameQueue.size() + 1); // Copy the input frame to store it into the queue AudioFrame* newAudioFrame = new AudioFrame(_audioFrameDesc, false); const size_t expectedNbSamples = frame->getDataSize() / (newAudioFrame->getNbChannels() * newAudioFrame->getBytesPerSample()); diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 862ce4dd..9f05156f 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -32,7 +32,7 @@ class FrameBuffer */ bool isEmpty() const { return _frameQueue.empty() && _totalDataSize == 0; } /** - * @brief Return the total amount of data contained in the frames of the buffer. + * @brief Return the total amount of available data contained in the frames of the buffer. */ size_t getDataSize() const { return _totalDataSize; } /** diff --git a/src/AvTranscoder/transcoder/StreamTranscoder.cpp b/src/AvTranscoder/transcoder/StreamTranscoder.cpp index f22d3c84..8ad966c4 100644 --- a/src/AvTranscoder/transcoder/StreamTranscoder.cpp +++ b/src/AvTranscoder/transcoder/StreamTranscoder.cpp @@ -582,6 +582,11 @@ bool StreamTranscoder::processTranscode() continue; } LOG_DEBUG("Some frames remain into filter graph buffer " << index); + + // Reset the non-decoded data as an empty frame + _decodedData.at(index)->freeData(); + _decodedData.at(index)->getAVFrame().format = -1; + _decodedData.at(index)->getAVFrame().nb_samples = 0; } } From 06b8b457cfb2c5de28b89874bb67718c9940f993 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Thu, 7 Sep 2017 15:42:56 +0200 Subject: [PATCH 10/18] FilterGraph: renames FrameBuffer to AudioFrameBuffer Since video frames cannot be split, this filter graph buffers will be specialized to audio frames. --- src/AvTranscoder/filter/FilterGraph.cpp | 49 ++++++++++++++----------- src/AvTranscoder/filter/FilterGraph.hpp | 19 +++++----- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 0041a0eb..be06fab0 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -18,11 +18,11 @@ namespace avtranscoder /****************** - FrameBuffer + AudioFramebuffer ******************/ -FrameBuffer::FrameBuffer(const AudioFrameDesc& audioFrameDesc) +AudioFramebuffer::AudioFramebuffer(const AudioFrameDesc& audioFrameDesc) : _audioFrameDesc(audioFrameDesc) , _frameQueue() , _totalDataSize(0) @@ -30,13 +30,13 @@ FrameBuffer::FrameBuffer(const AudioFrameDesc& audioFrameDesc) { } -FrameBuffer::~FrameBuffer() +AudioFramebuffer::~AudioFramebuffer() { for(size_t i = 0; i < _frameQueue.size(); ++i) popFrame(); } -void FrameBuffer::addFrame(IFrame* frame) +void AudioFramebuffer::addFrame(IFrame* frame) { LOG_DEBUG("Add a new " << frame->getDataSize() << " bytes frame to frame buffer. New buffer size: " << _frameQueue.size() + 1); // Copy the input frame to store it into the queue @@ -50,13 +50,13 @@ void FrameBuffer::addFrame(IFrame* frame) _frameQueue.push(newAudioFrame); } -void FrameBuffer::popFrame() +void AudioFramebuffer::popFrame() { _frameQueue.pop(); LOG_DEBUG("Pop frame from buffer. Remaining frames in buffer: " << _frameQueue.size()); } -IFrame* FrameBuffer::getFrame(const size_t size) +IFrame* AudioFramebuffer::getFrame(const size_t size) { LOG_DEBUG("Get a " << size << " bytes frame from a " << _totalDataSize << " bytes frame buffer"); IFrame* next = _frameQueue.front(); @@ -133,7 +133,7 @@ FilterGraph::FilterGraph(const ICodec& codec) FilterGraph::~FilterGraph() { - _inputFramesBuffer.clear(); + _inputAudioFramesBuffer.clear(); for(std::vector::iterator it = _filters.begin(); it < _filters.end(); ++it) { delete(*it); @@ -145,7 +145,7 @@ size_t FilterGraph::getAvailableFrameSize(const std::vector& inputs, co { size_t frameSize = inputs.at(index)->getDataSize(); if(frameSize == 0) - frameSize = _inputFramesBuffer.at(index).getDataSize(); + frameSize = _inputAudioFramesBuffer.at(index).getDataSize(); return frameSize; } @@ -166,10 +166,10 @@ size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) bool FilterGraph::hasBufferedFrames() { - if(!_inputFramesBuffer.size()) + if(!_inputAudioFramesBuffer.size()) return false; - for(std::vector::iterator it = _inputFramesBuffer.begin(); it != _inputFramesBuffer.end(); ++it) + for(std::vector::iterator it = _inputAudioFramesBuffer.begin(); it != _inputAudioFramesBuffer.end(); ++it) { if(it->isEmpty()) return false; @@ -179,13 +179,20 @@ bool FilterGraph::hasBufferedFrames() bool FilterGraph::hasBufferedFrames(const size_t index) { - if(index >= _inputFramesBuffer.size()) + if(index >= _inputAudioFramesBuffer.size()) return false; - return !_inputFramesBuffer.at(index).isEmpty(); + return !_inputAudioFramesBuffer.at(index).isEmpty(); } -bool FilterGraph::areInputFrameSizeEqual(const std::vector& inputs) +bool FilterGraph::areInputAudioFrames(const std::vector& inputs) +{ + if(!inputs.size()) + return false; + return typeid(*(inputs.at(0))) == typeid(AudioFrame); +} + +bool FilterGraph::areInputFrameSizesEqual(const std::vector& inputs) { if(!inputs.size() || inputs.size() == 1) return true; @@ -201,10 +208,10 @@ bool FilterGraph::areInputFrameSizeEqual(const std::vector& inputs) bool FilterGraph::areFrameBuffersEmpty() { - if(!_inputFramesBuffer.size()) + if(!_inputAudioFramesBuffer.size()) return true; - for(std::vector::iterator it = _inputFramesBuffer.begin(); it != _inputFramesBuffer.end(); ++it) + for(std::vector::iterator it = _inputAudioFramesBuffer.begin(); it != _inputAudioFramesBuffer.end(); ++it) { if(!it->isEmpty()) return false; @@ -218,8 +225,8 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) if(!_isInit) init(inputs, output); - // Check whether we can bypass the input buffers - const bool bypassBuffers = areInputFrameSizeEqual(inputs) && areFrameBuffersEmpty(); + // Check whether we can bypass the input audio buffers + const bool bypassBuffers = !areInputAudioFrames(inputs) || (areInputFrameSizesEqual(inputs) && areFrameBuffersEmpty()); size_t minInputFrameSize = 0; if(!bypassBuffers) @@ -229,10 +236,10 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) { if(!inputs.at(index)->getDataSize()) { - LOG_DEBUG("Empty frame from filter graph input " << index << ". Remaining frames in buffer: " << _inputFramesBuffer.at(index).getBufferSize()); + LOG_DEBUG("Empty frame from filter graph input " << index << ". Remaining audio frames in buffer: " << _inputAudioFramesBuffer.at(index).getBufferSize()); continue; } - _inputFramesBuffer.at(index).addFrame(inputs.at(index)); + _inputAudioFramesBuffer.at(index).addFrame(inputs.at(index)); } // Get the minimum input frames size @@ -244,7 +251,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) for(size_t index = 0; index < inputs.size(); ++index) { // Retrieve frame from buffer or directly from input - IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputFramesBuffer.at(index).getFrame(minInputFrameSize); + IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputAudioFramesBuffer.at(index).getFrame(minInputFrameSize); const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); if(ret < 0) @@ -356,7 +363,7 @@ void FilterGraph::addInBuffer(const std::vector& inputs) const AudioFrameDesc audioFrameDesc(audioFrame->getSampleRate(), audioFrame->getNbChannels(), getSampleFormatName(audioFrame->getSampleFormat())); - _inputFramesBuffer.push_back(FrameBuffer(audioFrameDesc)); + _inputAudioFramesBuffer.push_back(AudioFramebuffer(audioFrameDesc)); } // video frame else if((*it)->isVideoFrame()) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 9f05156f..49181ac2 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -16,16 +16,16 @@ namespace avtranscoder { /** - * @brief Filter graph input frame buffer. - * This FIFO buffer contains IFrame pointers and can deliver specific size frames. - * - * @todo Only for audio frame, for the moment. Make it usable with video frames. + * @brief Filter graph input audio frame buffer. + * This FIFO buffer contains IFrame pointers and can deliver specific size audio frames. + * It makes no sense to use such buffers for video, since video frames are spatially consistent, + * so can not be divided nor concatenated. **/ -class FrameBuffer +class AudioFramebuffer { public: - FrameBuffer(const AudioFrameDesc& audioFrameDesc); - ~FrameBuffer(); + AudioFramebuffer(const AudioFrameDesc& audioFrameDesc); + ~AudioFramebuffer(); /** * @brief Return whether the buffer is empty or not. @@ -143,7 +143,8 @@ class AvExport FilterGraph */ size_t getMinInputFrameSize(const std::vector& inputs); - bool areInputFrameSizeEqual(const std::vector& inputs); + bool areInputAudioFrames(const std::vector& inputs); + bool areInputFrameSizesEqual(const std::vector& inputs); bool areFrameBuffersEmpty(); private: @@ -151,7 +152,7 @@ class AvExport FilterGraph std::vector _filters; ///< List of filters to process. const ICodec& _codec; ///< Codec of the stream on which the filters will be applied. - std::vector _inputFramesBuffer; + std::vector _inputAudioFramesBuffer; /** * @brief Is the FilterGraph initialized. From 969c6c3b4378f728d1c1084cede29da0e0877fc4 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Thu, 7 Sep 2017 15:48:31 +0200 Subject: [PATCH 11/18] FilterGraph: remove useless areInputAudioFrames() method Replace it by a simple AudioFrameBuffer vector empty() test, since buffers are initialized only for audio frames (into the addInBuffer() method) And rename _inputAudioFramesBuffer attribute to _inputAudioFrameBuffers --- src/AvTranscoder/filter/FilterGraph.cpp | 33 ++++++++++--------------- src/AvTranscoder/filter/FilterGraph.hpp | 3 +-- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index be06fab0..1ee9ee84 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -133,7 +133,7 @@ FilterGraph::FilterGraph(const ICodec& codec) FilterGraph::~FilterGraph() { - _inputAudioFramesBuffer.clear(); + _inputAudioFrameBuffers.clear(); for(std::vector::iterator it = _filters.begin(); it < _filters.end(); ++it) { delete(*it); @@ -145,7 +145,7 @@ size_t FilterGraph::getAvailableFrameSize(const std::vector& inputs, co { size_t frameSize = inputs.at(index)->getDataSize(); if(frameSize == 0) - frameSize = _inputAudioFramesBuffer.at(index).getDataSize(); + frameSize = _inputAudioFrameBuffers.at(index).getDataSize(); return frameSize; } @@ -166,10 +166,10 @@ size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) bool FilterGraph::hasBufferedFrames() { - if(!_inputAudioFramesBuffer.size()) + if(!_inputAudioFrameBuffers.size()) return false; - for(std::vector::iterator it = _inputAudioFramesBuffer.begin(); it != _inputAudioFramesBuffer.end(); ++it) + for(std::vector::iterator it = _inputAudioFrameBuffers.begin(); it != _inputAudioFrameBuffers.end(); ++it) { if(it->isEmpty()) return false; @@ -179,17 +179,10 @@ bool FilterGraph::hasBufferedFrames() bool FilterGraph::hasBufferedFrames(const size_t index) { - if(index >= _inputAudioFramesBuffer.size()) + if(index >= _inputAudioFrameBuffers.size()) return false; - return !_inputAudioFramesBuffer.at(index).isEmpty(); -} - -bool FilterGraph::areInputAudioFrames(const std::vector& inputs) -{ - if(!inputs.size()) - return false; - return typeid(*(inputs.at(0))) == typeid(AudioFrame); + return !_inputAudioFrameBuffers.at(index).isEmpty(); } bool FilterGraph::areInputFrameSizesEqual(const std::vector& inputs) @@ -208,10 +201,10 @@ bool FilterGraph::areInputFrameSizesEqual(const std::vector& inputs) bool FilterGraph::areFrameBuffersEmpty() { - if(!_inputAudioFramesBuffer.size()) + if(!_inputAudioFrameBuffers.size()) return true; - for(std::vector::iterator it = _inputAudioFramesBuffer.begin(); it != _inputAudioFramesBuffer.end(); ++it) + for(std::vector::iterator it = _inputAudioFrameBuffers.begin(); it != _inputAudioFrameBuffers.end(); ++it) { if(!it->isEmpty()) return false; @@ -226,7 +219,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) init(inputs, output); // Check whether we can bypass the input audio buffers - const bool bypassBuffers = !areInputAudioFrames(inputs) || (areInputFrameSizesEqual(inputs) && areFrameBuffersEmpty()); + const bool bypassBuffers = _inputAudioFrameBuffers.empty() || (areInputFrameSizesEqual(inputs) && areFrameBuffersEmpty()); size_t minInputFrameSize = 0; if(!bypassBuffers) @@ -236,10 +229,10 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) { if(!inputs.at(index)->getDataSize()) { - LOG_DEBUG("Empty frame from filter graph input " << index << ". Remaining audio frames in buffer: " << _inputAudioFramesBuffer.at(index).getBufferSize()); + LOG_DEBUG("Empty frame from filter graph input " << index << ". Remaining audio frames in buffer: " << _inputAudioFrameBuffers.at(index).getBufferSize()); continue; } - _inputAudioFramesBuffer.at(index).addFrame(inputs.at(index)); + _inputAudioFrameBuffers.at(index).addFrame(inputs.at(index)); } // Get the minimum input frames size @@ -251,7 +244,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) for(size_t index = 0; index < inputs.size(); ++index) { // Retrieve frame from buffer or directly from input - IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputAudioFramesBuffer.at(index).getFrame(minInputFrameSize); + IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputAudioFrameBuffers.at(index).getFrame(minInputFrameSize); const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); if(ret < 0) @@ -363,7 +356,7 @@ void FilterGraph::addInBuffer(const std::vector& inputs) const AudioFrameDesc audioFrameDesc(audioFrame->getSampleRate(), audioFrame->getNbChannels(), getSampleFormatName(audioFrame->getSampleFormat())); - _inputAudioFramesBuffer.push_back(AudioFramebuffer(audioFrameDesc)); + _inputAudioFrameBuffers.push_back(AudioFramebuffer(audioFrameDesc)); } // video frame else if((*it)->isVideoFrame()) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 49181ac2..3ce3549f 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -143,7 +143,6 @@ class AvExport FilterGraph */ size_t getMinInputFrameSize(const std::vector& inputs); - bool areInputAudioFrames(const std::vector& inputs); bool areInputFrameSizesEqual(const std::vector& inputs); bool areFrameBuffersEmpty(); @@ -152,7 +151,7 @@ class AvExport FilterGraph std::vector _filters; ///< List of filters to process. const ICodec& _codec; ///< Codec of the stream on which the filters will be applied. - std::vector _inputAudioFramesBuffer; + std::vector _inputAudioFrameBuffers; /** * @brief Is the FilterGraph initialized. From c823ad6788347a41168735d66c1cf8483b988898 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Fri, 8 Sep 2017 10:43:19 +0200 Subject: [PATCH 12/18] FilterGraph: fix AudioFrameBuffer class name case typo --- src/AvTranscoder/filter/FilterGraph.cpp | 18 +++++++++--------- src/AvTranscoder/filter/FilterGraph.hpp | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 1ee9ee84..7bf70306 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -18,11 +18,11 @@ namespace avtranscoder /****************** - AudioFramebuffer + AudioFrameBuffer ******************/ -AudioFramebuffer::AudioFramebuffer(const AudioFrameDesc& audioFrameDesc) +AudioFrameBuffer::AudioFrameBuffer(const AudioFrameDesc& audioFrameDesc) : _audioFrameDesc(audioFrameDesc) , _frameQueue() , _totalDataSize(0) @@ -30,13 +30,13 @@ AudioFramebuffer::AudioFramebuffer(const AudioFrameDesc& audioFrameDesc) { } -AudioFramebuffer::~AudioFramebuffer() +AudioFrameBuffer::~AudioFrameBuffer() { for(size_t i = 0; i < _frameQueue.size(); ++i) popFrame(); } -void AudioFramebuffer::addFrame(IFrame* frame) +void AudioFrameBuffer::addFrame(IFrame* frame) { LOG_DEBUG("Add a new " << frame->getDataSize() << " bytes frame to frame buffer. New buffer size: " << _frameQueue.size() + 1); // Copy the input frame to store it into the queue @@ -50,13 +50,13 @@ void AudioFramebuffer::addFrame(IFrame* frame) _frameQueue.push(newAudioFrame); } -void AudioFramebuffer::popFrame() +void AudioFrameBuffer::popFrame() { _frameQueue.pop(); LOG_DEBUG("Pop frame from buffer. Remaining frames in buffer: " << _frameQueue.size()); } -IFrame* AudioFramebuffer::getFrame(const size_t size) +IFrame* AudioFrameBuffer::getFrame(const size_t size) { LOG_DEBUG("Get a " << size << " bytes frame from a " << _totalDataSize << " bytes frame buffer"); IFrame* next = _frameQueue.front(); @@ -169,7 +169,7 @@ bool FilterGraph::hasBufferedFrames() if(!_inputAudioFrameBuffers.size()) return false; - for(std::vector::iterator it = _inputAudioFrameBuffers.begin(); it != _inputAudioFrameBuffers.end(); ++it) + for(std::vector::iterator it = _inputAudioFrameBuffers.begin(); it != _inputAudioFrameBuffers.end(); ++it) { if(it->isEmpty()) return false; @@ -204,7 +204,7 @@ bool FilterGraph::areFrameBuffersEmpty() if(!_inputAudioFrameBuffers.size()) return true; - for(std::vector::iterator it = _inputAudioFrameBuffers.begin(); it != _inputAudioFrameBuffers.end(); ++it) + for(std::vector::iterator it = _inputAudioFrameBuffers.begin(); it != _inputAudioFrameBuffers.end(); ++it) { if(!it->isEmpty()) return false; @@ -356,7 +356,7 @@ void FilterGraph::addInBuffer(const std::vector& inputs) const AudioFrameDesc audioFrameDesc(audioFrame->getSampleRate(), audioFrame->getNbChannels(), getSampleFormatName(audioFrame->getSampleFormat())); - _inputAudioFrameBuffers.push_back(AudioFramebuffer(audioFrameDesc)); + _inputAudioFrameBuffers.push_back(AudioFrameBuffer(audioFrameDesc)); } // video frame else if((*it)->isVideoFrame()) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 3ce3549f..6503300d 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -21,11 +21,11 @@ namespace avtranscoder * It makes no sense to use such buffers for video, since video frames are spatially consistent, * so can not be divided nor concatenated. **/ -class AudioFramebuffer +class AudioFrameBuffer { public: - AudioFramebuffer(const AudioFrameDesc& audioFrameDesc); - ~AudioFramebuffer(); + AudioFrameBuffer(const AudioFrameDesc& audioFrameDesc); + ~AudioFrameBuffer(); /** * @brief Return whether the buffer is empty or not. @@ -151,7 +151,7 @@ class AvExport FilterGraph std::vector _filters; ///< List of filters to process. const ICodec& _codec; ///< Codec of the stream on which the filters will be applied. - std::vector _inputAudioFrameBuffers; + std::vector _inputAudioFrameBuffers; /** * @brief Is the FilterGraph initialized. From d2a7f545b70d448c42a9f7c9c8e07bd6e362e4dc Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Fri, 8 Sep 2017 12:13:08 +0200 Subject: [PATCH 13/18] FilterGraph: fix AudioFrameBuffers order --- src/AvTranscoder/filter/FilterGraph.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 7bf70306..2d97e422 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -356,7 +356,7 @@ void FilterGraph::addInBuffer(const std::vector& inputs) const AudioFrameDesc audioFrameDesc(audioFrame->getSampleRate(), audioFrame->getNbChannels(), getSampleFormatName(audioFrame->getSampleFormat())); - _inputAudioFrameBuffers.push_back(AudioFrameBuffer(audioFrameDesc)); + _inputAudioFrameBuffers.insert(_inputAudioFrameBuffers.begin(), AudioFrameBuffer(audioFrameDesc)); } // video frame else if((*it)->isVideoFrame()) From 2dcdc1c0428476001d74c97d33caac64d59d2fa4 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Fri, 8 Sep 2017 14:07:44 +0200 Subject: [PATCH 14/18] FilterGraph: fix audio frames management with different sample formats Comparing the frame sizes in samples (instead of bytes) --- src/AvTranscoder/filter/FilterGraph.cpp | 50 ++++++++++++++++++++----- src/AvTranscoder/filter/FilterGraph.hpp | 10 ++++- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/src/AvTranscoder/filter/FilterGraph.cpp b/src/AvTranscoder/filter/FilterGraph.cpp index 2d97e422..8b960ef9 100644 --- a/src/AvTranscoder/filter/FilterGraph.cpp +++ b/src/AvTranscoder/filter/FilterGraph.cpp @@ -36,6 +36,11 @@ AudioFrameBuffer::~AudioFrameBuffer() popFrame(); } +size_t AudioFrameBuffer::getBytesPerSample() +{ + return av_get_bytes_per_sample(_audioFrameDesc._sampleFormat); +} + void AudioFrameBuffer::addFrame(IFrame* frame) { LOG_DEBUG("Add a new " << frame->getDataSize() << " bytes frame to frame buffer. New buffer size: " << _frameQueue.size() + 1); @@ -113,6 +118,12 @@ IFrame* AudioFrameBuffer::getFrame(const size_t size) return newAudioFrame; } +IFrame* AudioFrameBuffer::getFrameSampleNb(const size_t sampleNb) +{ + const size_t expectedSize = sampleNb * getBytesPerSample(); + return getFrame(expectedSize); +} + /****************** @@ -149,19 +160,29 @@ size_t FilterGraph::getAvailableFrameSize(const std::vector& inputs, co return frameSize; } -size_t FilterGraph::getMinInputFrameSize(const std::vector& inputs) +size_t FilterGraph::getAvailableFrameSamplesNb(const std::vector& inputs, const size_t& index) +{ + if(_inputAudioFrameBuffers.empty()) + throw std::runtime_error("Cannot compute filter graph input samples number for non-audio frames."); + + const size_t bytesPerSample = _inputAudioFrameBuffers.at(index).getBytesPerSample(); + const size_t availableSamplesNb = getAvailableFrameSize(inputs, index) / bytesPerSample; + return availableSamplesNb; +} + +size_t FilterGraph::getMinInputFrameSamplesNb(const std::vector& inputs) { if(!inputs.size()) return 0; - size_t minFrameSize = getAvailableFrameSize(inputs, 0); + size_t minFrameSamplesNb = getAvailableFrameSamplesNb(inputs, 0); for(size_t index = 1; index < inputs.size(); ++index) { - const size_t availableFrameSize = getAvailableFrameSize(inputs, index); - if(minFrameSize > availableFrameSize) - minFrameSize = availableFrameSize; + const size_t availableFrameSampleNb = getAvailableFrameSamplesNb(inputs, index); + if(minFrameSamplesNb > availableFrameSampleNb) + minFrameSamplesNb = availableFrameSampleNb; } - return minFrameSize; + return minFrameSamplesNb; } bool FilterGraph::hasBufferedFrames() @@ -194,7 +215,16 @@ bool FilterGraph::areInputFrameSizesEqual(const std::vector& inputs) for(size_t index = 1; index < inputs.size(); ++index) { if(frameSize != inputs.at(index)->getDataSize()) - return false; + { + if(_inputAudioFrameBuffers.empty()) + return false; + else + { + const size_t refSampleNb = frameSize / _inputAudioFrameBuffers.at(0).getBytesPerSample(); + const size_t sampleNb = inputs.at(index)->getDataSize() / _inputAudioFrameBuffers.at(index).getBytesPerSample(); + return (refSampleNb == sampleNb); + } + } } return true; } @@ -220,7 +250,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) // Check whether we can bypass the input audio buffers const bool bypassBuffers = _inputAudioFrameBuffers.empty() || (areInputFrameSizesEqual(inputs) && areFrameBuffersEmpty()); - size_t minInputFrameSize = 0; + size_t minInputFrameSamplesNb = 0; if(!bypassBuffers) { @@ -236,7 +266,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) } // Get the minimum input frames size - minInputFrameSize = getMinInputFrameSize(inputs); + minInputFrameSamplesNb = getMinInputFrameSamplesNb(inputs); } @@ -244,7 +274,7 @@ void FilterGraph::process(const std::vector& inputs, IFrame& output) for(size_t index = 0; index < inputs.size(); ++index) { // Retrieve frame from buffer or directly from input - IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputAudioFrameBuffers.at(index).getFrame(minInputFrameSize); + IFrame* inputFrame = (bypassBuffers)? inputs.at(index) : _inputAudioFrameBuffers.at(index).getFrameSampleNb(minInputFrameSamplesNb); const int ret = av_buffersrc_add_frame_flags(_filters.at(index)->getAVFilterContext(), &inputFrame->getAVFrame(), AV_BUFFERSRC_FLAG_PUSH); if(ret < 0) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 6503300d..5d23d98c 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -39,6 +39,10 @@ class AudioFrameBuffer * @brief Return the number of frames contained in the buffer. */ size_t getBufferSize() const { return _frameQueue.size(); } + /** + * @brief Return the number of bytes by sample from the internal AudioFrameDesc. + */ + size_t getBytesPerSample(); /** * @brief Push a frame at the end of the buffer. @@ -50,6 +54,7 @@ class AudioFrameBuffer * If no size is specified, the whole first IFrame pointer is returned. */ IFrame* getFrame(const size_t size = 0); + IFrame* getFrameSampleNb(const size_t sampleNb); private: void popFrame(); @@ -138,10 +143,11 @@ class AvExport FilterGraph * @brief Return the input frame size if not null, or the available size into the corresponding frame buffer */ size_t getAvailableFrameSize(const std::vector& inputs, const size_t& index); + size_t getAvailableFrameSamplesNb(const std::vector& inputs, const size_t& index); /** - * @brief Get the minimum size between input frames, or available frame buffers + * @brief Get the minimum samples number between input frames, or available frame buffers */ - size_t getMinInputFrameSize(const std::vector& inputs); + size_t getMinInputFrameSamplesNb(const std::vector& inputs); bool areInputFrameSizesEqual(const std::vector& inputs); bool areFrameBuffersEmpty(); From 343ef4e829daa8c27cf82bbe400fa3a9737a31e6 Mon Sep 17 00:00:00 2001 From: Valentin Noel Date: Fri, 8 Sep 2017 16:43:29 +0200 Subject: [PATCH 15/18] Test: add a new python test for audio channels muxing Audio channels are extracted from different file format sources, to stereo and 5.1 output audio streams. --- test/pyTest/testMuxAudioChannels.py | 101 ++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 test/pyTest/testMuxAudioChannels.py diff --git a/test/pyTest/testMuxAudioChannels.py b/test/pyTest/testMuxAudioChannels.py new file mode 100644 index 00000000..ab46a4e7 --- /dev/null +++ b/test/pyTest/testMuxAudioChannels.py @@ -0,0 +1,101 @@ +import os + + # Check if environment is setup to run the tests +if os.environ.get('AVTRANSCODER_TEST_AUDIO_WAVE_FILE') is None or \ + os.environ.get('AVTRANSCODER_TEST_AUDIO_MOV_FILE') is None: + from nose.plugins.skip import SkipTest + raise SkipTest("Need to define environment variables " + "AVTRANSCODER_TEST_AUDIO_WAVE_FILE and " + "AVTRANSCODER_TEST_AUDIO_MOV_FILE") + +from nose.tools import * + +from pyAvTranscoder import avtranscoder as av + +def testMuxAudioChannelsFromDifferentFormatInputs_20(): + """ + Mux audio channels from different formats files, and generate one audio stereo stream + """ + # inputs + inputFileName1 = os.environ['AVTRANSCODER_TEST_AUDIO_MOV_FILE'] + inputFileName2 = os.environ['AVTRANSCODER_TEST_AUDIO_WAVE_FILE'] + assert_not_equals(inputFileName1, inputFileName2) + + inputs = av.InputStreamDescVector() + inputs.append(av.InputStreamDesc(inputFileName1, 1, 1)) + inputs.append(av.InputStreamDesc(inputFileName2, 0, 2)) + + # output + outputFileName = "testMuxAudioChannelsFromDifferentFormatInputs_20.wav" + ouputFile = av.OutputFile(outputFileName) + + transcoder = av.Transcoder(ouputFile) + transcoder.addStream(inputs, "wave24b48kstereo") + + progress = av.ConsoleProgress() + processStat = transcoder.process( progress ) + + # check process stat returned + audioStat = processStat.getAudioStat(0) + + inputFile1 = av.InputFile(inputFileName1) + inputFile2 = av.InputFile(inputFileName2) + + src_audioStream1 = inputFile1.getProperties().getAudioProperties()[0] + src_audioStream2 = inputFile2.getProperties().getAudioProperties()[0] + + min_src_duration = min(src_audioStream1.getDuration(), src_audioStream2.getDuration()) + + assert_equals(min_src_duration, audioStat.getDuration()) + + # check dst audio streams + dst_inputFile = av.InputFile(outputFileName) + dst_audioProperties = dst_inputFile.getProperties().getAudioProperties() + assert_equals(1, len(dst_audioProperties)) + assert_equals(2, dst_audioProperties[0].getNbChannels()) + +def testMuxAudioChannelsFromDifferentFormatInputs_51(): + """ + Mux audio channels from different formats files, and generate one audio stereo stream + """ + # inputs + inputFileName1 = os.environ['AVTRANSCODER_TEST_AUDIO_MOV_FILE'] + inputFileName2 = os.environ['AVTRANSCODER_TEST_AUDIO_WAVE_FILE'] + assert_not_equals(inputFileName1, inputFileName2) + + inputs = av.InputStreamDescVector() + inputs.append(av.InputStreamDesc(inputFileName1, 1, 1)) + inputs.append(av.InputStreamDesc(inputFileName1, 1, 0)) + inputs.append(av.InputStreamDesc(inputFileName2, 0, 2)) + inputs.append(av.InputStreamDesc(inputFileName2, 0, 5)) + inputs.append(av.InputStreamDesc(inputFileName2, 0, 1)) + inputs.append(av.InputStreamDesc(inputFileName2, 0, 3)) + + # output + outputFileName = "testMuxAudioChannelsFromDifferentFormatInputs_51.wav" + ouputFile = av.OutputFile(outputFileName) + + transcoder = av.Transcoder(ouputFile) + transcoder.addStream(inputs, "wave24b48k5_1") + + progress = av.ConsoleProgress() + processStat = transcoder.process( progress ) + + # check process stat returned + audioStat = processStat.getAudioStat(0) + + inputFile1 = av.InputFile(inputFileName1) + inputFile2 = av.InputFile(inputFileName2) + + src_audioStream1 = inputFile1.getProperties().getAudioProperties()[0] + src_audioStream2 = inputFile2.getProperties().getAudioProperties()[0] + + min_src_duration = min(src_audioStream1.getDuration(), src_audioStream2.getDuration()) + + assert_equals(min_src_duration, audioStat.getDuration()) + + # check dst audio streams + dst_inputFile = av.InputFile(outputFileName) + dst_audioProperties = dst_inputFile.getProperties().getAudioProperties() + assert_equals(1, len(dst_audioProperties)) + assert_equals(6, dst_audioProperties[0].getNbChannels()) From a26bae9bef46299ab823cbc805ddf89bf6cdd366 Mon Sep 17 00:00:00 2001 From: Valentin NOEL Date: Mon, 11 Sep 2017 15:29:38 +0200 Subject: [PATCH 16/18] FilterGraph: fixes Python binding compilation on Windows Export AudioFrameBuffer class symbol for DLL --- src/AvTranscoder/filter/FilterGraph.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AvTranscoder/filter/FilterGraph.hpp b/src/AvTranscoder/filter/FilterGraph.hpp index 5d23d98c..524a8357 100644 --- a/src/AvTranscoder/filter/FilterGraph.hpp +++ b/src/AvTranscoder/filter/FilterGraph.hpp @@ -21,7 +21,7 @@ namespace avtranscoder * It makes no sense to use such buffers for video, since video frames are spatially consistent, * so can not be divided nor concatenated. **/ -class AudioFrameBuffer +class AvExport AudioFrameBuffer { public: AudioFrameBuffer(const AudioFrameDesc& audioFrameDesc); From 9abcb90e563daad37b84fd02ed98468f2557b05a Mon Sep 17 00:00:00 2001 From: Valentin Noel Date: Tue, 12 Sep 2017 16:31:14 +0200 Subject: [PATCH 17/18] Test: check output files duration directly through file properties Into audio channels muxing tests --- test/pyTest/testMuxAudioChannels.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/pyTest/testMuxAudioChannels.py b/test/pyTest/testMuxAudioChannels.py index ab46a4e7..01ab6720 100644 --- a/test/pyTest/testMuxAudioChannels.py +++ b/test/pyTest/testMuxAudioChannels.py @@ -48,9 +48,13 @@ def testMuxAudioChannelsFromDifferentFormatInputs_20(): assert_equals(min_src_duration, audioStat.getDuration()) - # check dst audio streams + # check dst file properties dst_inputFile = av.InputFile(outputFileName) - dst_audioProperties = dst_inputFile.getProperties().getAudioProperties() + dst_fileProperties = dst_inputFile.getProperties() + assert_equals(min_src_duration, dst_fileProperties.getDuration()) + + # check dst audio streams + dst_audioProperties = dst_fileProperties.getAudioProperties() assert_equals(1, len(dst_audioProperties)) assert_equals(2, dst_audioProperties[0].getNbChannels()) @@ -94,8 +98,12 @@ def testMuxAudioChannelsFromDifferentFormatInputs_51(): assert_equals(min_src_duration, audioStat.getDuration()) - # check dst audio streams + # check dst file properties dst_inputFile = av.InputFile(outputFileName) + dst_fileProperties = dst_inputFile.getProperties() + assert_equals(min_src_duration, dst_fileProperties.getDuration()) + + # check dst audio streams dst_audioProperties = dst_inputFile.getProperties().getAudioProperties() assert_equals(1, len(dst_audioProperties)) assert_equals(6, dst_audioProperties[0].getNbChannels()) From 0e4975152e3a022e75152a94528c31648dd32c7f Mon Sep 17 00:00:00 2001 From: Valentin Noel Date: Tue, 12 Sep 2017 16:36:31 +0200 Subject: [PATCH 18/18] StreamTranscoder: avoid checking every filter graph buffer if the process won't be continued --- src/AvTranscoder/transcoder/StreamTranscoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AvTranscoder/transcoder/StreamTranscoder.cpp b/src/AvTranscoder/transcoder/StreamTranscoder.cpp index 8ad966c4..6c79e4d5 100644 --- a/src/AvTranscoder/transcoder/StreamTranscoder.cpp +++ b/src/AvTranscoder/transcoder/StreamTranscoder.cpp @@ -579,7 +579,7 @@ bool StreamTranscoder::processTranscode() if(!_filterGraph->hasFilters() || !_filterGraph->hasBufferedFrames(index)) { continueProcess = false; - continue; + break; } LOG_DEBUG("Some frames remain into filter graph buffer " << index); pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy