Applied the same context-driver treatment to the audio subsystem (still needs some extra work to ensure all audio plays fine and a good cleanup)

Fixed README.md to properly reflect all dependencies in the command examples
Added support for audio looping

Signed-off-by: Alexis Maiquez <almamu@almamu.com>

.github/workflows/cmake.yml

@@ -21,7 +21,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install dependencies
-      run: sudo apt-get update && sudo apt-get -y install libxrandr-dev libfreeimage-dev libxinerama-dev libxcursor-dev libxi-dev libgl-dev libglew-dev freeglut3-dev libsdl1.2-dev libsdl-mixer1.2-dev liblz4-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libxxf86vm-dev libglm-dev libglfw3-dev libmpv-dev mpv libmpv1
+      run: sudo apt-get update && sudo apt-get -y install libxrandr-dev libfreeimage-dev libxinerama-dev libxcursor-dev libxi-dev libgl-dev libglew-dev freeglut3-dev libsdl2-dev liblz4-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libxxf86vm-dev libglm-dev libglfw3-dev libmpv-dev mpv libmpv1
 
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.

CMakeLists.txt

@@ -7,7 +7,7 @@ set(OpenGL_GL_PREFERENCE "LEGACY")
 
 # if you're developing you might find this debug option useful for shader output, although RenderDoc is encouraged
 add_compile_definitions(DEBUG=1)
-add_compile_definitions(ERRORONLY=0)
+add_compile_definitions(ERRORONLY=1)
 
 find_package(X11 REQUIRED)
 find_package(Xrandr REQUIRED)
@@ -15,7 +15,7 @@ find_package(OpenGL REQUIRED)
 find_package(GLEW REQUIRED)
 find_package(GLUT REQUIRED)
 find_package(ZLIB REQUIRED)
-find_package(SDL REQUIRED)
+find_package(SDL2 REQUIRED)
 find_package(MPV REQUIRED)
 find_package(LZ4 REQUIRED)
 find_package(FFMPEG REQUIRED)
@@ -27,8 +27,7 @@ include_directories(
     ${XRANDR_INCLUDE_DIR}
     ${GLEW_INCLUDE_DIR}
     ${LZ4_INCLUDE_DIR}
-    ${SDL_INCLUDE_DIRS}
+    ${SDL2_INCLUDE_DIRS}
-    ${SDL_MIXER_INCLUDE_DIRS}
     ${FFMPEG_INCLUDE_DIR}
     ${FREEIMAGE_INCLUDE_DIR}
     src
@@ -65,6 +64,12 @@ add_executable(
     src/WallpaperEngine/Core/Core.h
     src/WallpaperEngine/Core/Core.cpp
 
+    src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp
+    src/WallpaperEngine/Audio/Drivers/CAudioDriver.h
+    src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp
+    src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h
+    src/WallpaperEngine/Audio/CAudioContext.cpp
+    src/WallpaperEngine/Audio/CAudioContext.h
     src/WallpaperEngine/Audio/CAudioStream.cpp
     src/WallpaperEngine/Audio/CAudioStream.h
 
@@ -227,8 +232,7 @@ target_link_libraries(linux-wallpaperengine
     ${GLUT_LIBRARIES}
     ${ZLIB_LIBRARIES}
     ${LZ4_LIBRARY}
-    ${SDL_LIBRARY}
+    ${SDL2_LIBRARIES}
-    ${SDL_MIXER_LIBRARIES}
     ${FFMPEG_LIBRARIES}
     ${FREEIMAGE_LIBRARIES}
     ${MPV_LIBRARY}

README.md

@@ -37,16 +37,8 @@ Wallpaper Engine is a software designed by [Kristjan Skutta](https://store.steam
 
 ## Commands
 ```
-sudo apt update
-```
-```
 sudo apt-get update
-```
+sudo apt-get install build-essential cmake libxrandr-dev libfreeimage-dev libxinerama-dev libxcursor-dev libxi-dev libgl-dev libglew-dev freeglut3-dev libsdl2-dev liblz4-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libxxf86vm-dev libglm-dev libglfw3-dev libmpv-dev mpv libmpv1
-```
-sudo apt -y install cmake lz4 zlib1g ffmpeg libxxf86vm-dev libglm-dev freeglut3-dev libxrandr-dev libavcodec-dev libavformat-dev libavfilter-dev libsdl1.2-dev libsdl-mixer1.2-dev
-```
-```
-sudo apt-get -y install libsdl2-2.0 libsdl-image1.2-dev libsdl1.2-dev libglfw3 libglfw3-dev libmpv-dev mpv libmpv1
 ```
 
 # 5. How to use

main.cpp

@@ -2,7 +2,6 @@
 #include <GL/glew.h>
 #include <GLFW/glfw3.h>
 #include <SDL.h>
-#include <SDL_mixer.h>
 #include <csignal>
 #include <filesystem>
 #include <getopt.h>
@@ -23,8 +22,10 @@
 #include "WallpaperEngine/Assets/CCombinedContainer.h"
 #include "WallpaperEngine/Assets/CPackageLoadException.h"
 
-#include "WallpaperEngine/Render/Drivers/COpenGLDriver.h"
 #include "Steam/FileSystem/FileSystem.h"
+#include "WallpaperEngine/Audio/CAudioContext.h"
+#include "WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h"
+#include "WallpaperEngine/Render/Drivers/COpenGLDriver.h"
 #include "common.h"
 
 #define WORKSHOP_APP_ID 431960
@@ -34,7 +35,7 @@ float g_Time;
 float g_TimeLast;
 bool g_KeepRunning = true;
 bool g_AudioEnabled = true;
-int g_AudioVolume = 15;
+int g_AudioVolume = 128;
 
 void print_help (const char* route)
 {
@@ -445,12 +446,10 @@ int main (int argc, char* argv[])
     std::signal(SIGINT, signalhandler);
     std::signal(SIGTERM, signalhandler);
 
-    if (g_AudioEnabled == true && SDL_Init (SDL_INIT_AUDIO) < 0)
+    // initialize sdl audio driver
-    {
+    WallpaperEngine::Audio::Drivers::CSDLAudioDriver audioDriver;
-        sLog.error ("Cannot initialize SDL audio system, SDL_GetError: ", SDL_GetError());
+    // initialize audio context
-        sLog.error ("Continuing without audio support");
+    WallpaperEngine::Audio::CAudioContext audioContext (&audioDriver);
-    }
-
     // initialize OpenGL driver
     WallpaperEngine::Render::Drivers::COpenGLDriver videoDriver ("Wallpaper Engine");
     // initialize render context
@@ -459,7 +458,7 @@ int main (int argc, char* argv[])
     context.setMouse (new CMouseInput (videoDriver.getWindow ()));
     // ensure the context knows what wallpaper to render
     context.setWallpaper (
-        WallpaperEngine::Render::CWallpaper::fromWallpaper (project->getWallpaper (), &context)
+        WallpaperEngine::Render::CWallpaper::fromWallpaper (project->getWallpaper (), &context, &audioContext)
     );
 
     float startTime, endTime, minimumTime = 1.0f / maximumFPS;
@@ -494,8 +493,6 @@ int main (int argc, char* argv[])
 
     sLog.out ("Stop requested");
 
-    // terminate SDL
-    SDL_QuitSubSystem(SDL_INIT_AUDIO);
     SDL_Quit ();
 
     return 0;

src/WallpaperEngine/Audio/CAudioContext.cpp

@@ -0,0 +1,31 @@
+#include "CAudioContext.h"
+#include "WallpaperEngine/Audio/Drivers/CAudioDriver.h"
+
+using namespace WallpaperEngine::Audio;
+using namespace WallpaperEngine::Audio::Drivers;
+
+CAudioContext::CAudioContext (CAudioDriver* driver) :
+    m_driver (driver)
+{
+
+}
+
+void CAudioContext::addStream (CAudioStream* stream)
+{
+    this->m_driver->addStream (stream);
+}
+
+AVSampleFormat CAudioContext::getFormat () const
+{
+    return this->m_driver->getFormat ();
+}
+
+int CAudioContext::getSampleRate () const
+{
+    return this->m_driver->getSampleRate ();
+}
+
+int CAudioContext::getChannels () const
+{
+    return this->m_driver->getChannels ();
+}

src/WallpaperEngine/Audio/CAudioContext.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <libavutil/samplefmt.h>
+#include <vector>
+
+namespace WallpaperEngine::Audio::Drivers
+{
+    class CAudioDriver;
+}
+
+namespace WallpaperEngine::Audio
+{
+    class CAudioStream;
+
+    class CAudioContext
+    {
+    public:
+        CAudioContext (Drivers::CAudioDriver* driver);
+
+        void addStream (CAudioStream* stream);
+
+        AVSampleFormat getFormat () const;
+        int getSampleRate () const;
+        int getChannels () const;
+    private:
+        Drivers::CAudioDriver* m_driver;
+    };
+}

src/WallpaperEngine/Audio/CAudioStream.cpp

@@ -4,74 +4,52 @@
 #include <iostream>
 #include <math.h>
 
+// maximum size of the queue to prevent reading too much data
+#define MAX_QUEUE_SIZE (5 * 1024 * 1024)
+#define MIN_FRAMES 25
+
 extern int g_AudioVolume;
 extern bool g_KeepRunning;
 
 using namespace WallpaperEngine::Audio;
 
-// callback for sdl to play our audio
-void audio_callback (void* userdata, uint8_t* stream, int length)
-{
-    auto audioStream = static_cast <CAudioStream*> (userdata);
-    int len1, audio_size;
-
-    static uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE * 3) / 2];
-    static unsigned int audio_buf_size = 0;
-    static unsigned int audio_buf_index = 0;
-
-    while (length > 0 && g_KeepRunning)
-    {
-        if (audio_buf_index >= audio_buf_size)
-        {
-            /* We have already sent all our data; get more */
-            audio_size = audioStream->decodeFrame (audio_buf, sizeof (audio_buf));
-
-            if (audio_size < 0)
-            {
-                /* If error, output silence */
-                audio_buf_size = 1024; // arbitrary?
-                memset(audio_buf, 0, audio_buf_size);
-            }
-            else
-            {
-                audio_buf_size = audio_size;
-            }
-
-            audio_buf_index = 0;
-        }
-
-        len1 = audio_buf_size - audio_buf_index;
-
-        if (len1 > length)
-            len1 = length;
-
-        // mix the audio so the volume is right
-        SDL_MixAudio (stream, (uint8_t*) audio_buf + audio_buf_index, len1, g_AudioVolume);
-
-        length -= len1;
-        stream += len1;
-        audio_buf_index += len1;
-
-        if (audioStream->isInitialized () == false)
-            break;
-    }
-}
-
 int audio_read_thread (void* arg)
 {
+    SDL_mutex* waitMutex = SDL_CreateMutex ();
     CAudioStream* stream = static_cast <CAudioStream*> (arg);
     AVPacket* packet = av_packet_alloc ();
     int ret = 0;
 
+    if (waitMutex == nullptr)
+        sLog.exception ("Cannot create mutex for audio playback waiting");
+
     while (ret >= 0 && g_KeepRunning == true)
     {
+        // give the cpu some time to play the queued frames if there's enough info there
+        if (
+            stream->getQueueSize () >= MAX_QUEUE_SIZE ||
+            (stream->getQueuePacketCount () > MIN_FRAMES &&
+             (av_q2d (stream->getTimeBase ()) * stream->getQueueDuration () > 1.0))
+            )
+        {
+            SDL_LockMutex (waitMutex);
+            SDL_CondWaitTimeout (stream->getWaitCondition (), waitMutex, 10);
+            SDL_UnlockMutex (waitMutex);
+            continue;
+        }
+
         ret = av_read_frame (stream->getFormatContext (), packet);
 
         if (ret == AVERROR_EOF)
         {
             // seek to the beginning of the file again
-            av_seek_frame (stream->getFormatContext (), stream->getAudioStream (), 0, AVSEEK_FLAG_FRAME);
+            avformat_seek_file (stream->getFormatContext (), stream->getAudioStream (), 0, 0, 0, ~AVSEEK_FLAG_FRAME);
             avcodec_flush_buffers (stream->getContext ());
+
+            // ensure the thread is not killed if audio has to be looped
+            if (stream->isRepeat() == true)
+                ret = 0;
+
             continue;
         }
 
@@ -79,7 +57,7 @@ int audio_read_thread (void* arg)
         if (packet->stream_index == stream->getAudioStream ())
             stream->queuePacket (packet);
         else
-            av_packet_unref (packet);
+            av_packet_free (&packet);
 
         if (stream->isInitialized () == false)
             break;
@@ -87,6 +65,7 @@ int audio_read_thread (void* arg)
 
     // stop the audio too just in case
     stream->stop ();
+    SDL_DestroyMutex (waitMutex);
 
     return 0;
 }
@@ -126,21 +105,15 @@ int64_t audio_seek_data_callback (void* streamarg, int64_t offset, int whence)
     return offset;
 }
 
-CAudioStream::CAudioStream (const std::string& filename)
+CAudioStream::CAudioStream (CAudioContext* context, const std::string& filename) :
+    m_audioContext (context)
 {
-    // do not do anything if sdl audio was not initialized
-    if (SDL_WasInit (SDL_INIT_AUDIO) != SDL_INIT_AUDIO)
-        return;
-
     this->loadCustomContent (filename.c_str ());
 }
 
-CAudioStream::CAudioStream (const void* buffer, int length)
+CAudioStream::CAudioStream (CAudioContext* context, const void* buffer, int length) :
+    m_audioContext (context)
 {
-    // do not do anything if sdl audio was not initialized
-    if (SDL_WasInit (SDL_INIT_AUDIO) != SDL_INIT_AUDIO)
-        return;
-
     // setup a custom context first
     this->m_formatContext = avformat_alloc_context ();
 
@@ -169,14 +142,11 @@ CAudioStream::CAudioStream (const void* buffer, int length)
     this->loadCustomContent ();
 }
 
-CAudioStream::CAudioStream(AVCodecContext* context)
+CAudioStream::CAudioStream(CAudioContext* audioContext, AVCodecContext* context) :
-    : m_context (context),
+    m_context (context),
-    m_queue (new PacketQueue)
+    m_queue (new PacketQueue),
+    m_audioContext (audioContext)
 {
-    // do not do anything if sdl audio was not initialized
-    if (SDL_WasInit (SDL_INIT_AUDIO) != SDL_INIT_AUDIO)
-        return;
-
     this->initialize ();
 }
 
@@ -222,38 +192,18 @@ void CAudioStream::loadCustomContent (const char* filename)
     this->initialize ();
 
     // initialize an SDL thread to read the file
-    SDL_CreateThread (audio_read_thread, this);
+    SDL_CreateThread (audio_read_thread, filename, this);
 }
 
 void CAudioStream::initialize ()
 {
     // allocate the FIFO buffer
-    this->m_queue->packetList = av_fifo_alloc (sizeof (MyAVPacketList));
+    this->m_queue->packetList = av_fifo_alloc2 (1, sizeof (MyAVPacketList), AV_FIFO_FLAG_AUTO_GROW);
 
     // setup the queue information
     this->m_queue->mutex = SDL_CreateMutex ();
     this->m_queue->cond = SDL_CreateCond ();
-
+    this->m_queue->wait = SDL_CreateCond ();
-    // take control of the audio device
-
-    SDL_AudioSpec requestedSpec, finalSpec;
-
-    // Set audio settings from codec info
-    requestedSpec.freq = this->m_context->sample_rate;
-    requestedSpec.format = AUDIO_S16SYS;
-    requestedSpec.channels = this->m_context->channels;
-    requestedSpec.silence = 0;
-    requestedSpec.samples = SDL_AUDIO_BUFFER_SIZE;
-    requestedSpec.callback = audio_callback;
-    requestedSpec.userdata = this;
-
-    if (SDL_OpenAudio (&requestedSpec, &finalSpec) < 0)
-    {
-        sLog.error ("SDL_OpenAudio: ", SDL_GetError ());
-        return;
-    }
-
-    SDL_PauseAudio (0);
 
     this->m_initialized = true;
 }
@@ -281,21 +231,11 @@ void CAudioStream::queuePacket(AVPacket *pkt)
 
 bool CAudioStream::doQueue (AVPacket* pkt)
 {
-    MyAVPacketList entry;
+    MyAVPacketList entry { pkt };
 
-    // ensure the FIFO has enough space to hold the new entry
+    // write the entry if possible
-    if (av_fifo_space (this->m_queue->packetList) < sizeof (entry))
+    if (av_fifo_write (this->m_queue->packetList, &entry, 1) < 0)
-    {
+        return false;
-        if (av_fifo_grow (this->m_queue->packetList, sizeof (entry)) < 0)
-        {
-            return false;
-        }
-    }
-
-    entry.packet = pkt;
-
-    // write to the FIFO
-    av_fifo_generic_write (this->m_queue->packetList, &entry, sizeof (entry), nullptr);
 
     this->m_queue->nb_packets ++;
     this->m_queue->size += entry.packet->size + sizeof (entry);
@@ -315,12 +255,11 @@ void CAudioStream::dequeuePacket (AVPacket* output)
     while (g_KeepRunning)
     {
         // enough data available, read it
-        if (av_fifo_size (this->m_queue->packetList) >= sizeof (entry))
+        if (av_fifo_read (this->m_queue->packetList, &entry, 1) >= 0)
         {
-            av_fifo_generic_read (this->m_queue->packetList, &entry, sizeof (entry), nullptr);
-
             this->m_queue->nb_packets --;
-            this->m_queue->size -= entry.packet->duration;
+            this->m_queue->size -= entry.packet->size + sizeof (entry);
+            this->m_queue->duration -= entry.packet->duration;
 
             // move the reference and free the old one
             av_packet_move_ref (output, entry.packet);
@@ -329,7 +268,7 @@ void CAudioStream::dequeuePacket (AVPacket* output)
         }
 
         // make the thread wait if nothing was available
-        SDL_CondWaitTimeout (this->m_queue->cond, this->m_queue->mutex, 1000);
+        SDL_CondWait (this->m_queue->cond, this->m_queue->mutex);
     }
 
     SDL_UnlockMutex (this->m_queue->mutex);
@@ -385,14 +324,46 @@ void CAudioStream::setPosition (int current)
     this->m_position = current;
 }
 
+SDL_cond* CAudioStream::getWaitCondition ()
+{
+    return this->m_queue->wait;
+}
+
+int CAudioStream::getQueueSize ()
+{
+    return this->m_queue->size;
+}
+
+int CAudioStream::getQueuePacketCount ()
+{
+    return this->m_queue->nb_packets;
+}
+
+AVRational CAudioStream::getTimeBase ()
+{
+    return this->m_formatContext->streams [this->m_audioStream]->time_base;
+}
+
+int64_t CAudioStream::getQueueDuration ()
+{
+    return this->m_queue->duration;
+}
+
+bool CAudioStream::isQueueEmpty ()
+{
+    return this->m_queue->nb_packets == 0;
+}
+
+SDL_mutex* CAudioStream::getMutex ()
+{
+    return this->m_queue->mutex;
+}
+
 void CAudioStream::stop ()
 {
     if (this->isInitialized () == false)
         return;
 
-    // pause audio
-    SDL_PauseAudio (1);
-
     // stop the threads running
     this->m_initialized = false;
 }
@@ -421,7 +392,7 @@ int CAudioStream::resampleAudio (
 
     if (!swr_ctx)
     {
-        printf("swr_alloc error.\n");
+        sLog.error("swr_alloc error.\n");
         return -1;
     }
 
@@ -434,7 +405,7 @@ int CAudioStream::resampleAudio (
     // check input audio channels correctly retrieved
     if (in_channel_layout <= 0)
     {
-        printf("in_channel_layout error.\n");
+        sLog.error("in_channel_layout error.\n");
         return -1;
     }
 
@@ -456,7 +427,7 @@ int CAudioStream::resampleAudio (
     in_nb_samples = decoded_audio_frame->nb_samples;
     if (in_nb_samples <= 0)
     {
-        printf("in_nb_samples error.\n");
+        sLog.error("in_nb_samples error.");
         return -1;
     }
 
@@ -513,7 +484,7 @@ int CAudioStream::resampleAudio (
     ret = swr_init(swr_ctx);;
     if (ret < 0)
     {
-        printf("Failed to initialize the resampling context.\n");
+        sLog.error("Failed to initialize the resampling context.");
         return -1;
     }
 
@@ -527,7 +498,7 @@ int CAudioStream::resampleAudio (
     // check rescaling was successful
     if (max_out_nb_samples <= 0)
     {
-        printf("av_rescale_rnd error.\n");
+        sLog.error("av_rescale_rnd error.");
         return -1;
     }
 
@@ -545,7 +516,7 @@ int CAudioStream::resampleAudio (
 
     if (ret < 0)
     {
-        printf("av_samples_alloc_array_and_samples() error: Could not allocate destination samples.\n");
+        sLog.error("av_samples_alloc_array_and_samples() error: Could not allocate destination samples.");
         return -1;
     }
 
@@ -560,7 +531,7 @@ int CAudioStream::resampleAudio (
     // check output samples number was correctly retrieved
     if (out_nb_samples <= 0)
     {
-        printf("av_rescale_rnd error\n");
+        sLog.error("av_rescale_rnd error");
         return -1;
     }
 
@@ -582,50 +553,42 @@ int CAudioStream::resampleAudio (
         // check samples buffer correctly allocated
         if (ret < 0)
         {
-            printf("av_samples_alloc failed.\n");
+            sLog.error("av_samples_alloc failed.");
             return -1;
         }
 
         max_out_nb_samples = out_nb_samples;
     }
 
-    if (swr_ctx)
+    // do the actual audio data resampling
+    ret = swr_convert(
+        swr_ctx,
+        resampled_data,
+        out_nb_samples,
+        (const uint8_t **) decoded_audio_frame->data,
+        decoded_audio_frame->nb_samples
+    );
+
+    // check audio conversion was successful
+    if (ret < 0)
     {
-        // do the actual audio data resampling
+        sLog.error("swr_convert_error.");
-        ret = swr_convert(
+        return -1;
-            swr_ctx,
-            resampled_data,
-            out_nb_samples,
-            (const uint8_t **) decoded_audio_frame->data,
-            decoded_audio_frame->nb_samples
-        );
-
-        // check audio conversion was successful
-        if (ret < 0)
-        {
-            printf("swr_convert_error.\n");
-            return -1;
-        }
-
-        // Get the required buffer size for the given audio parameters
-        resampled_data_size = av_samples_get_buffer_size(
-            &out_linesize,
-            out_nb_channels,
-            ret,
-            out_sample_fmt,
-            1
-        );
-
-        // check audio buffer size
-        if (resampled_data_size < 0)
-        {
-            printf("av_samples_get_buffer_size error.\n");
-            return -1;
-        }
     }
-    else
+
+    // Get the required buffer size for the given audio parameters
+    resampled_data_size = av_samples_get_buffer_size(
+        &out_linesize,
+        out_nb_channels,
+        ret,
+        out_sample_fmt,
+        1
+    );
+
+    // check audio buffer size
+    if (resampled_data_size < 0)
     {
-        printf("swr_ctx null error.\n");
+        sLog.error ("av_samples_get_buffer_size error.");
         return -1;
     }
 
@@ -666,39 +629,28 @@ int CAudioStream::decodeFrame (uint8_t* audioBuffer, int bufferSize)
     avFrame = av_frame_alloc();
     if (!avFrame)
     {
-        printf("Could not allocate AVFrame.\n");
+        sLog.error("Could not allocate AVFrame.\n");
         return -1;
     }
 
-    for (; g_KeepRunning;) {
+    // block until there's any data in the buffers
+    while (g_KeepRunning) {
         while (audio_pkt_size > 0) {
             int got_frame = 0;
             int ret = avcodec_receive_frame(this->getContext (), avFrame);
 
             if (ret == 0)
-            {
                 got_frame = 1;
-            }
             if (ret == AVERROR(EAGAIN))
-            {
                 ret = 0;
-            }
             if (ret == 0)
-            {
                 ret = avcodec_send_packet(this->getContext (), pkt);
-            }
             if (ret == AVERROR(EAGAIN))
-            {
                 ret = 0;
-            }
             else if (ret < 0)
-            {
                 return -1;
-            }
             else
-            {
                 len1 = pkt->size;
-            }
 
             if (len1 < 0)
             {
@@ -715,18 +667,18 @@ int CAudioStream::decodeFrame (uint8_t* audioBuffer, int bufferSize)
                 // audio resampling
                 data_size = this->resampleAudio (
                     avFrame,
-                    AV_SAMPLE_FMT_S16,
+                    this->m_audioContext->getFormat (),
-                    this->getContext ()->channels,
+                    this->m_audioContext->getChannels (),
-                    this->getContext ()->sample_rate,
+                    this->m_audioContext->getSampleRate (),
                     audioBuffer
                 );
                 assert(data_size <= bufferSize);
             }
             if (data_size <= 0) {
-                /* No data yet, get more frames */
+                // no data found, keep waiting
                 continue;
             }
-            /* We have data, return it and come back for more later */
+            // some data was found
             return data_size;
         }
         if (pkt->data)

src/WallpaperEngine/Audio/CAudioStream.h

@@ -15,74 +15,79 @@ extern "C"
 #include <SDL.h>
 #include <SDL_thread.h>
 
-#define SDL_AUDIO_BUFFER_SIZE 1024
+#include "WallpaperEngine/Audio/CAudioContext.h"
-#define MAX_AUDIO_FRAME_SIZE 192000
 
-namespace WallpaperEngine
+namespace WallpaperEngine::Audio
 {
-    namespace Audio
+    class CAudioStream
     {
-        class CAudioStream
+    public:
+        CAudioStream (CAudioContext* context, const std::string& filename);
+        CAudioStream (CAudioContext* context, const void* buffer, int length);
+        CAudioStream (CAudioContext* audioContext, AVCodecContext* context);
+
+        void queuePacket (AVPacket* pkt);
+
+        /**
+         * Gets the next packet in the queue
+         *
+         * WARNING: BLOCKS UNTIL SOME DATA IS READ FROM IT
+         *
+         * @return
+         */
+        void dequeuePacket (AVPacket* output);
+
+        AVCodecContext* getContext ();
+        AVFormatContext* getFormatContext ();
+        int getAudioStream ();
+        bool isInitialized ();
+        void setRepeat (bool newRepeat = true);
+        bool isRepeat ();
+        void stop ();
+        const void* getBuffer ();
+        int getLength ();
+        int getPosition ();
+        void setPosition (int current);
+        SDL_cond* getWaitCondition ();
+        int getQueueSize ();
+        int getQueuePacketCount ();
+        int64_t getQueueDuration ();
+        AVRational getTimeBase ();
+        bool isQueueEmpty ();
+        SDL_mutex* getMutex ();
+
+        int decodeFrame (uint8_t* audioBuffer, int bufferSize);
+
+    private:
+        void loadCustomContent (const char* filename = nullptr);
+        int resampleAudio (AVFrame * decoded_audio_frame, enum AVSampleFormat out_sample_fmt, int out_channels, int out_sample_rate, uint8_t* out_buf);
+        bool doQueue (AVPacket* pkt);
+        void initialize ();
+
+        CAudioContext* m_audioContext;
+        bool m_initialized;
+        bool m_repeat;
+        AVCodecContext* m_context = nullptr;
+        AVFormatContext* m_formatContext = nullptr;
+        int m_audioStream = -1;
+        const void* m_buffer;
+        int m_length;
+        int m_position = 0;
+
+        struct MyAVPacketList
         {
-        public:
+            AVPacket *packet;
-            CAudioStream (const std::string& filename);
-            CAudioStream (const void* buffer, int length);
-            CAudioStream (AVCodecContext* context);
-
-            void queuePacket (AVPacket* pkt);
-
-            /**
-             * Gets the next packet in the queue
-             *
-             * WARNING: BLOCKS UNTIL SOME DATA IS READ FROM IT
-             *
-             * @return
-             */
-            void dequeuePacket (AVPacket* output);
-
-            AVCodecContext* getContext ();
-            AVFormatContext* getFormatContext ();
-            int getAudioStream ();
-            bool isInitialized ();
-            void setRepeat (bool newRepeat = true);
-            bool isRepeat ();
-            void stop ();
-            const void* getBuffer ();
-            int getLength ();
-            int getPosition ();
-            void setPosition (int current);
-
-            int decodeFrame (uint8_t* audioBuffer, int bufferSize);
-
-        private:
-            void loadCustomContent (const char* filename = nullptr);
-            int resampleAudio (AVFrame * decoded_audio_frame, enum AVSampleFormat out_sample_fmt, int out_channels, int out_sample_rate, uint8_t* out_buf);
-            bool doQueue (AVPacket* pkt);
-            void initialize ();
-
-            bool m_initialized;
-            bool m_repeat;
-            AVCodecContext* m_context = nullptr;
-            AVFormatContext* m_formatContext = nullptr;
-            int m_audioStream = -1;
-            const void* m_buffer;
-            int m_length;
-            int m_position = 0;
-
-            struct MyAVPacketList
-            {
-                AVPacket *packet;
-            };
-
-            struct PacketQueue
-            {
-                AVFifoBuffer* packetList;
-                int nb_packets;
-                int size;
-                int64_t duration;
-                SDL_mutex *mutex;
-                SDL_cond *cond;
-            } *m_queue;
         };
-    }
+
+        struct PacketQueue
+        {
+            AVFifo* packetList;
+            int nb_packets;
+            int size;
+            int64_t duration;
+            SDL_mutex *mutex;
+            SDL_cond *wait;
+            SDL_cond *cond;
+        } *m_queue;
+    };
 };

src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp

@@ -0,0 +1,4 @@
+#include "CAudioDriver.h"
+
+using namespace WallpaperEngine::Audio;
+using namespace WallpaperEngine::Audio::Drivers;

src/WallpaperEngine/Audio/Drivers/CAudioDriver.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <vector>
+
+#include "WallpaperEngine/Audio/CAudioStream.h"
+
+namespace WallpaperEngine::Audio
+{
+    class CAudioStream;
+}
+
+namespace WallpaperEngine::Audio::Drivers
+{
+    class CAudioDriver
+    {
+    public:
+        virtual void addStream (CAudioStream* stream) = 0;
+
+        virtual AVSampleFormat getFormat () const = 0;
+        virtual int getSampleRate () const = 0;
+        virtual int getChannels () const = 0;
+    private:
+    };
+}

src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp

@@ -0,0 +1,154 @@
+#include "common.h"
+#include "CSDLAudioDriver.h"
+
+#define SDL_AUDIO_BUFFER_SIZE 4096
+#define MAX_AUDIO_FRAME_SIZE 192000
+
+extern int g_AudioVolume;
+extern bool g_KeepRunning;
+
+using namespace WallpaperEngine::Audio;
+using namespace WallpaperEngine::Audio::Drivers;
+
+void audio_callback (void* userdata, uint8_t* streamData, int length)
+{
+    CSDLAudioDriver* driver = reinterpret_cast <CSDLAudioDriver*> (userdata);
+
+    memset (streamData, 0, length);
+
+    for (const auto& buffer : driver->getStreams ())
+    {
+        uint8_t* streamDataPointer = streamData;
+        int streamLength = length;
+        int len1, audio_size;
+
+        // sound is not initialized or stopped and is not in loop mode
+        // ignore mixing it in
+        if (buffer->stream->isInitialized () == false)
+            continue;
+
+        // check if thread is empty and signal the read thread
+        if (buffer->stream->isQueueEmpty () == true)
+        {
+            SDL_CondSignal (buffer->stream->getWaitCondition ());
+            continue;
+        }
+
+        while (streamLength > 0 && g_KeepRunning)
+        {
+            if (buffer->audio_buf_index >= buffer->audio_buf_size)
+            {
+                // get more data to fill the buffer
+                audio_size = buffer->stream->decodeFrame (buffer->audio_buf, sizeof (buffer->audio_buf));
+
+                if (audio_size < 0)
+                {
+                    // fallback for errors, silence
+                    buffer->audio_buf_size = 1024;
+                    memset(buffer->audio_buf, 0, buffer->audio_buf_size);
+                }
+                else
+                {
+                    buffer->audio_buf_size = audio_size;
+                }
+
+                buffer->audio_buf_index = 0;
+            }
+
+            len1 = buffer->audio_buf_size - buffer->audio_buf_index;
+
+            if (len1 > streamLength)
+                len1 = streamLength;
+
+            // mix the audio
+            SDL_MixAudioFormat (
+                streamDataPointer, &buffer->audio_buf [buffer->audio_buf_index],
+                driver->getSpec ().format, len1, g_AudioVolume
+            );
+
+            streamLength -= len1;
+            streamDataPointer += len1;
+            buffer->audio_buf_index += len1;
+        }
+    }
+}
+
+CSDLAudioDriver::CSDLAudioDriver () :
+    m_initialized (false)
+{
+    if (SDL_InitSubSystem (SDL_INIT_AUDIO) < 0)
+    {
+        sLog.error ("Cannot initialize SDL audio system, SDL_GetError: ", SDL_GetError ());
+        sLog.error ("Continuing without audio support");
+
+        return;
+    }
+
+    SDL_AudioSpec requestedSpec;
+
+    memset (&requestedSpec, 0, sizeof (requestedSpec));
+
+    requestedSpec.freq = 48000;
+    requestedSpec.format = AUDIO_F32;
+    requestedSpec.channels = 2;
+    requestedSpec.samples = SDL_AUDIO_BUFFER_SIZE;
+    requestedSpec.callback = audio_callback;
+    requestedSpec.userdata = this;
+
+    this->m_deviceID = SDL_OpenAudioDevice (nullptr, false, &requestedSpec, &this->m_audioSpec, SDL_AUDIO_ALLOW_ANY_CHANGE);
+
+    if (this->m_deviceID == 0)
+    {
+        sLog.error ("SDL_OpenAudioDevice: ", SDL_GetError ());
+        return;
+    }
+
+    SDL_PauseAudioDevice (this->m_deviceID, 0);
+
+    this->m_initialized = true;
+}
+
+CSDLAudioDriver::~CSDLAudioDriver ()
+{
+    if (this->m_initialized == false)
+        return;
+
+    SDL_CloseAudioDevice (this->m_deviceID);
+    SDL_QuitSubSystem (SDL_INIT_AUDIO);
+}
+
+void CSDLAudioDriver::addStream (CAudioStream* stream)
+{
+    this->m_streams.push_back (new CSDLAudioBuffer { stream });
+}
+const std::vector <CSDLAudioBuffer*>& CSDLAudioDriver::getStreams ()
+{
+    return this->m_streams;
+}
+
+AVSampleFormat CSDLAudioDriver::getFormat () const
+{
+    switch (this->m_audioSpec.format)
+    {
+        case AUDIO_U8: case AUDIO_S8: return AV_SAMPLE_FMT_U8;
+        case AUDIO_U16MSB: case AUDIO_U16LSB: case AUDIO_S16LSB: case AUDIO_S16MSB: return AV_SAMPLE_FMT_S16;
+        case AUDIO_S32LSB: case AUDIO_S32MSB: return AV_SAMPLE_FMT_S32;
+        case AUDIO_F32LSB: case AUDIO_F32MSB: return AV_SAMPLE_FMT_FLT;
+    }
+
+    sLog.exception ("Cannot convert from SDL format to ffmpeg format, aborting...");
+}
+
+int CSDLAudioDriver::getSampleRate () const
+{
+    return this->m_audioSpec.freq;
+}
+
+int CSDLAudioDriver::getChannels () const
+{
+    return this->m_audioSpec.channels;
+}
+const SDL_AudioSpec& CSDLAudioDriver::getSpec () const
+{
+    return this->m_audioSpec;
+}

src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <vector>
+#include <map>
+
+#include "WallpaperEngine/Audio/CAudioStream.h"
+#include "WallpaperEngine/Audio/Drivers/CAudioDriver.h"
+
+#include <SDL.h>
+
+#define MAX_AUDIO_FRAME_SIZE 192000
+
+namespace WallpaperEngine::Audio::Drivers
+{
+    struct CSDLAudioBuffer
+    {
+        CAudioStream* stream;
+        uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE * 3) / 2];
+        unsigned int audio_buf_size = 0;
+        unsigned int audio_buf_index = 0;
+    };
+
+    class CSDLAudioDriver : public CAudioDriver
+    {
+    public:
+        CSDLAudioDriver ();
+        ~CSDLAudioDriver ();
+
+        void addStream (CAudioStream* stream) override;
+        const std::vector <CSDLAudioBuffer*>& getStreams ();
+
+        AVSampleFormat getFormat () const override;
+        int getSampleRate () const override;
+        int getChannels () const override;
+        const SDL_AudioSpec& getSpec () const;
+    private:
+        SDL_AudioDeviceID m_deviceID;
+        bool m_initialized;
+        SDL_AudioSpec m_audioSpec;
+        std::vector <CSDLAudioBuffer*> m_streams;
+    };
+}

src/WallpaperEngine/Core/Objects/CSound.cpp

@@ -5,14 +5,17 @@
 using namespace WallpaperEngine::Core::Objects;
 
 CSound::CSound (
-        CScene* scene,
+    CScene* scene,
-        CUserSettingBoolean* visible,
+    CUserSettingBoolean* visible,
-        uint32_t id,
+    uint32_t id,
-        std::string name,
+    std::string name,
-        CUserSettingVector3* origin,
+    CUserSettingVector3* origin,
-        CUserSettingVector3* scale,
+    CUserSettingVector3* scale,
-        const glm::vec3& angles) :
+    const glm::vec3& angles,
-        CObject (scene, visible, id, std::move(name), Type, origin, scale, angles)
+    bool repeat
+) :
+    CObject (scene, visible, id, std::move(name), Type, origin, scale, angles),
+    m_repeat (repeat)
 {
 }
 
@@ -26,8 +29,13 @@ WallpaperEngine::Core::CObject* CSound::fromJSON (
         CUserSettingVector3* scale,
         const glm::vec3& angles)
 {
+    bool repeat = false;
     // TODO: PARSE AUDIO VOLUME
     auto sound_it = jsonFindRequired (data, "sound", "Sound information not present");
+    auto playbackmode = jsonFindDefault <std::string> (data, "playbackmode", "");
+
+    if (playbackmode == "loop")
+        repeat = true;
 
     if ((*sound_it).is_array () == false)
         sLog.exception ("Expected sound list on element ", name);
@@ -39,7 +47,8 @@ WallpaperEngine::Core::CObject* CSound::fromJSON (
         name,
         origin,
         scale,
-        angles
+        angles,
+        repeat
     );
 
     for (const auto& cur : (*sound_it))
@@ -57,5 +66,9 @@ const std::vector<std::string>& CSound::getSounds () const
 {
     return this->m_sounds;
 }
+bool CSound::isRepeat () const
+{
+    return this->m_repeat;
+}
 
 const std::string CSound::Type = "sound";

src/WallpaperEngine/Core/Objects/CSound.h

@@ -27,6 +27,7 @@ namespace WallpaperEngine::Core::Objects
 
         void insertSound (std::string filename);
         const std::vector<std::string>& getSounds () const;
+        bool isRepeat () const;
 
     protected:
         CSound (
@@ -36,11 +37,13 @@ namespace WallpaperEngine::Core::Objects
             std::string name,
             CUserSettingVector3* origin,
             CUserSettingVector3* scale,
-            const glm::vec3& angles
+            const glm::vec3& angles,
+            bool repeat
         );
 
         static const std::string Type;
     private:
+        bool m_repeat;
         std::vector<std::string> m_sounds;
     };
 }

src/WallpaperEngine/Render/CScene.cpp

@@ -14,8 +14,8 @@ extern float g_TimeLast;
 using namespace WallpaperEngine;
 using namespace WallpaperEngine::Render;
 
-CScene::CScene (Core::CScene* scene, CRenderContext* context) :
+CScene::CScene (Core::CScene* scene, CRenderContext* context, CAudioContext* audioContext) :
-    CWallpaper (scene, Type, context)
+    CWallpaper (scene, Type, context, audioContext)
 {
     // setup the scene camera
     this->m_camera = new CCamera (this, scene->getCamera ());

src/WallpaperEngine/Render/CScene.h

@@ -15,7 +15,7 @@ namespace WallpaperEngine::Render
     class CScene : public CWallpaper
     {
     public:
-        CScene (Core::CScene* scene, CRenderContext* context);
+        CScene (Core::CScene* scene, CRenderContext* context, CAudioContext* audioContext);
 
         CCamera* getCamera () const;
 

src/WallpaperEngine/Render/CVideo.cpp

@@ -14,8 +14,8 @@ void* get_proc_address (void* ctx, const char* name)
     return reinterpret_cast <void*> (glfwGetProcAddress (name));
 }
 
-CVideo::CVideo (Core::CVideo* video, CRenderContext* context) :
+CVideo::CVideo (Core::CVideo* video, CRenderContext* context, CAudioContext* audioContext) :
-    CWallpaper (video, Type, context),
+    CWallpaper (video, Type, context, audioContext),
     m_width (16),
     m_height (16)
 {

src/WallpaperEngine/Render/CVideo.h

@@ -12,7 +12,7 @@ namespace WallpaperEngine::Render
     class CVideo : public CWallpaper
     {
     public:
-        CVideo (Core::CVideo* video, CRenderContext* context);
+        CVideo (Core::CVideo* video, CRenderContext* context, CAudioContext* audioContext);
 
         Core::CVideo* getVideo ();
 

src/WallpaperEngine/Render/CWallpaper.cpp

@@ -9,11 +9,12 @@
 
 using namespace WallpaperEngine::Render;
 
-CWallpaper::CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context) :
+CWallpaper::CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context, CAudioContext* audioContext) :
     m_wallpaperData (wallpaperData),
     m_type (std::move(type)),
     m_context (context),
-    m_destFramebuffer (GL_NONE)
+    m_destFramebuffer (GL_NONE),
+    m_audioContext (audioContext)
 {
     // generate the VAO to stop opengl from complaining
     glGenVertexArrays (1, &this->m_vaoBuffer);
@@ -329,6 +330,11 @@ CRenderContext* CWallpaper::getContext ()
     return this->m_context;
 }
 
+CAudioContext* CWallpaper::getAudioContext ()
+{
+    return this->m_audioContext;
+}
+
 CFBO* CWallpaper::createFBO (const std::string& name, ITexture::TextureFormat format, ITexture::TextureFlags flags, float scale, uint32_t realWidth, uint32_t realHeight, uint32_t textureWidth, uint32_t textureHeight)
 {
     CFBO* fbo = new CFBO (name, format, flags, scale, realWidth, realHeight, textureWidth, textureHeight);
@@ -359,12 +365,12 @@ CFBO* CWallpaper::getFBO () const
     return this->m_sceneFBO;
 }
 
-CWallpaper* CWallpaper::fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context)
+CWallpaper* CWallpaper::fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context, CAudioContext* audioContext)
 {
     if (wallpaper->is <Core::CScene> () == true)
-        return new WallpaperEngine::Render::CScene (wallpaper->as <Core::CScene> (), context);
+        return new WallpaperEngine::Render::CScene (wallpaper->as <Core::CScene> (), context, audioContext);
     else if (wallpaper->is <Core::CVideo> () == true)
-        return new WallpaperEngine::Render::CVideo (wallpaper->as <Core::CVideo> (), context);
+        return new WallpaperEngine::Render::CVideo (wallpaper->as <Core::CVideo> (), context, audioContext);
     else
         sLog.exception ("Unsupported wallpaper type");
 }

src/WallpaperEngine/Render/CWallpaper.h

@@ -10,8 +10,10 @@
 #include "CFBO.h"
 #include "CRenderContext.h"
 #include "WallpaperEngine/Assets/CContainer.h"
+#include "WallpaperEngine/Audio/CAudioContext.h"
 
 using namespace WallpaperEngine::Assets;
+using namespace WallpaperEngine::Audio;
 
 namespace WallpaperEngine::Render
 {
@@ -25,7 +27,7 @@ namespace WallpaperEngine::Render
 
         template<class T> bool is () { return this->m_type == T::Type; }
 
-        CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context);
+        CWallpaper (Core::CWallpaper* wallpaperData, std::string type, CRenderContext* context, CAudioContext* audioContext);
         ~CWallpaper ();
 
         /**
@@ -43,6 +45,11 @@ namespace WallpaperEngine::Render
          */
         CRenderContext* getContext ();
 
+        /**
+         * @return The current audio context for this wallpaper
+         */
+        CAudioContext* getAudioContext ();
+
         /**
          * @return The scene's framebuffer
          */
@@ -105,7 +112,7 @@ namespace WallpaperEngine::Render
          *
          * @return
          */
-        static CWallpaper* fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context);
+        static CWallpaper* fromWallpaper (Core::CWallpaper* wallpaper, CRenderContext* context, CAudioContext* audioContext);
 
     protected:
         /**
@@ -160,5 +167,9 @@ namespace WallpaperEngine::Render
          * Context that is using this wallpaper
          */
         CRenderContext* m_context;
+        /*
+         * Audio context that is using this wallpaper
+         */
+        CAudioContext* m_audioContext;
     };
 }

src/WallpaperEngine/Render/Objects/CSound.cpp

@@ -18,8 +18,15 @@ void CSound::load ()
         uint32_t filesize = 0;
         const void* filebuffer = this->getContainer ()->readFile (cur, &filesize);
 
-        this->m_audioStreams.push_back (new Audio::CAudioStream (filebuffer, filesize));
+        auto stream = new Audio::CAudioStream (this->getScene ()->getAudioContext (), filebuffer, filesize);
+
+        stream->setRepeat (this->m_sound->isRepeat ());
+
+        this->m_audioStreams.push_back (stream);
         this->m_soundBuffer.push_back (filebuffer);
+
+        // add the stream to the context so it can be played
+        this->getScene ()->getAudioContext ()->addStream (stream);
     }
 }