mirrors/linux-wallpaperengine
1
0
Fork 0
mirror of https://github.com/Almamu/linux-wallpaperengine.git synced 2025-07-13 21:02:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added basic support for audio processing so backgrounds can react to sound playing in the system

Browse Source 
Signed-off-by: Alexis Maiquez <almamu@almamu.com>
This commit is contained in:
Alexis Maiquez 2023-03-25 11:54:29 +01:00
parent a39bfa1f01
commit a89d8ebb22
21 changed files with 630 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CMakeLists.txt
View File

@ -11,6 +11,8 @@ if(NOT ERRORONLY)
set(ERRORONLY 0)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing")
# if you're developing you might find this debug option useful for shader output, although RenderDoc is encouraged
add_compile_definitions(ERRORONLY=${ERRORONLY})
add_compile_definitions(DATADIR="${DATADIR}")
@ -45,6 +47,9 @@ add_executable(
linux-wallpaperengine
main.cpp
src/External/Android/fft.cpp
src/External/Android/fft.h
src/Steam/FileSystem/FileSystem.h
src/Steam/FileSystem/FileSystem.cpp
@ -77,6 +82,11 @@ add_executable(
src/WallpaperEngine/Core/Core.h
src/WallpaperEngine/Core/Core.cpp
src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.cpp
src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.h
src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.cpp
src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.h
src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.cpp
src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.h
src/WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.cpp
@ -288,7 +298,5 @@ if(HAVE_XSETIOERROREXITHANDLER)
endif()
# set some install parameters if not in debug mode
if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
install(TARGETS linux-wallpaperengine)
install(DIRECTORY share/ DESTINATION share/${PROJECT_NAME})
endif()
install(TARGETS linux-wallpaperengine)
install(DIRECTORY share/ DESTINATION share/${PROJECT_NAME})

187
src/External/Android/fft.cpp vendored Normal file
View File

@ -0,0 +1,187 @@
#include "fft.h"
namespace External::Android
{
/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* A Fixed point implementation of Fast Fourier Transform (FFT). Complex numbers
* are represented by 32-bit integers, where higher 16 bits are real part and
* lower ones are imaginary part. Few compromises are made between efficiency,
* accuracy, and maintainability. To make it fast, arithmetic shifts are used
* instead of divisions, and bitwise inverses are used instead of negates. To
* keep it small, only radix-2 Cooley-Tukey algorithm is implemented, and only
* half of the twiddle factors are stored. Although there are still ways to make
* it even faster or smaller, it costs too much on one of the aspects.
*/
#include <stdio.h>
#include <stdint.h>
#ifdef __arm__
#include <machine/cpu-features.h>
#endif
#define LOG_FFT_SIZE 10
#define MAX_FFT_SIZE (1 << LOG_FFT_SIZE)
static const int32_t twiddle[MAX_FFT_SIZE / 4] = {
0x00008000, 0xff378001, 0xfe6e8002, 0xfda58006, 0xfcdc800a, 0xfc13800f,
0xfb4a8016, 0xfa81801e, 0xf9b88027, 0xf8ef8032, 0xf827803e, 0xf75e804b,
0xf6958059, 0xf5cd8068, 0xf5058079, 0xf43c808b, 0xf374809e, 0xf2ac80b2,
0xf1e480c8, 0xf11c80de, 0xf05580f6, 0xef8d8110, 0xeec6812a, 0xedff8146,
0xed388163, 0xec718181, 0xebab81a0, 0xeae481c1, 0xea1e81e2, 0xe9588205,
0xe892822a, 0xe7cd824f, 0xe7078276, 0xe642829d, 0xe57d82c6, 0xe4b982f1,
0xe3f4831c, 0xe3308349, 0xe26d8377, 0xe1a983a6, 0xe0e683d6, 0xe0238407,
0xdf61843a, 0xde9e846e, 0xdddc84a3, 0xdd1b84d9, 0xdc598511, 0xdb998549,
0xdad88583, 0xda1885be, 0xd95885fa, 0xd8988637, 0xd7d98676, 0xd71b86b6,
0xd65c86f6, 0xd59e8738, 0xd4e1877b, 0xd42487c0, 0xd3678805, 0xd2ab884c,
0xd1ef8894, 0xd13488dd, 0xd0798927, 0xcfbe8972, 0xcf0489be, 0xce4b8a0c,
0xcd928a5a, 0xccd98aaa, 0xcc218afb, 0xcb698b4d, 0xcab28ba0, 0xc9fc8bf5,
0xc9468c4a, 0xc8908ca1, 0xc7db8cf8, 0xc7278d51, 0xc6738dab, 0xc5c08e06,
0xc50d8e62, 0xc45b8ebf, 0xc3a98f1d, 0xc2f88f7d, 0xc2488fdd, 0xc198903e,
0xc0e990a1, 0xc03a9105, 0xbf8c9169, 0xbedf91cf, 0xbe329236, 0xbd86929e,
0xbcda9307, 0xbc2f9371, 0xbb8593dc, 0xbadc9448, 0xba3394b5, 0xb98b9523,
0xb8e39592, 0xb83c9603, 0xb7969674, 0xb6f196e6, 0xb64c9759, 0xb5a897ce,
0xb5059843, 0xb46298b9, 0xb3c09930, 0xb31f99a9, 0xb27f9a22, 0xb1df9a9c,
0xb1409b17, 0xb0a29b94, 0xb0059c11, 0xaf689c8f, 0xaecc9d0e, 0xae319d8e,
0xad979e0f, 0xacfd9e91, 0xac659f14, 0xabcd9f98, 0xab36a01c, 0xaaa0a0a2,
0xaa0aa129, 0xa976a1b0, 0xa8e2a238, 0xa84fa2c2, 0xa7bda34c, 0xa72ca3d7,
0xa69ca463, 0xa60ca4f0, 0xa57ea57e, 0xa4f0a60c, 0xa463a69c, 0xa3d7a72c,
0xa34ca7bd, 0xa2c2a84f, 0xa238a8e2, 0xa1b0a976, 0xa129aa0a, 0xa0a2aaa0,
0xa01cab36, 0x9f98abcd, 0x9f14ac65, 0x9e91acfd, 0x9e0fad97, 0x9d8eae31,
0x9d0eaecc, 0x9c8faf68, 0x9c11b005, 0x9b94b0a2, 0x9b17b140, 0x9a9cb1df,
0x9a22b27f, 0x99a9b31f, 0x9930b3c0, 0x98b9b462, 0x9843b505, 0x97ceb5a8,
0x9759b64c, 0x96e6b6f1, 0x9674b796, 0x9603b83c, 0x9592b8e3, 0x9523b98b,
0x94b5ba33, 0x9448badc, 0x93dcbb85, 0x9371bc2f, 0x9307bcda, 0x929ebd86,
0x9236be32, 0x91cfbedf, 0x9169bf8c, 0x9105c03a, 0x90a1c0e9, 0x903ec198,
0x8fddc248, 0x8f7dc2f8, 0x8f1dc3a9, 0x8ebfc45b, 0x8e62c50d, 0x8e06c5c0,
0x8dabc673, 0x8d51c727, 0x8cf8c7db, 0x8ca1c890, 0x8c4ac946, 0x8bf5c9fc,
0x8ba0cab2, 0x8b4dcb69, 0x8afbcc21, 0x8aaaccd9, 0x8a5acd92, 0x8a0cce4b,
0x89becf04, 0x8972cfbe, 0x8927d079, 0x88ddd134, 0x8894d1ef, 0x884cd2ab,
0x8805d367, 0x87c0d424, 0x877bd4e1, 0x8738d59e, 0x86f6d65c, 0x86b6d71b,
0x8676d7d9, 0x8637d898, 0x85fad958, 0x85beda18, 0x8583dad8, 0x8549db99,
0x8511dc59, 0x84d9dd1b, 0x84a3dddc, 0x846ede9e, 0x843adf61, 0x8407e023,
0x83d6e0e6, 0x83a6e1a9, 0x8377e26d, 0x8349e330, 0x831ce3f4, 0x82f1e4b9,
0x82c6e57d, 0x829de642, 0x8276e707, 0x824fe7cd, 0x822ae892, 0x8205e958,
0x81e2ea1e, 0x81c1eae4, 0x81a0ebab, 0x8181ec71, 0x8163ed38, 0x8146edff,
0x812aeec6, 0x8110ef8d, 0x80f6f055, 0x80def11c, 0x80c8f1e4, 0x80b2f2ac,
0x809ef374, 0x808bf43c, 0x8079f505, 0x8068f5cd, 0x8059f695, 0x804bf75e,
0x803ef827, 0x8032f8ef, 0x8027f9b8, 0x801efa81, 0x8016fb4a, 0x800ffc13,
0x800afcdc, 0x8006fda5, 0x8002fe6e, 0x8001ff37,
};
/* Returns the multiplication of \conj{a} and {b}. */
static inline int32_t mult (int32_t a, int32_t b)
{
#if __ARM_ARCH__ >= 6
int32_t t = b;
__asm__("smuad %0, %0, %1" : "+r" (t) : "r" (a));
__asm__("smusdx %0, %0, %1" : "+r" (b) : "r" (a));
__asm__("pkhtb %0, %0, %1, ASR #16" : "+r" (t) : "r" (b));
return t;
#else
return (((a >> 16) * (b >> 16) + (int16_t) a * (int16_t) b) & ~0xFFFF) |
((((a >> 16) * (int16_t) b - (int16_t) a * (b >> 16)) >> 16) & 0xFFFF);
#endif
}
static inline int32_t half (int32_t a)
{
#if __ARM_ARCH__ >= 6
__asm__("shadd16 %0, %0, %1" : "+r" (a) : "r" (0));
return a;
#else
return ((a >> 1) & ~0x8000) | (a & 0x8000);
#endif
}
void fixed_fft (int n, int32_t* v)
{
int scale = LOG_FFT_SIZE, i, p, r;
for (r = 0, i = 1; i < n; ++i)
{
for (p = n; !(p & r); p >>= 1, r ^= p);
if (i < r)
{
int32_t t = v[i];
v[i] = v[r];
v[r] = t;
}
}
for (p = 1; p < n; p <<= 1)
{
--scale;
for (i = 0; i < n; i += p << 1)
{
int32_t x = half (v[i]);
int32_t y = half (v[i + p]);
v[i] = x + y;
v[i + p] = x - y;
}
for (r = 1; r < p; ++r)
{
int32_t w = MAX_FFT_SIZE / 4 - (r << scale);
i = w >> 31;
w = twiddle[(w ^ i) - i] ^ (i << 16);
for (i = r; i < n; i += p << 1)
{
int32_t x = half (v[i]);
int32_t y = mult (w, v[i + p]);
v[i] = x - y;
v[i + p] = x + y;
}
}
}
}
void fixed_fft_real (int n, int32_t* v)
{
int scale = LOG_FFT_SIZE, m = n >> 1, i;
fixed_fft (n, v);
for (i = 1; i <= n; i <<= 1, --scale);
v[0] = mult (~v[0], 0x80008000);
v[m] = half (v[m]);
for (i = 1; i < n >> 1; ++i)
{
int32_t x = half (v[i]);
int32_t z = half (v[n - i]);
int32_t y = z - (x ^ 0xFFFF);
x = half (x + (z ^ 0xFFFF));
y = mult (y, twiddle[i << scale]);
v[i] = x - y;
v[n - i] = (x + y) ^ 0xFFFF;
}
}
bool doFft (uint8_t* fft, uint8_t* waveform)
{
int32_t workspace[WAVE_BUFFER_SIZE >> 1];
int32_t nonzero = 0;
for (uint32_t i = 0; i < WAVE_BUFFER_SIZE; i += 2)
{
workspace[i >> 1] =
((waveform[i] ^ 0x80) << 24) | ((waveform[i + 1] ^ 0x80) << 8);
nonzero |= workspace[i >> 1];
}
if (nonzero)
{
fixed_fft_real (WAVE_BUFFER_SIZE >> 1, workspace);
}
for (uint32_t i = 0; i < WAVE_BUFFER_SIZE; i += 2)
{
short tmp = workspace[i >> 1] >> 21;
while (tmp > 127 || tmp < -128) tmp >>= 1;
fft[i] = tmp;
tmp = workspace[i >> 1];
tmp >>= 5;
while (tmp > 127 || tmp < -128) tmp >>= 1;
fft[i + 1] = tmp;
}
return true;
}
}

10
src/External/Android/fft.h vendored Normal file
View File

@ -0,0 +1,10 @@
#pragma once
#include <iostream>
#define WAVE_BUFFER_SIZE 1024
namespace External::Android
{
bool doFft (uint8_t* fft, uint8_t* waveform);
}

6
src/WallpaperEngine/Application/CWallpaperApplication.cpp
View File

@ -274,10 +274,12 @@ namespace WallpaperEngine::Application
WallpaperEngine::Render::Drivers::Output::COutput* output;
// fullscreen detector is common for the different render modes
WallpaperEngine::Render::Drivers::Detectors::CX11FullScreenDetector fullscreenDetector (videoDriver);
// stereo mix recorder for audio processing
WallpaperEngine::Audio::Drivers::Recorders::CPulseAudioPlaybackRecorder audioRecorder;
// audio playing detector
WallpaperEngine::Audio::Drivers::Detectors::CPulseAudioPlayingDetector audioDetector (this->m_context, fullscreenDetector);
// initialize sdl audio driver
WallpaperEngine::Audio::Drivers::CSDLAudioDriver audioDriver (this->m_context, audioDetector);
WallpaperEngine::Audio::Drivers::CSDLAudioDriver audioDriver (this->m_context, audioDetector, audioRecorder);
// initialize audio context
WallpaperEngine::Audio::CAudioContext audioContext (audioDriver);
@ -314,6 +316,8 @@ namespace WallpaperEngine::Application
while (!videoDriver.closeRequested () && this->m_context.state.general.keepRunning)
{
// update audio recorder
audioDriver.update ();
// update input information
inputContext.update ();
// keep track of the previous frame's time

5
src/WallpaperEngine/Audio/CAudioContext.cpp
View File

@ -32,4 +32,9 @@ namespace WallpaperEngine::Audio
{
return this->m_driver.getApplicationContext ();
}
Drivers::Recorders::CPlaybackRecorder& CAudioContext::getRecorder ()
{
return this->m_driver.getRecorder ();
}
}

10
src/WallpaperEngine/Audio/CAudioContext.h
View File

@ -3,6 +3,7 @@
#include <libavutil/samplefmt.h>
#include <vector>
#include "WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.h"
#include "WallpaperEngine/Application/CApplicationContext.h"
namespace WallpaperEngine
@ -17,6 +18,11 @@ namespace WallpaperEngine
namespace Drivers
{
class CAudioDriver;
namespace Recorders
{
class CPulseAudioPlaybackRecorder;
}
}
class CAudioStream;
@ -51,6 +57,10 @@ namespace WallpaperEngine
* @return The application context under which the audio driver is initialized
*/
Application::CApplicationContext& getApplicationContext ();
/**
* @return The audio recorder to use to capture stereo mix data
*/
[[nodiscard]] Drivers::Recorders::CPlaybackRecorder& getRecorder ();
private:
/** The audio driver in use */

16
src/WallpaperEngine/Audio/Drivers/CAudioDriver.cpp
View File

@ -2,12 +2,19 @@
namespace WallpaperEngine::Audio::Drivers
{
CAudioDriver::CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector) :
CAudioDriver::CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder) :
m_applicationContext (applicationContext),
m_detector (detector)
m_detector (detector),
m_recorder (recorder)
{
}
void CAudioDriver::update ()
{
this->m_recorder.update ();
this->m_detector.update ();
}
Application::CApplicationContext& CAudioDriver::getApplicationContext ()
{
return this->m_applicationContext;
@ -16,4 +23,9 @@ namespace WallpaperEngine::Audio::Drivers
{
return this->m_detector;
}
Recorders::CPlaybackRecorder& CAudioDriver::getRecorder ()
{
return this->m_recorder;
}
}

28
src/WallpaperEngine/Audio/Drivers/CAudioDriver.h
View File

@ -3,6 +3,7 @@
#include <vector>
#include "WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.h"
#include "WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.h"
#include "WallpaperEngine/Application/CApplicationContext.h"
#include "WallpaperEngine/Audio/CAudioStream.h"
@ -17,20 +18,25 @@ namespace WallpaperEngine
{
class CAudioStream;
namespace Detectors
{
class CAudioPlayingDetector;
}
namespace Drivers
{
namespace Detectors
{
class CAudioPlayingDetector;
}
namespace Recorders
{
class CPulseAudioPlaybackRecorder;
}
/**
* Base class for audio driver implementations
*/
class CAudioDriver
{
public:
explicit CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector);
explicit CAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder);
/**
* Registers the given stream in the driver for playing
@ -39,6 +45,11 @@ namespace WallpaperEngine
*/
virtual void addStream (CAudioStream* stream) = 0;
/**
* Updates status of the different audio settings
*/
virtual void update ();
/**
* TODO: MAYBE THIS SHOULD BE OUR OWN DEFINITIONS INSTEAD OF LIBRARY SPECIFIC ONES?
*
@ -61,10 +72,15 @@ namespace WallpaperEngine
* @return The audio playing detector to use to stop playing sound when something else starts playing
*/
[[nodiscard]] Detectors::CAudioPlayingDetector& getAudioDetector ();
/**
* @return The audio recorder to use to capture stereo mix data
*/
[[nodiscard]] Recorders::CPlaybackRecorder& getRecorder ();
private:
Application::CApplicationContext& m_applicationContext;
Detectors::CAudioPlayingDetector& m_detector;
Recorders::CPlaybackRecorder& m_recorder;
};
}
}

4
src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.cpp
View File

@ -74,8 +74,8 @@ void audio_callback (void* userdata, uint8_t* streamData, int length)
}
}
CSDLAudioDriver::CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector) :
CAudioDriver (applicationContext, detector),
CSDLAudioDriver::CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder) :
CAudioDriver (applicationContext, detector, recorder),
m_initialized (false),
m_audioSpec ()
{

2
src/WallpaperEngine/Audio/Drivers/CSDLAudioDriver.h
View File

@ -29,7 +29,7 @@ namespace WallpaperEngine::Audio::Drivers
class CSDLAudioDriver : public CAudioDriver
{
public:
CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector);
CSDLAudioDriver (Application::CApplicationContext& applicationContext, Detectors::CAudioPlayingDetector& detector, Recorders::CPlaybackRecorder& recorder);
~CSDLAudioDriver ();
/** @inheritdoc */

13
src/WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.cpp
View File

@ -6,10 +6,16 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
Application::CApplicationContext& appContext,
Render::Drivers::Detectors::CFullScreenDetector& fullscreenDetector) :
m_applicationContext (appContext),
m_fullscreenDetector (fullscreenDetector)
m_fullscreenDetector (fullscreenDetector),
m_isPlaying (false)
{
}
bool CAudioPlayingDetector::anythingPlaying () const
{
return this->m_isPlaying;
}
Application::CApplicationContext& CAudioPlayingDetector::getApplicationContext ()
{
return this->m_applicationContext;
@ -19,4 +25,9 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
{
return this->m_fullscreenDetector;
}
void CAudioPlayingDetector::setIsPlaying (bool newState)
{
this->m_isPlaying = newState;
}
}

19
src/WallpaperEngine/Audio/Drivers/Detectors/CAudioPlayingDetector.h
View File

@ -17,6 +17,9 @@ namespace WallpaperEngine
namespace Audio::Drivers::Detectors
{
/**
* Base class for any implementation of audio playing detection
*/
class CAudioPlayingDetector
{
public:
@ -25,7 +28,19 @@ namespace WallpaperEngine
/**
* @return If any kind of sound is currently playing on the default audio device
*/
virtual bool anythingPlaying () = 0;
[[nodiscard]] bool anythingPlaying () const;
/**
* Updates the playing status to the specified value
*
* @param newState
*/
void setIsPlaying (bool newState);
/**
* Checks if any audio is playing and updates state accordingly
*/
virtual void update () = 0;
/**
* @return The application context using this detector
*/
@ -36,6 +51,8 @@ namespace WallpaperEngine
[[nodiscard]] Render::Drivers::Detectors::CFullScreenDetector& getFullscreenDetector ();
private:
bool m_isPlaying;
Application::CApplicationContext& m_applicationContext;
Render::Drivers::Detectors::CFullScreenDetector& m_fullscreenDetector;
};

15
src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.cpp
View File

@ -36,8 +36,7 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
CAudioPlayingDetector (appContext, fullscreenDetector),
m_mainloop (nullptr),
m_mainloopApi (nullptr),
m_context (nullptr),
m_isPlaying (false)
m_context (nullptr)
{
this->m_mainloop = pa_mainloop_new ();
this->m_mainloopApi = pa_mainloop_get_api (this->m_mainloop);
@ -62,17 +61,13 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
pa_mainloop_free (this->m_mainloop);
}
void CPulseAudioPlayingDetector::setIsPlaying (bool newState)
{
this->m_isPlaying = newState;
}
bool CPulseAudioPlayingDetector::anythingPlaying ()
void CPulseAudioPlayingDetector::update ()
{
if (!this->getApplicationContext ().settings.audio.automute)
return false;
return this->setIsPlaying (false);
if (this->getFullscreenDetector ().anythingFullscreen ())
return true;
return this->setIsPlaying (true);
// reset playing state
this->setIsPlaying (false);
@ -85,7 +80,5 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
pa_mainloop_iterate (this->m_mainloop, 1, nullptr);
pa_operation_unref (op);
return this->m_isPlaying;
}
}

6
src/WallpaperEngine/Audio/Drivers/Detectors/CPulseAudioPlayingDetector.h
View File

@ -13,15 +13,11 @@ namespace WallpaperEngine::Audio::Drivers::Detectors
explicit CPulseAudioPlayingDetector (Application::CApplicationContext& appContext, Render::Drivers::Detectors::CFullScreenDetector&);
~CPulseAudioPlayingDetector ();
[[nodiscard]] bool anythingPlaying () override;
void setIsPlaying (bool newState);
void update () override;
private:
bool m_isPlaying;
pa_mainloop* m_mainloop;
pa_mainloop_api* m_mainloopApi;
pa_context* m_context;
pa_operation* m_operation;
};
}

1
src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.cpp Normal file
View File

@ -0,0 +1 @@
#include "CPlaybackRecorder.h"

14
src/WallpaperEngine/Audio/Drivers/Recorders/CPlaybackRecorder.h Normal file
View File

@ -0,0 +1,14 @@
#pragma once
namespace WallpaperEngine::Audio::Drivers::Recorders
{
class CPlaybackRecorder
{
public:
virtual void update () = 0;
float audio16[16] = {0};
float audio32[32] = {0};
float audio64[64] = {0};
};
}

216
src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.cpp Normal file
View File

@ -0,0 +1,216 @@
#include <cstring>
#include <cmath>
#include <glm/common.hpp>
#include "CPulseAudioPlaybackRecorder.h"
#include "WallpaperEngine/Logging/CLog.h"
#include "External/Android/fft.h"
namespace WallpaperEngine::Audio::Drivers::Recorders
{
float movetowards(float current, float target, float maxDelta)
{
if (abs(target - current) <= maxDelta)
return target;
return current + glm::sign(target - current) * maxDelta;
}
void pa_stream_notify_cb(pa_stream *stream, void* /*userdata*/)
{
const pa_stream_state state = pa_stream_get_state(stream);
switch (state) {
case PA_STREAM_FAILED:
sLog.error ("Cannot open stream for capture. Audio processing is disabled");
break;
case PA_STREAM_READY:
sLog.debug ("Capture stream ready");
break;
}
}
void pa_stream_read_cb(pa_stream *stream, const size_t /*nbytes*/, void* userdata)
{
auto* recorder = reinterpret_cast<CPulseAudioPlaybackRecorder*>(userdata);
// Careful when to pa_stream_peek() and pa_stream_drop()!
// c.f. https://www.freedesktop.org/software/pulseaudio/doxygen/stream_8h.html#ac2838c449cde56e169224d7fe3d00824
uint8_t *data = nullptr;
size_t currentSize;
if (pa_stream_peek(stream, (const void**)&data, &currentSize) != 0) {
sLog.error ("Failed to peek at stream data...");
return;
}
if (data == nullptr && currentSize == 0) {
// No data in the buffer, ignore.
return;
} else if (data == nullptr && currentSize > 0) {
// Hole in the buffer. We must drop it.
if (pa_stream_drop(stream) != 0) {
sLog.error ("Failed to drop a hole while capturing!");
return;
}
} else if (currentSize > 0 && data) {
size_t dataToCopy = std::min (currentSize, WAVE_BUFFER_SIZE - recorder->currentWritePointer);
memcpy (&recorder->audio_buffer_tmp [recorder->currentWritePointer], data, dataToCopy * sizeof (uint8_t));
recorder->currentWritePointer += dataToCopy;
if (recorder->currentWritePointer == WAVE_BUFFER_SIZE) {
// copy to the final buffer
memcpy (recorder->audio_buffer, recorder->audio_buffer_tmp, WAVE_BUFFER_SIZE * sizeof (uint8_t));
// reset the write pointer
recorder->currentWritePointer = 0;
recorder->fullframeReady = true;
}
// any data read left?
if (dataToCopy < currentSize) {
while ((currentSize - dataToCopy) > WAVE_BUFFER_SIZE)
dataToCopy += WAVE_BUFFER_SIZE; // there's more than one full frame available, skip it entirely
// data pending, keep it in the buffer
memcpy (recorder->audio_buffer_tmp, data + dataToCopy, (currentSize - dataToCopy) * sizeof (uint8_t));
recorder->currentWritePointer = currentSize - dataToCopy;
}
}
if (pa_stream_drop(stream) != 0) {
sLog.error ("Failed to drop data after peeking");
}
}
void pa_server_info_cb(pa_context *ctx, const pa_server_info *info, void* userdata)
{
auto* recorder = reinterpret_cast<CPulseAudioPlaybackRecorder*>(userdata);
pa_sample_spec spec;
spec.format = PA_SAMPLE_U8;
spec.rate = 44100;
spec.channels = 1;
if (recorder->getCaptureStream ())
pa_stream_unref (recorder->getCaptureStream ());
pa_stream* captureStream = pa_stream_new(ctx, "output monitor", &spec, nullptr);
pa_stream_set_state_callback(captureStream, &pa_stream_notify_cb, userdata);
pa_stream_set_read_callback(captureStream, &pa_stream_read_cb, userdata);
std::string monitor_name(info->default_sink_name);
monitor_name += ".monitor";
if (pa_stream_connect_record(captureStream, monitor_name.c_str(), nullptr, PA_STREAM_NOFLAGS) != 0) {
sLog.error ("Failed to connect to input for recording");
return;
}
recorder->setCaptureStream (captureStream);
}
void pa_context_subscribe_cb (pa_context *ctx, pa_subscription_event_type_t t, uint32_t idx, void *userdata)
{
// sink changes mean re-take the stream
pa_context_get_server_info(ctx, &pa_server_info_cb, userdata);
}
void pa_context_notify_cb(pa_context *ctx, void* userdata)
{
const pa_context_state state = pa_context_get_state(ctx);
switch (state) {
case PA_CONTEXT_READY:
{
//set callback
pa_context_set_subscribe_callback (ctx, pa_context_subscribe_cb, userdata);
//set events mask and enable event callback.
pa_operation* o = pa_context_subscribe (
ctx, static_cast<pa_subscription_mask_t>(PA_SUBSCRIPTION_MASK_SINK | PA_SUBSCRIPTION_MASK_SOURCE),
NULL, NULL
);
if (o)
pa_operation_unref (o);
// context being ready means to fetch the sink too
pa_context_get_server_info(ctx, &pa_server_info_cb, userdata);
break;
}
case PA_CONTEXT_FAILED:
sLog.error ("PulseAudio context initialization failed. Audio processing is disabled");
break;
}
}
CPulseAudioPlaybackRecorder::CPulseAudioPlaybackRecorder ()
{
this->m_mainloop = pa_mainloop_new ();
this->m_mainloopApi = pa_mainloop_get_api (this->m_mainloop);
this->m_context = pa_context_new (this->m_mainloopApi, "wallpaperengine-audioprocessing");
pa_context_set_state_callback (this->m_context, &pa_context_notify_cb, this);
if (pa_context_connect(this->m_context, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0) {
sLog.error ("PulseAudio connection failed! Audio processing is disabled");
return;
}
// wait until the context is ready
while (pa_context_get_state (this->m_context) != PA_CONTEXT_READY)
pa_mainloop_iterate (this->m_mainloop, 1, nullptr);
}
CPulseAudioPlaybackRecorder::~CPulseAudioPlaybackRecorder ()
{
pa_context_disconnect(this->m_context);
pa_mainloop_free(this->m_mainloop);
}
pa_stream* CPulseAudioPlaybackRecorder::getCaptureStream ()
{
return this->m_captureStream;
}
void CPulseAudioPlaybackRecorder::setCaptureStream (pa_stream* stream)
{
this->m_captureStream = stream;
}
void CPulseAudioPlaybackRecorder::update ()
{
pa_mainloop_iterate (this->m_mainloop, 1, nullptr);
// interpolate current values to the destination
for (int i = 0; i < 64; i ++) {
this->audio64 [i] = movetowards (this->audio64[i], fft_destination64[i], 0.1f);
if (i >= 32)
continue;
this->audio32 [i] = movetowards (this->audio32[i], fft_destination32[i], 0.1f);
if (i >= 16)
continue;
this->audio16 [i] = movetowards (this->audio16[i], fft_destination16[i], 0.1f);
}
if (this->fullframeReady == false)
return;
this->fullframeReady = false;
External::Android::doFft (audio_fft, audio_buffer);
for (int i = 0; i < 64; i ++) {
int paramInt = (i + 2) * 2;
float f1 = audio_fft[paramInt];
float f2 = audio_fft[paramInt + 1];
f2 = f1 * f1 + f2 * f2;
f1 = 0.0F;
if (f2 > 0.0F)
f1 = 0.35F * (float)log10(f2);
this->fft_destination64[i] = fmin(1.0F, f1 * (float)(2.0f - pow(M_E, (1.0F - i / 63.0F) * 1.0f - 0.5f)));
this->fft_destination32[i >> 1] = fmin(1.0F, f1 * (float)(2.0f - pow(M_E, (1.0F - i / 31.0F) * 1.0f - 0.5f)));
this->fft_destination16[i >> 2] = fmin(1.0F, f1 * (float)(2.0f - pow(M_E, (1.0F - i / 15.0F) * 1.0f - 0.5f)));
}
}
}

45
src/WallpaperEngine/Audio/Drivers/Recorders/CPulseAudioPlaybackRecorder.h Normal file
View File

@ -0,0 +1,45 @@
#pragma once
#include <pulse/pulseaudio.h>
#include "External/Android/fft.h"
#include "CPlaybackRecorder.h"
namespace WallpaperEngine::Audio::Drivers::Recorders
{
class CPlaybackRecorder;
class CPulseAudioPlaybackRecorder : public CPlaybackRecorder
{
public:
CPulseAudioPlaybackRecorder ();
~CPulseAudioPlaybackRecorder ();
void update () override;
/**
* @return The current stream we're capturing from
*/
[[nodiscard]] pa_stream* getCaptureStream ();
/**
* @param stream The new stream to be capturing off from
*/
void setCaptureStream (pa_stream* stream);
uint8_t audio_buffer [WAVE_BUFFER_SIZE] = {0x80};
uint8_t audio_buffer_tmp [WAVE_BUFFER_SIZE] = {0x80};
uint8_t audio_fft [WAVE_BUFFER_SIZE] = {0};
size_t currentWritePointer = 0;
bool fullframeReady = false;
private:
pa_mainloop* m_mainloop;
pa_mainloop_api* m_mainloopApi;
pa_context* m_context;
pa_stream* m_captureStream;
float fft_destination64[64];
float fft_destination32[32];
float fft_destination16[16];
};
}

41
src/WallpaperEngine/Render/Objects/Effects/CPass.cpp
View File

@ -180,14 +180,15 @@ void CPass::render ()
switch (entry->type)
{
case Double:
glUniform1d (entry->id, *reinterpret_cast <const double*> (entry->value));
glUniform1dv (entry->id, entry->count, reinterpret_cast <const double*> (entry->value));
break;
case Float:
glUniform1f (entry->id, *reinterpret_cast <const float*> (entry->value));
glUniform1fv (entry->id, entry->count, reinterpret_cast <const float*> (entry->value));
break;
case Integer:
glUniform1i (entry->id, *reinterpret_cast <const int*> (entry->value));
glUniform1iv (entry->id, entry->count, reinterpret_cast <const int*> (entry->value));
break;
// TODO: THESE MIGHT NEED SPECIAL TREATMENT? IDK ONLY SUPPORT 1 FOR NOW
case Vector4:
glUniform4fv (entry->id, 1, glm::value_ptr (*reinterpret_cast <const glm::vec4*> (entry->value)));
break;
@ -595,6 +596,12 @@ void CPass::setupUniforms ()
this->addUniform ("g_EffectTextureProjectionMatrixInverse", glm::mat4(1.0));
this->addUniform ("g_TexelSize", glm::vec2 (1.0 / projection->getWidth (), 1.0 / projection->getHeight ()));
this->addUniform ("g_TexelSizeHalf", glm::vec2 (0.5 / projection->getWidth (), 0.5 / projection->getHeight ()));
this->addUniform ("g_AudioSpectrum16Left", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio16, 16);
this->addUniform ("g_AudioSpectrum16Right", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio16, 16);
this->addUniform ("g_AudioSpectrum32Left", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio32, 32);
this->addUniform ("g_AudioSpectrum32Right", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio32, 32);
this->addUniform ("g_AudioSpectrum64Left", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio64, 64);
this->addUniform ("g_AudioSpectrum64Right", this->getMaterial ()->getImage ()->getScene ()->getAudioContext ().getRecorder ().audio64, 64);
}
void CPass::addAttribute (const std::string& name, GLint type, GLint elements, const GLuint* value)
@ -623,12 +630,12 @@ void CPass::addUniform (const std::string& name, UniformType type, T value)
// uniform found, add it to the list
this->m_uniforms.insert (
std::make_pair (name, new UniformEntry (id, name, type, newValue))
std::make_pair (name, new UniformEntry (id, name, type, newValue, 1))
);
}
template <typename T>
void CPass::addUniform (const std::string& name, UniformType type, T* value)
void CPass::addUniform (const std::string& name, UniformType type, T* value, int count)
{
// this version is used to reference to system variables so things like g_Time works fine
GLint id = glGetUniformLocation (this->m_programID, name.c_str ());
@ -639,7 +646,7 @@ void CPass::addUniform (const std::string& name, UniformType type, T* value)
// uniform found, add it to the list
this->m_uniforms.insert (
std::make_pair (name, new UniformEntry (id, name, type, value))
std::make_pair (name, new UniformEntry (id, name, type, value, count))
);
}
@ -804,9 +811,9 @@ void CPass::addUniform (const std::string& name, int value)
this->addUniform (name, UniformType::Integer, value);
}
void CPass::addUniform (const std::string& name, const int* value)
void CPass::addUniform (const std::string& name, const int* value, int count)
{
this->addUniform (name, UniformType::Integer, value);
this->addUniform (name, UniformType::Integer, value, count);
}
void CPass::addUniform (const std::string& name, const int** value)
@ -819,9 +826,9 @@ void CPass::addUniform (const std::string& name, double value)
this->addUniform (name, UniformType::Double, value);
}
void CPass::addUniform (const std::string& name, const double* value)
void CPass::addUniform (const std::string& name, const double* value, int count)
{
this->addUniform (name, UniformType::Double, value);
this->addUniform (name, UniformType::Double, value, count);
}
void CPass::addUniform (const std::string& name, const double** value)
@ -834,9 +841,9 @@ void CPass::addUniform (const std::string& name, float value)
this->addUniform (name, UniformType::Float, value);
}
void CPass::addUniform (const std::string& name, const float* value)
void CPass::addUniform (const std::string& name, const float* value, int count)
{
this->addUniform (name, UniformType::Float, value);
this->addUniform (name, UniformType::Float, value, count);
}
void CPass::addUniform (const std::string& name, const float** value)
@ -852,12 +859,12 @@ void CPass::addUniform (const std::string& name, glm::vec2 value)
void CPass::addUniform (const std::string& name, const glm::vec2* value)
{
this->addUniform (name, UniformType::Vector2, value);
this->addUniform (name, UniformType::Vector2, value, 1);
}
void CPass::addUniform (const std::string& name, const glm::vec2** value)
{
this->addUniform (name, UniformType::Vector2, value);
this->addUniform (name, UniformType::Vector2, value, 1);
}
@ -868,7 +875,7 @@ void CPass::addUniform (const std::string& name, glm::vec3 value)
void CPass::addUniform (const std::string& name, const glm::vec3* value)
{
this->addUniform (name, UniformType::Vector3, value);
this->addUniform (name, UniformType::Vector3, value, 1);
}
void CPass::addUniform (const std::string& name, const glm::vec3** value)
@ -884,7 +891,7 @@ void CPass::addUniform (const std::string& name, glm::vec4 value)
void CPass::addUniform (const std::string& name, const glm::vec4* value)
{
this->addUniform (name, UniformType::Vector4, value);
this->addUniform (name, UniformType::Vector4, value, 1);
}
void CPass::addUniform (const std::string& name, const glm::vec4** value)
@ -899,7 +906,7 @@ void CPass::addUniform (const std::string& name, glm::mat4 value)
void CPass::addUniform (const std::string& name, const glm::mat4* value)
{
this->addUniform (name, UniformType::Matrix4, value);
this->addUniform (name, UniformType::Matrix4, value, 1);
}
void CPass::addUniform (const std::string& name, const glm::mat4** value)

13
src/WallpaperEngine/Render/Objects/Effects/CPass.h
View File

@ -52,13 +52,14 @@ namespace WallpaperEngine::Render::Objects::Effects
class UniformEntry
{
public:
UniformEntry (GLint id, std::string name, UniformType type, const void* value) :
id (id), name (std::move (name)), type (type), value (value) { }
UniformEntry (GLint id, std::string name, UniformType type, const void* value, int count) :
id (id), name (std::move (name)), type (type), value (value), count (count) { }
GLint id;
std::string name;
UniformType type;
const void* value;
int count;
};
class ReferenceUniformEntry
@ -102,9 +103,9 @@ namespace WallpaperEngine::Render::Objects::Effects
void addUniform (const std::string& name, glm::vec3 value);
void addUniform (const std::string& name, glm::vec4 value);
void addUniform (const std::string& name, glm::mat4 value);
void addUniform (const std::string& name, const int* value);
void addUniform (const std::string& name, const double* value);
void addUniform (const std::string& name, const float* value);
void addUniform (const std::string& name, const int* value, int count = 1);
void addUniform (const std::string& name, const double* value, int count = 1);
void addUniform (const std::string& name, const float* value, int count = 1);
void addUniform (const std::string& name, const glm::vec2* value);
void addUniform (const std::string& name, const glm::vec3* value);
void addUniform (const std::string& name, const glm::vec4* value);
@ -117,7 +118,7 @@ namespace WallpaperEngine::Render::Objects::Effects
void addUniform (const std::string& name, const glm::vec4** value);
void addUniform (const std::string& name, const glm::mat4** value);
template <typename T> void addUniform (const std::string& name, UniformType type, T value);
template <typename T> void addUniform (const std::string& name, UniformType type, T* value);
template <typename T> void addUniform (const std::string& name, UniformType type, T* value, int count = 1);
template <typename T> void addUniform (const std::string& name, UniformType type, T** value);
const ITexture* resolveTexture (const ITexture* expected, int index, const ITexture* previous = nullptr);

48
src/WallpaperEngine/Render/Shaders/Compiler.cpp
View File

@ -701,38 +701,30 @@ namespace WallpaperEngine::Render::Shaders
// so only define the ones that are not already defined
if (entry == this->m_combos->end ())
{
if (type != data.end () && (*type) == "audioprocessingoptions")
if (type != data.end ())
sLog.error ("Resorting to default value as type ", *type, " is unknown");
// if no combo is defined just load the default settings
if (defvalue == data.end ())
{
sLog.out ("Found audioprocessing value, nothing working yet");
this->m_combos->insert (std::make_pair <std::string, int> (*combo, 1));
// TODO: PROPERLY SUPPORT EMPTY COMBOS
this->m_combos->insert (std::make_pair <std::string, int> (*combo, (int) defaultValue));
}
else if ((*defvalue).is_number_float ())
{
sLog.exception ("float combos are not supported in shader ", this->m_file, ". ", *combo);
}
else if ((*defvalue).is_number_integer ())
{
this->m_combos->insert (std::make_pair <std::string, int> (*combo, (*defvalue).get <int> ()));
}
else if ((*defvalue).is_string ())
{
sLog.exception ("string combos are not supported in shader ", this->m_file, ". ", *combo);
}
else
{
if (type != data.end ())
sLog.error ("Resorting to default value as type ", *type, " is unknown");
// if no combo is defined just load the default settings
if (defvalue == data.end ())
{
// TODO: PROPERLY SUPPORT EMPTY COMBOS
this->m_combos->insert (std::make_pair <std::string, int> (*combo, (int) defaultValue));
}
else if ((*defvalue).is_number_float ())
{
sLog.exception ("float combos are not supported in shader ", this->m_file, ". ", *combo);
}
else if ((*defvalue).is_number_integer ())
{
this->m_combos->insert (std::make_pair <std::string, int> (*combo, (*defvalue).get <int> ()));
}
else if ((*defvalue).is_string ())
{
sLog.exception ("string combos are not supported in shader ", this->m_file, ". ", *combo);
}
else
{
sLog.exception ("cannot parse combo information ", *combo, ". unknown type for ", defvalue->dump ());
}
sLog.exception ("cannot parse combo information ", *combo, ". unknown type for ", defvalue->dump ());
}
}
}