From 1b5ba6b584fffc504ffdfec4d26710311fc31979 Mon Sep 17 00:00:00 2001 From: shweet Date: Tue, 10 Mar 2026 00:36:21 -0400 Subject: [PATCH] fix audio desync for realsies, stderr to log.txt --- compile_commands.json | 2 +- src/imgui/window/animation_preview.cpp | 125 +++++++++++++++++++++++-- src/imgui/window/animation_preview.hpp | 3 + src/log.cpp | 111 +++++++++++++++++++++- src/log.hpp | 12 +++ src/render.cpp | 28 ++++-- src/render.hpp | 3 +- src/resource/audio.cpp | 8 ++ src/resource/audio.hpp | 1 + src/resource/audio_stream.cpp | 36 ++++++- src/resource/audio_stream.hpp | 8 +- src/state.cpp | 29 +++++- src/state.hpp | 2 + 13 files changed, 347 insertions(+), 21 deletions(-) diff --git a/compile_commands.json b/compile_commands.json index 5e95284..bc0dce8 120000 --- a/compile_commands.json +++ b/compile_commands.json @@ -1 +1 @@ -/home/anon/sda/Personal/Repos/anm2ed/build/compile_commands.json \ No newline at end of file +/home/anon/sda/Personal/Repos/anm2ed/out/build/linux-debug/compile_commands.json \ No newline at end of file diff --git a/src/imgui/window/animation_preview.cpp b/src/imgui/window/animation_preview.cpp index e4a7a20..bdfe0d5 100644 --- a/src/imgui/window/animation_preview.cpp +++ b/src/imgui/window/animation_preview.cpp @@ -1,9 +1,11 @@ #include "animation_preview.hpp" #include +#include #include #include #include +#include #include #include #include @@ -82,6 +84,51 @@ namespace anm2ed::imgui directory.clear(); frames.clear(); } + + bool render_audio_stream_generate(AudioStream& audioStream, std::map& sounds, + const std::vector& frameSoundIDs, int fps) + { + audioStream.stream.clear(); + if (frameSoundIDs.empty() || fps <= 0) return true; + + SDL_AudioSpec mixSpec = audioStream.spec; + mixSpec.format = SDL_AUDIO_F32; + auto* mixer = MIX_CreateMixer(&mixSpec); + if (!mixer) return false; + + auto channels = std::max(mixSpec.channels, 1); + auto sampleRate = std::max(mixSpec.freq, 1); + auto framesPerStep = (double)sampleRate / (double)fps; + auto sampleFrameAccumulator = 0.0; + auto frameBuffer = std::vector{}; + + for (auto soundID : frameSoundIDs) + { + if (soundID != -1 && sounds.contains(soundID)) sounds.at(soundID).audio.play(false, mixer); + + sampleFrameAccumulator += framesPerStep; + auto sampleFramesToGenerate = (int)std::floor(sampleFrameAccumulator); + sampleFramesToGenerate = std::max(sampleFramesToGenerate, 1); + sampleFrameAccumulator -= (double)sampleFramesToGenerate; + + frameBuffer.resize((std::size_t)sampleFramesToGenerate * (std::size_t)channels); + if (!MIX_Generate(mixer, frameBuffer.data(), (int)(frameBuffer.size() * sizeof(float)))) + { + for (auto& [_, sound] : sounds) + sound.audio.track_detach(mixer); + MIX_DestroyMixer(mixer); + audioStream.stream.clear(); + return false; + } + + audioStream.stream.insert(audioStream.stream.end(), frameBuffer.begin(), frameBuffer.end()); + } + + for (auto& [_, sound] : sounds) + sound.audio.track_detach(mixer); + MIX_DestroyMixer(mixer); + return true; + } } AnimationPreview::AnimationPreview() : Canvas(vec2()) {} @@ -106,6 +153,8 @@ namespace anm2ed::imgui if (playback.time > end || playback.isFinished) { + if (settings.timelineIsSound) audioStream.capture_end(mixer); + if (type == render::PNGS) { if (!renderTempFrames.empty()) @@ -185,7 +234,20 @@ namespace anm2ed::imgui } else { - if (animation_render(ffmpegPath, path, renderTempFrames, audioStream, (render::Type)type, anm2.info.fps)) + if (settings.timelineIsSound && type != render::GIF) + { + if (!render_audio_stream_generate(audioStream, anm2.content.sounds, renderFrameSoundIDs, anm2.info.fps)) + { + toasts.push(localize.get(TOAST_EXPORT_RENDERED_ANIMATION_FAILED)); + logger.error("Failed to generate deterministic render audio stream; exporting without audio."); + audioStream.stream.clear(); + } + } + else + audioStream.stream.clear(); + + if (animation_render(ffmpegPath, path, renderTempFrames, renderTempFrameDurations, audioStream, + (render::Type)type, anm2.info.fps)) { toasts.push(std::vformat(localize.get(TOAST_EXPORT_RENDERED_ANIMATION), std::make_format_args(pathString))); logger.info(std::vformat(localize.get(TOAST_EXPORT_RENDERED_ANIMATION, anm2ed::ENGLISH), @@ -204,9 +266,15 @@ namespace anm2ed::imgui { renderTempDirectory.clear(); renderTempFrames.clear(); + renderTempFrameDurations.clear(); + renderFrameSoundIDs.clear(); } else + { render_temp_cleanup(renderTempDirectory, renderTempFrames); + renderTempFrameDurations.clear(); + renderFrameSoundIDs.clear(); + } if (settings.renderIsRawAnimation) { @@ -220,8 +288,6 @@ namespace anm2ed::imgui isCheckerPanInitialized = false; } - if (settings.timelineIsSound) audioStream.capture_end(mixer); - playback.isPlaying = false; playback.isFinished = false; manager.isRecording = false; @@ -229,6 +295,28 @@ namespace anm2ed::imgui } else { + if (settings.timelineIsSound && renderTempFrames.empty()) audioStream.capture_begin(mixer); + auto frameSoundID = -1; + if (settings.timelineIsSound && !anm2.content.sounds.empty()) + { + if (auto animation = document.animation_get(); + animation && animation->triggers.isVisible && (!settings.timelineIsOnlyShowLayers || manager.isRecording)) + { + if (auto trigger = animation->triggers.frame_generate(playback.time, anm2::TRIGGER); trigger.isVisible) + { + if (!trigger.soundIDs.empty()) + { + auto soundIndex = trigger.soundIDs.size() > 1 + ? (size_t)math::random_in_range(0.0f, (float)trigger.soundIDs.size()) + : (size_t)0; + soundIndex = std::min(soundIndex, trigger.soundIDs.size() - 1); + auto soundID = trigger.soundIDs[soundIndex]; + if (anm2.content.sounds.contains(soundID)) frameSoundID = soundID; + } + } + } + } + renderFrameSoundIDs.push_back(frameSoundID); bind(); auto pixels = pixels_get(); @@ -237,6 +325,24 @@ namespace anm2ed::imgui if (Texture::write_pixels_png(framePath, size, pixels.data())) { renderTempFrames.push_back(framePath); + auto nowCounter = SDL_GetPerformanceCounter(); + auto counterFrequency = SDL_GetPerformanceFrequency(); + auto fallbackDuration = 1.0 / (double)std::max(anm2.info.fps, 1); + + if (renderTempFrames.size() == 1) + { + renderCaptureCounterPrev = nowCounter; + renderTempFrameDurations.push_back(fallbackDuration); + } + else + { + auto elapsedCounter = nowCounter - renderCaptureCounterPrev; + auto frameDuration = counterFrequency > 0 ? (double)elapsedCounter / (double)counterFrequency : 0.0; + frameDuration = std::max(frameDuration, 1.0 / 1000.0); + renderTempFrameDurations.back() = frameDuration; + renderTempFrameDurations.push_back(frameDuration); + renderCaptureCounterPrev = nowCounter; + } } else { @@ -244,6 +350,8 @@ namespace anm2ed::imgui logger.error(std::vformat(localize.get(TOAST_EXPORT_RENDERED_ANIMATION_FAILED, anm2ed::ENGLISH), std::make_format_args(pathString))); if (type != render::PNGS) render_temp_cleanup(renderTempDirectory, renderTempFrames); + renderTempFrameDurations.clear(); + renderFrameSoundIDs.clear(); playback.isPlaying = false; playback.isFinished = false; manager.isRecording = false; @@ -258,7 +366,7 @@ namespace anm2ed::imgui auto& isSound = settings.timelineIsSound; auto& isOnlyShowLayers = settings.timelineIsOnlyShowLayers; - if (!anm2.content.sounds.empty() && isSound) + if (!manager.isRecording && !anm2.content.sounds.empty() && isSound) { if (auto animation = document.animation_get(); animation && animation->triggers.isVisible && (!isOnlyShowLayers || manager.isRecording)) @@ -474,8 +582,6 @@ namespace anm2ed::imgui { savedSettings = settings; - if (settings.timelineIsSound) audioStream.capture_begin(mixer); - if (settings.renderIsRawAnimation) { settings.previewBackgroundColor = vec4(); @@ -502,6 +608,9 @@ namespace anm2ed::imgui manager.isRecordingStart = false; manager.isRecording = true; renderTempFrames.clear(); + renderTempFrameDurations.clear(); + renderFrameSoundIDs.clear(); + renderCaptureCounterPrev = 0; if (settings.renderType == render::PNGS) { renderTempDirectory = settings.renderPath; @@ -1033,9 +1142,13 @@ namespace anm2ed::imgui { renderTempDirectory.clear(); renderTempFrames.clear(); + renderTempFrameDurations.clear(); } else + { render_temp_cleanup(renderTempDirectory, renderTempFrames); + renderTempFrameDurations.clear(); + } pan = savedPan; zoom = savedZoom; diff --git a/src/imgui/window/animation_preview.hpp b/src/imgui/window/animation_preview.hpp index 7645a4f..152a229 100644 --- a/src/imgui/window/animation_preview.hpp +++ b/src/imgui/window/animation_preview.hpp @@ -30,6 +30,9 @@ namespace anm2ed::imgui glm::vec2 moveOffset{}; std::filesystem::path renderTempDirectory{}; std::vector renderTempFrames{}; + std::vector renderTempFrameDurations{}; + std::vector renderFrameSoundIDs{}; + Uint64 renderCaptureCounterPrev{}; public: AnimationPreview(); diff --git a/src/log.cpp b/src/log.cpp index 19374f4..f5a3f24 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -1,16 +1,26 @@ #include "log.hpp" +#include #include +#include "path_.hpp" #include "sdl.hpp" #include "time_.hpp" +#if _WIN32 + #include + #include +#else + #include +#endif + using namespace anm2ed::util; namespace anm2ed { void Logger::write_raw(const std::string& message) { + std::lock_guard lock(mutex); std::println("{}", message); if (file.is_open()) file << message << '\n' << std::flush; } @@ -26,7 +36,105 @@ namespace anm2ed void Logger::error(const std::string& message) { write(ERROR, message); } void Logger::fatal(const std::string& message) { write(FATAL, message); } void Logger::command(const std::string& message) { write(COMMAND, message); } - void Logger::open(const std::filesystem::path& path) { file.open(path, std::ios::out | std::ios::app); } + + void Logger::stderr_pump() + { + std::string pending{}; + std::array buffer{}; + while (isStderrRedirecting) + { + int readBytes{}; +#if _WIN32 + readBytes = _read(stderrPipeReadFd, buffer.data(), (unsigned int)buffer.size()); +#else + readBytes = (int)read(stderrPipeReadFd, buffer.data(), buffer.size()); +#endif + if (readBytes <= 0) break; + + pending.append(buffer.data(), (std::size_t)readBytes); + + std::size_t lineEnd{}; + while ((lineEnd = pending.find('\n')) != std::string::npos) + { + auto line = pending.substr(0, lineEnd); + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (!line.empty()) write_raw(line); + pending.erase(0, lineEnd + 1); + } + } + + if (!pending.empty()) write_raw(pending); + } + + void Logger::stderr_redirect_start() + { + if (isStderrRedirecting) return; + + int pipeFds[2]{-1, -1}; +#if _WIN32 + if (_pipe(pipeFds, 4096, _O_BINARY) != 0) return; + stderrOriginalFd = _dup(_fileno(stderr)); + if (stderrOriginalFd < 0 || _dup2(pipeFds[1], _fileno(stderr)) != 0) + { + _close(pipeFds[0]); + _close(pipeFds[1]); + return; + } + _close(pipeFds[1]); +#else + if (pipe(pipeFds) != 0) return; + stderrOriginalFd = dup(fileno(stderr)); + if (stderrOriginalFd < 0 || dup2(pipeFds[1], fileno(stderr)) < 0) + { + close(pipeFds[0]); + close(pipeFds[1]); + return; + } + close(pipeFds[1]); +#endif + + std::setvbuf(stderr, nullptr, _IONBF, 0); + + stderrPipeReadFd = pipeFds[0]; + isStderrRedirecting = true; + stderrThread = std::thread([this]() { stderr_pump(); }); + } + + void Logger::stderr_redirect_stop() + { + if (!isStderrRedirecting) return; + isStderrRedirecting = false; + + if (stderrOriginalFd >= 0) + { +#if _WIN32 + _dup2(stderrOriginalFd, _fileno(stderr)); + _close(stderrOriginalFd); +#else + dup2(stderrOriginalFd, fileno(stderr)); + close(stderrOriginalFd); +#endif + stderrOriginalFd = -1; + } + + if (stderrPipeReadFd >= 0) + { +#if _WIN32 + _close(stderrPipeReadFd); +#else + close(stderrPipeReadFd); +#endif + stderrPipeReadFd = -1; + } + + if (stderrThread.joinable()) stderrThread.join(); + } + + void Logger::open(const std::filesystem::path& path) + { + file.open(path, std::ios::out | std::ios::app); + stderr_redirect_start(); + } std::filesystem::path Logger::path() { return sdl::preferences_directory_get() / "log.txt"; } @@ -39,6 +147,7 @@ namespace anm2ed Logger::~Logger() { info("Exiting Anm2Ed"); + stderr_redirect_stop(); if (file.is_open()) file.close(); } } diff --git a/src/log.hpp b/src/log.hpp index f041520..dd7e692 100644 --- a/src/log.hpp +++ b/src/log.hpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include namespace anm2ed { @@ -44,6 +46,16 @@ namespace anm2ed class Logger { std::ofstream file{}; + std::mutex mutex{}; + std::thread stderrThread{}; + bool isStderrRedirecting{}; + int stderrPipeReadFd{-1}; + int stderrPipeWriteFd{-1}; + int stderrOriginalFd{-1}; + + void stderr_redirect_start(); + void stderr_redirect_stop(); + void stderr_pump(); public: static std::filesystem::path path(); diff --git a/src/render.cpp b/src/render.cpp index 74018f5..008b7bf 100644 --- a/src/render.cpp +++ b/src/render.cpp @@ -35,10 +35,11 @@ namespace anm2ed } bool animation_render(const std::filesystem::path& ffmpegPath, const std::filesystem::path& path, - const std::vector& framePaths, AudioStream& audioStream, - render::Type type, int fps) + const std::vector& framePaths, + const std::vector& frameDurations, AudioStream& audioStream, render::Type type, int fps) { if (framePaths.empty() || ffmpegPath.empty() || path.empty()) return false; + (void)frameDurations; fps = std::max(fps, 1); auto pathString = path::to_utf8(path); @@ -83,8 +84,16 @@ namespace anm2ed audioFile.write(data, byteCount); audioFile.close(); - audioInputArguments = std::format("-f f32le -ar {0} -ac {1} -i \"{2}\"", audioStream.spec.freq, - audioStream.spec.channels, path::to_utf8(audioPath)); + auto sampleRate = std::max(audioStream.spec.freq, 1); + auto channels = std::max(audioStream.spec.channels, 1); + auto audioDurationFilter = std::string{}; + auto frameCount = (double)std::max((int)framePaths.size(), 1); + auto expectedDurationSeconds = frameCount / (double)fps; + if (expectedDurationSeconds > 0.0) + audioDurationFilter += std::format("apad,atrim=duration={:.9f}", expectedDurationSeconds); + + audioInputArguments = + std::format("-f f32le -ar {0} -ac {1} -i \"{2}\"", sampleRate, channels, path::to_utf8(audioPath)); switch (type) { @@ -97,6 +106,9 @@ namespace anm2ed default: break; } + + if (!audioDurationFilter.empty()) + audioOutputArguments = std::format("-af \"{}\" {}", audioDurationFilter, audioOutputArguments); } else { @@ -114,10 +126,12 @@ namespace anm2ed return false; } - auto frameDuration = 1.0 / (double)fps; - for (const auto& framePath : framePaths) + auto defaultFrameDuration = 1.0 / (double)fps; + for (std::size_t index = 0; index < framePaths.size(); ++index) { + auto framePath = framePaths[index]; auto framePathString = path::to_utf8(framePath); + auto frameDuration = defaultFrameDuration; framesListFile << "file '" << ffmpeg_concat_escape(framePathString) << "'\n"; framesListFile << "duration " << std::format("{:.9f}", frameDuration) << "\n"; } @@ -127,9 +141,9 @@ namespace anm2ed auto framesListPathString = path::to_utf8(framesListPath); command = std::format("\"{0}\" -y -f concat -safe 0 -i \"{1}\"", ffmpegPathString, framesListPathString); - command += std::format(" -fps_mode cfr -r {}", fps); if (!audioInputArguments.empty()) command += " " + audioInputArguments; + command += std::format(" -fps_mode cfr -r {}", fps); switch (type) { diff --git a/src/render.hpp b/src/render.hpp index 95d1bd8..5c6639b 100644 --- a/src/render.hpp +++ b/src/render.hpp @@ -37,5 +37,6 @@ namespace anm2ed { std::filesystem::path ffmpeg_log_path(); bool animation_render(const std::filesystem::path&, const std::filesystem::path&, - const std::vector&, AudioStream&, render::Type, int); + const std::vector&, const std::vector&, AudioStream&, + render::Type, int); } diff --git a/src/resource/audio.cpp b/src/resource/audio.cpp index 79c3699..9ea969f 100644 --- a/src/resource/audio.cpp +++ b/src/resource/audio.cpp @@ -117,6 +117,14 @@ namespace anm2ed::resource MIX_StopTrack(track, 0); } + void Audio::track_detach(MIX_Mixer* mixer) + { + if (!track) return; + if (mixer && MIX_GetTrackMixer(track) != mixer) return; + MIX_DestroyTrack(track); + track = nullptr; + } + bool Audio::is_playing() const { return track && MIX_TrackPlaying(track); } Audio::Audio(const Audio& other) diff --git a/src/resource/audio.hpp b/src/resource/audio.hpp index 65301cc..f93773e 100644 --- a/src/resource/audio.hpp +++ b/src/resource/audio.hpp @@ -28,6 +28,7 @@ namespace anm2ed::resource bool is_valid(); void play(bool loop = false, MIX_Mixer* = nullptr); void stop(MIX_Mixer* = nullptr); + void track_detach(MIX_Mixer* = nullptr); bool is_playing() const; }; } diff --git a/src/resource/audio_stream.cpp b/src/resource/audio_stream.cpp index 2ae3dfc..d569d71 100644 --- a/src/resource/audio_stream.cpp +++ b/src/resource/audio_stream.cpp @@ -1,5 +1,7 @@ #include "audio_stream.hpp" +#include + #if defined(__clang__) || defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -10,16 +12,46 @@ namespace anm2ed void AudioStream::callback(void* userData, MIX_Mixer* mixer, const SDL_AudioSpec* spec, float* pcm, int samples) { auto self = (AudioStream*)userData; + if (!self->isFirstCallbackCaptured) + { + self->firstCallbackCounter = SDL_GetPerformanceCounter(); + self->isFirstCallbackCaptured = true; + } + self->callbackSamples = samples; self->stream.insert(self->stream.end(), pcm, pcm + samples); } AudioStream::AudioStream(MIX_Mixer* mixer) { MIX_GetMixerFormat(mixer, &spec); } - void AudioStream::capture_begin(MIX_Mixer* mixer) { MIX_SetPostMixCallback(mixer, callback, this); } + void AudioStream::capture_begin(MIX_Mixer* mixer) + { + stream.clear(); + callbackSamples = 0; + captureStartCounter = SDL_GetPerformanceCounter(); + firstCallbackCounter = 0; + isFirstCallbackCaptured = false; + MIX_SetPostMixCallback(mixer, callback, this); + } void AudioStream::capture_end(MIX_Mixer* mixer) { MIX_SetPostMixCallback(mixer, nullptr, this); stream.clear(); } -} \ No newline at end of file + + double AudioStream::callback_latency_seconds_get() const + { + auto freq = std::max(spec.freq, 1); + auto channels = std::max(spec.channels, 1); + auto framesPerCallback = (double)callbackSamples / (double)channels; + return framesPerCallback / (double)freq; + } + + double AudioStream::capture_start_delay_seconds_get() const + { + if (!isFirstCallbackCaptured || captureStartCounter == 0 || firstCallbackCounter < captureStartCounter) return 0.0; + auto frequency = SDL_GetPerformanceFrequency(); + if (frequency == 0) return 0.0; + return (double)(firstCallbackCounter - captureStartCounter) / (double)frequency; + } +} diff --git a/src/resource/audio_stream.hpp b/src/resource/audio_stream.hpp index 1cbb90f..f5fe532 100644 --- a/src/resource/audio_stream.hpp +++ b/src/resource/audio_stream.hpp @@ -12,9 +12,15 @@ namespace anm2ed public: std::vector stream{}; SDL_AudioSpec spec{}; + int callbackSamples{}; + Uint64 captureStartCounter{}; + Uint64 firstCallbackCounter{}; + bool isFirstCallbackCaptured{}; AudioStream(MIX_Mixer*); void capture_begin(MIX_Mixer*); void capture_end(MIX_Mixer*); + double callback_latency_seconds_get() const; + double capture_start_delay_seconds_get() const; }; -} \ No newline at end of file +} diff --git a/src/state.cpp b/src/state.cpp index 9864ea2..8668bcc 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -159,6 +159,31 @@ namespace anm2ed { auto currentTick = SDL_GetTicks(); auto currentUpdate = SDL_GetTicks(); + auto isRecording = manager.isRecording; + auto tickIntervalMs = (double)TICK_INTERVAL; + + if (isRecording) + { + if (auto document = manager.get()) + { + auto fps = std::max(document->anm2.info.fps, 1); + tickIntervalMs = std::max(1.0, 1000.0 / (double)fps); + } + } + + if (isRecording != wasRecording) + { + // Drop any accumulated backlog when entering/leaving recording mode. + tickAccumulatorMs = 0.0; + previousTick = currentTick; + wasRecording = isRecording; + } + + if (previousTick == 0) previousTick = currentTick; + auto tickDeltaMs = currentTick - previousTick; + tickDeltaMs = std::min(tickDeltaMs, 250); + tickAccumulatorMs += (double)tickDeltaMs; + previousTick = currentTick; if (currentUpdate - previousUpdate >= UPDATE_INTERVAL) { @@ -167,10 +192,10 @@ namespace anm2ed previousUpdate = currentUpdate; } - if (currentTick - previousTick >= TICK_INTERVAL) + if (tickAccumulatorMs >= tickIntervalMs) { tick(settings); - previousTick = currentTick; + tickAccumulatorMs -= tickIntervalMs; } SDL_Delay(1); diff --git a/src/state.hpp b/src/state.hpp index e911559..200791e 100644 --- a/src/state.hpp +++ b/src/state.hpp @@ -26,6 +26,8 @@ namespace anm2ed uint64_t previousTick{}; uint64_t previousUpdate{}; + double tickAccumulatorMs{}; + bool wasRecording{}; State(SDL_Window*&, Settings& settings, std::vector&);