D3D12: Improve robustness of command allocator and fence tracking

2 files changed, 83 insertions, 57 deletions

diff --git a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp
index fc4a98ae84..4fe6875f13 100644
--- a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp
+++ b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.cpp
@@ -2,6 +2,7 @@
 // Licensed under GPLv2+
 // Refer to the license.txt file included.
 
+#include <algorithm>
 #include <queue>
 #include <vector>
 
@@ -69,6 +70,9 @@ D3DCommandListManager::D3DCommandListManager(
 	}
 
 	m_current_deferred_destruction_list = 0;
+
+	std::fill(m_command_allocator_list_fences.begin(), m_command_allocator_list_fences.end(), 0);
+	std::fill(m_deferred_destruction_list_fences.begin(), m_deferred_destruction_list_fences.end(), 0);
 }
 
 void D3DCommandListManager::SetInitialCommandListState()
@@ -114,37 +118,26 @@ void D3DCommandListManager::ExecuteQueuedWork(bool wait_for_gpu_completion)
 
 	m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value);
 
-	ResetCommandListWithIdleCommandAllocator();
-
-	m_queued_command_list->ProcessQueuedItems();
+	m_queued_command_list->ProcessQueuedItems(wait_for_gpu_completion);
 #else
 	CheckHR(m_backing_command_list->Close());
 
-	ID3D12CommandList* const commandListsToExecute[1] = { m_backing_command_list };
-	m_command_queue->ExecuteCommandLists(1, commandListsToExecute);
-
-	if (wait_for_gpu_completion)
-	{
-		CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value));
-	}
-
-	if (m_current_command_allocator == 0)
-	{
-		PerformGpuRolloverChecks();
-	}
+	ID3D12CommandList* const execute_list[1] = { m_backing_command_list };
+	m_command_queue->ExecuteCommandLists(1, execute_list);
 
-	ResetCommandListWithIdleCommandAllocator();
+	CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value));
 #endif
 
+	// Notify observers of the fence value for the current work to finish.
 	for (auto it : m_queue_fence_callbacks)
 		it.second(it.first, m_queue_fence_value);
 
-	SetInitialCommandListState();
-
 	if (wait_for_gpu_completion)
-	{
-		WaitOnCPUForFence(m_queue_fence, m_queue_fence_value);
-	}
+		WaitForGPUCompletion();
+
+	// Re-open the command list, using the current allocator.
+	ResetCommandList();
+	SetInitialCommandListState();
 }
 
 void D3DCommandListManager::ExecuteQueuedWorkAndPresent(IDXGISwapChain* swap_chain, UINT sync_interval, UINT flags)
@@ -154,60 +147,67 @@ void D3DCommandListManager::ExecuteQueuedWorkAndPresent(IDXGISwapChain* swap_cha
 #ifdef USE_D3D12_QUEUED_COMMAND_LISTS
 	CheckHR(m_queued_command_list->Close());
 	m_queued_command_list->QueueExecute();
-	m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value);
 	m_queued_command_list->QueuePresent(swap_chain, sync_interval, flags);
+	m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value);
 	m_queued_command_list->ProcessQueuedItems(true);
+#else
+	CheckHR(m_backing_command_list->Close());
 
-	if (m_current_command_allocator == 0)
-	{
-		PerformGpuRolloverChecks();
-	}
-
-	m_current_command_allocator = (m_current_command_allocator + 1) % m_command_allocator_lists[m_current_command_allocator_list].size();
-
-	ResetCommandListWithIdleCommandAllocator();
+	ID3D12CommandList* const execute_list[1] = { m_backing_command_list };
+	m_command_queue->ExecuteCommandLists(1, execute_list);
 
-	SetInitialCommandListState();
-#else
-	ExecuteQueuedWork();
-	m_command_queue->Signal(m_queue_fence, m_queue_fence_value);
 	CheckHR(swap_chain->Present(sync_interval, flags));
+	CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value));
 #endif
 
+	// Notify observers of the fence value for the current work to finish.
 	for (auto it : m_queue_fence_callbacks)
 		it.second(it.first, m_queue_fence_value);
+
+	// Move to the next command allocator, this may mean switching allocator lists.
+	MoveToNextCommandAllocator();
+	ResetCommandList();
+	SetInitialCommandListState();
 }
 
-void D3DCommandListManager::WaitForQueuedWorkToBeExecutedOnGPU()
+void D3DCommandListManager::WaitForGPUCompletion()
 {
 	// Wait for GPU to finish all outstanding work.
-	m_queue_fence_value++;
+	// This method assumes that no command lists are open.
+	m_queue_frame_fence_value++;
 
 #ifdef USE_D3D12_QUEUED_COMMAND_LISTS
-	m_queued_command_list->QueueExecute();
-	m_queued_command_list->QueueFenceGpuSignal(m_queue_fence, m_queue_fence_value);
-
+	m_queued_command_list->QueueFenceGpuSignal(m_queue_frame_fence, m_queue_frame_fence_value);
 	m_queued_command_list->ProcessQueuedItems(true);
 #else
-	CheckHR(m_command_queue->Signal(m_queue_fence, m_queue_fence_value));
+	CheckHR(m_command_queue->Signal(m_queue_frame_fence, m_queue_frame_fence_value));
 #endif
 
-	WaitOnCPUForFence(m_queue_fence, m_queue_fence_value);
+	WaitOnCPUForFence(m_queue_frame_fence, m_queue_frame_fence_value);
+
+	// GPU is up to date with us. Therefore, it has finished with any pending resources.
+	ImmediatelyDestroyAllResourcesScheduledForDestruction();
+
+	// Command allocators are also up-to-date, so reset these.
+	ResetAllCommandAllocators();
 }
 
 void D3DCommandListManager::PerformGpuRolloverChecks()
 {
-	// Insert fence to measure GPU progress, ensure we aren't using in-use command allocators.
-	if (m_queue_frame_fence->GetCompletedValue() < m_queue_frame_fence_value)
-	{
-		WaitOnCPUForFence(m_queue_frame_fence, m_queue_frame_fence_value);
-	}
+	m_queue_frame_fence_value++;
+
+#ifdef USE_D3D12_QUEUED_COMMAND_LISTS
+	m_queued_command_list->QueueFenceGpuSignal(m_queue_frame_fence, m_queue_frame_fence_value);
+#else
+	CheckHR(m_command_queue->Signal(m_queue_frame_fence, m_queue_frame_fence_value));
+#endif
 
 	// We now know that the previous 'set' of command lists has completed on GPU, and it is safe to
 	// release resources / start back at beginning of command allocator list.
 
 	// Begin Deferred Resource Destruction
 	UINT safe_to_delete_deferred_destruction_list = (m_current_deferred_destruction_list - 1) % m_deferred_destruction_lists.size();
+	WaitOnCPUForFence(m_queue_frame_fence, m_deferred_destruction_list_fences[safe_to_delete_deferred_destruction_list]);
 
 	for (UINT i = 0; i < m_deferred_destruction_lists[safe_to_delete_deferred_destruction_list].size(); i++)
 	{
@@ -216,30 +216,37 @@ void D3DCommandListManager::PerformGpuRolloverChecks()
 
 	m_deferred_destruction_lists[safe_to_delete_deferred_destruction_list].clear();
 
+	m_deferred_destruction_list_fences[m_current_deferred_destruction_list] = m_queue_frame_fence_value;
 	m_current_deferred_destruction_list = (m_current_deferred_destruction_list + 1) % m_deferred_destruction_lists.size();
 	// End Deferred Resource Destruction
 
 
 	// Begin Command Allocator Resets
 	UINT safe_to_reset_command_allocator_list = (m_current_command_allocator_list - 1) % m_command_allocator_lists.size();
+	WaitOnCPUForFence(m_queue_frame_fence, m_command_allocator_list_fences[safe_to_reset_command_allocator_list]);
 
 	for (UINT i = 0; i < m_command_allocator_lists[safe_to_reset_command_allocator_list].size(); i++)
 	{
 		CheckHR(m_command_allocator_lists[safe_to_reset_command_allocator_list][i]->Reset());
 	}
 
+	m_command_allocator_list_fences[m_current_command_allocator_list] = m_queue_frame_fence_value;
 	m_current_command_allocator_list = (m_current_command_allocator_list + 1) % m_command_allocator_lists.size();
+	m_current_command_allocator = 0;
 	// End Command Allocator Resets
+}
 
-	m_queue_frame_fence_value++;
-#ifdef USE_D3D12_QUEUED_COMMAND_LISTS
-	m_queued_command_list->QueueFenceGpuSignal(m_queue_frame_fence, m_queue_frame_fence_value);
-#else
-	CheckHR(m_command_queue->Signal(m_queue_frame_fence, m_queue_frame_fence_value));
-#endif
+void D3DCommandListManager::MoveToNextCommandAllocator()
+{
+	// Move to the next allocator in the current allocator list.
+	m_current_command_allocator = (m_current_command_allocator + 1) % m_command_allocator_lists[m_current_command_allocator_list].size();
+
+	// Did we wrap around? Move to the next set of allocators.
+	if (m_current_command_allocator == 0)
+		PerformGpuRolloverChecks();
 }
 
-void D3DCommandListManager::ResetCommandListWithIdleCommandAllocator()
+void D3DCommandListManager::ResetCommandList()
 {
 #ifdef USE_D3D12_QUEUED_COMMAND_LISTS
 	ID3D12QueuedCommandList* command_list = m_queued_command_list;
@@ -268,6 +275,20 @@ void D3DCommandListManager::ImmediatelyDestroyAllResourcesScheduledForDestructio
 	}
 }
 
+void D3DCommandListManager::ResetAllCommandAllocators()
+{
+	for (auto& allocator_list : m_command_allocator_lists)
+	{
+		for (auto& allocator : allocator_list)
+			allocator->Reset();
+	}
+
+	// Move back to the start, using the first allocator of first list.
+	m_current_command_allocator = 0;
+	m_current_command_allocator_list = 0;
+	m_current_deferred_destruction_list = 0;
+}
+
 void D3DCommandListManager::ClearQueueAndWaitForCompletionOfInflightWork()
 {
 	// Wait for GPU to finish all outstanding work.
@@ -305,8 +326,10 @@ D3DCommandListManager::~D3DCommandListManager()
 
 void D3DCommandListManager::WaitOnCPUForFence(ID3D12Fence* fence, UINT64 fence_value)
 {
-	CheckHR(fence->SetEventOnCompletion(fence_value, m_wait_on_cpu_fence_event));
+	if (fence->GetCompletedValue() >= fence_value)
+		return;
 
+	CheckHR(fence->SetEventOnCompletion(fence_value, m_wait_on_cpu_fence_event));
 	WaitForSingleObject(m_wait_on_cpu_fence_event, INFINITE);
 }
 
diff --git a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h
index b9622df5fc..3f932dfc0e 100644
--- a/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h
+++ b/Source/Core/VideoBackends/D3D12/D3DCommandListManager.h
@@ -38,11 +38,10 @@ public:
 	void ExecuteQueuedWork(bool wait_for_gpu_completion = false);
 	void ExecuteQueuedWorkAndPresent(IDXGISwapChain* swap_chain, UINT sync_interval, UINT flags);
 
-	void WaitForQueuedWorkToBeExecutedOnGPU();
-
 	void ClearQueueAndWaitForCompletionOfInflightWork();
 	void DestroyResourceAfterCurrentCommandListExecuted(ID3D12Resource* resource);
 	void ImmediatelyDestroyAllResourcesScheduledForDestruction();
+	void ResetAllCommandAllocators();
 
 	void SetCommandListDirtyState(unsigned int command_list_state, bool dirty);
 	bool GetCommandListDirtyState(COMMAND_LIST_STATE command_list_state) const;
@@ -65,8 +64,10 @@ public:
 
 private:
 
+	void WaitForGPUCompletion();
 	void PerformGpuRolloverChecks();
-	void ResetCommandListWithIdleCommandAllocator();
+	void MoveToNextCommandAllocator();
+	void ResetCommandList();
 
 	unsigned int m_command_list_dirty_state = UINT_MAX;
 	D3D_PRIMITIVE_TOPOLOGY m_command_list_current_topology = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED;
@@ -85,6 +86,7 @@ private:
 	UINT m_current_command_allocator;
 	UINT m_current_command_allocator_list;
 	std::array<std::vector<ID3D12CommandAllocator*>, 2> m_command_allocator_lists;
+	std::array<UINT64, 2> m_command_allocator_list_fences;
 
 	ID3D12GraphicsCommandList* m_backing_command_list;
 	ID3D12QueuedCommandList* m_queued_command_list;
@@ -93,6 +95,7 @@ private:
 
 	UINT m_current_deferred_destruction_list;
 	std::array<std::vector<ID3D12Resource*>, 2> m_deferred_destruction_lists;
+	std::array<UINT64, 2> m_deferred_destruction_list_fences;
 };
 
 }  // namespace
\ No newline at end of file