From ef08e87da51f065b115e095700e8f28fcf11f7da Mon Sep 17 00:00:00 2001
From: ManoleAlexandru99 <manole.alexandru99@yahoo.ro>
Date: Mon, 10 Apr 2023 18:24:33 +0300
Subject: [PATCH] CudnnBatchNormBackward nan temporary fix #0012

:)
---
 train.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/train.py b/train.py
index 3aeacd05c8ff..e725543e70ec 100644
--- a/train.py
+++ b/train.py
@@ -338,7 +338,10 @@ def train(hyp, opt, device, callbacks):  # hyp is path/to/hyp.yaml or hyp dictio
                     loss *= 4.
 
             # Backward
-            scaler.scale(loss).backward(retain_graph=True)
+            try:
+                scaler.scale(loss).backward(retain_graph=True)
+            except Exception as e:
+                print('\n-------- FOUND EXCEPTION: ', e, ' Life goes on.-------\n')
 
             scaler_seg.scale(loss_seg).backward()