retry tests which failed due to load

deckard.py

@@ -372,7 +372,7 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
         # leads the resolver to close the port and to the test failing in the
         # end. We partially detect these by checking the PCAP for ICMP packets.
         if check_for_icmp():
-            logging.error("Deckard is under load.\
+            raise DeckardUnderLoadError("Deckard is under load.\
 Other errors might be false negatives.\
 Consider retrying the job later.")
         raise ValueError('the scenario does not define all necessary answers (see error log)')

deckard_pytest.py

 import logging
 import os
 import subprocess
+import random
 import sys
+import time
 
 import pytest
 
@@ -33,13 +35,22 @@ logging.getLogger("augeas").setLevel(logging.ERROR)
 check_platform()
 
 
-def run_test(path, qmin, config):
+def run_test(path, qmin, config, retries=0):
     set_coverage_env(path, qmin)
     try:
         del os.environ["SOCKET_WRAPPER_DIR"]
     except KeyError:
         pass
-    deckard.process_file(path, qmin, config)
+    try:
+        deckard.process_file(path, qmin, config)
+    except deckard.DeckardUnderLoadError as e:
+        if retries < 3:
+            logging.error("Deckard under load. Retrying…")
+            # Exponential backoff
+            time.sleep((2 ** (retries + 1)) + (random.randint(0, 1000) / 1000))
+            run_test(path, qmin, config, retries + 1)
+        else:
+            raise e
 
 
 def test_passes_qmin_on(scenario):