fix(client.command.cryo_card): Catch EPICS timeout on read and retry.

python/pysmurf/client/base/base_class.py

@@ -181,7 +181,8 @@ def __init__(self, log=None, epics_root=None, offline=False,
         self.timing_status = self.amctiming + 'TimingFrameRx:'
 
         self.C = CryoCard(self.rtm_spi_cryo_root + 'read',
-                          self.rtm_spi_cryo_root + 'write')
+                          self.rtm_spi_cryo_root + 'write',
+                          log=self.log)
         self.freq_resp = {}
 
         # RTM slow DAC parameters (used, e.g., for TES biasing). The

python/pysmurf/client/command/cryo_card.py

@@ -17,6 +17,8 @@
 import time
 import os
 
+from ..base.logger import SmurfLogger
+
 try:
     import epics
 except ModuleNotFoundError:
@@ -30,7 +32,7 @@ def write_csv(filename, header, line):
         f.write(line+'\n')
 
 class CryoCard():
-    def __init__(self, readpv_in, writepv_in):
+    def __init__(self, readpv_in, writepv_in, log=None):
         """
         Interact with the cryocard via the PIC. To interact via the RTM, use SmurfCommandMixin.
         Needs to be compatible with the C02 and C04 cryocards.
@@ -54,12 +56,16 @@ def __init__(self, readpv_in, writepv_in):
         self.temperature_offset =.25
         self.bias_scale = 1.0
         self.max_retries = 5 #number of re-tries waiting for response
+        self.timeout = 5  # timeout in between retries
         self.retry = 0 # counts nubmer of retries
         self.busy_retry = 0  # counts number of retries due to relay busy status
         self.list_of_c02_amps = ['50k', 'hemt']
         self.list_of_c04_amps = ['50k1', '50k2', 'hemt1', 'hemt2']
         self.list_of_c02_and_c04_amps = self.list_of_c02_amps + self.list_of_c04_amps
 
+        # basic logging capacity
+        self.log = SmurfLogger() if log is None else log
+
     def do_read(self, address, use_monitor=False):
         r"""Writes query to cryostat card PIC and reads reply.
 
@@ -83,13 +89,18 @@ def do_read(self, address, use_monitor=False):
         self.writepv.put(cmd_make(1, address, 0))
         for self.retry in range(0, self.max_retries):
             self.writepv.put(cmd_make(1, address, 0))
-            data = self.readpv.get(use_monitor=use_monitor)
-            addrrb = cmd_address(data)
-            if (addrrb == address):
-                return (data)
-        return (0)
-
-        return (self.readpv.get(use_monitor=use_monitor))
+            data = self.readpv.get(use_monitor=use_monitor, timeout=self.timeout)
+            if data is None:
+                self.log(f"CryoCard.do_read: EPICS PV timed out after {self.timeout}s.")
+            else:
+                addrrb = cmd_address(data)
+                if (addrrb == address):
+                    return (data)
+            self.log(
+                f"CryoCard.do_read failed, retry {self.retry + 1} / {self.max_retries}."
+            )
+
+        raise Exception(f"Failed to read address 0x{address:X} after {self.max_retries} attempts.")
 
     def do_write(self, address, value):
         """Write the given value directly to the address on the PIC. Make sure