Fixed bug of first scanned frequency

.gitignore

@@ -1 +1,2 @@
 .venv/
+config.yaml

config.yaml

@@ -1,9 +1,9 @@
 # Configuration file for scanner service
 # it is automatically re-loaded between scans
-start_freq: 10000.0
-stop_freq:  0.1
+start_freq: 1000
+stop_freq:  0.2
 freq_step_multiply: 0.95
+allowed_noise_level: 9
+skip_scans_for_noisy_freqs: 5
 ampl: 1600
-interference_freq:      0.052
-interference_bandwidth: 0.008
 noise_scan: False

scanner.py

@@ -6,7 +6,7 @@ import re
 import telnetlib3
 import csv
 import time
-import pyodbc
+import pyodbc      # XXXDB
 import math
 from datetime import datetime
 import yaml
@@ -14,22 +14,23 @@ import os
 
 data_rows = []   # global 2-D list
 state = {}
+blacklist = []
+Inoise_baseline = 0.1
+n_clip_events = 0
 
 PATTERN = re.compile(
-    r"Va=(?P<Va>-?\d+\.?\d*)\s+"
-    r"Vp=(?P<Vp>-?\d+\.?\d*)\s+\|\s+"
-    r"Ia=(?P<Ia>-?\d+\.?\d*)\s+"
-    r"Ip=(?P<Ip>-?\d+\.?\d*)\s+\|\s+"
-    r"ZR=(?P<ZR>-?\d+\.?\d*)\s+"
-    r"ZX=(?P<ZX>-?\d+\.?\d*)"
-    r".*?irq=\d+\s+"              # Skip to 'irq=', match the first digits and space
-    r"(?P<adc_vmin>[0-9a-fA-F]+)-"      # First hex
-    r"(?P<adc_vmax>[0-9a-fA-F]+)\s+"    # Second hex
-    r"(?P<adc_imin>[0-9a-fA-F]+)-"      # Third hex
-    r"(?P<adc_imax>[0-9a-fA-F]+)"       # Fourth hex
+    r"Va=(?P<Va>[\d.-]+)\s+Vp=(?P<Vp>[\d.-]+)\s*\|\s*"
+    r"Ia=(?P<Ia>[\d.-]+)\s+Ip=(?P<Ip>[\d.-]+)\s*\|\s*"
+    r"ZR=(?P<ZR>[\d.-]+)\s+ZX=(?P<ZX>[\d.-]+).*?\|\s*"
+    r".*?irq=\w+\s+"
+    r"(?P<adc_vmin>[0-9a-fA-F]+)-(?P<adc_vmax>[0-9a-fA-F]+)\s+"
+    r"(?P<adc_imin>[0-9a-fA-F]+)-(?P<adc_imax>[0-9a-fA-F]+).*?\|\s*"
+    r"nv=(?P<nv>[\d.-]+)\s+ni=(?P<ni>[\d.-]+)\s*\|\s*"
+    r"QZR=(?P<QZR>[\d.-]+)\s+QZX=(?P<QZX>[\d.-]+)"
 )
 
-config_path = '/home/bart/python-scanner/config.yaml'
+config_path = '/home/bart/python-scanner/config.yaml'  # XXXDB
+#config_path = 'config.yaml'
 
 def load_yaml_config(state_dict):
     if os.path.exists(config_path):
@@ -63,12 +64,12 @@ def dump_into_database():
             if state["noise_scan"]==False:
                 # regular scan data
                 formatted_rows = [
-                    [sweep_insert_time, r[0], r[5], r[6], r[1], r[2], r[3], r[4]]
+                    [sweep_insert_time, r[0], r[13], r[14], r[1], r[2], r[3], r[4], r[11], r[12]]
                     for r in data_rows
                 ]
                 sql = """
-                      INSERT INTO SequenceValues (StartTimeOfSweep, Freq, ZR, ZX, Vampl, Vphase, Iampl, Iphase)
-                      VALUES (?, ?, ?, ?, ?, ?, ?, ?) \
+                      INSERT INTO SequenceValues (StartTimeOfSweep, Freq, ZR, ZX, Vampl, Vphase, Iampl, Iphase, Vnoise, Inoise)
+                      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \
                       """
             else:
                 # noise scan data
@@ -97,7 +98,7 @@ def dump_into_database():
 def dump_csv(filename="data.csv"):
     df = pd.DataFrame(
         data_rows,
-        columns=["Freq", "Va", "Vp", "Ia", "Ip", "ZR", "ZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax"]
+        columns=["Freq", "Va", "Vp", "Ia", "Ip", "QZR", "QZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax", "nv", "ni"]
     )
     df.to_csv(filename, index=False)
 
@@ -134,21 +135,23 @@ def extract_to_dataframe(line):
     if not match:
         return  # silently ignore malformed lines
     row = [
-        float(state["freq"]),
-        float(match.group("Va")),
-        float(match.group("Vp")),
-        float(match.group("Ia")),
-        float(match.group("Ip")),
-        float(match.group("ZR")),
-        float(match.group("ZX")),
-        int(match.group("adc_vmin"), 16),
-        int(match.group("adc_vmax"), 16),
-        int(match.group("adc_imin"), 16),
-        int(match.group("adc_imax"), 16),
+        float(state["freq"]),               # row 0
+        float(match.group("Va")),           # row 1
+        float(match.group("Vp")),           # row 2
+        float(match.group("Ia")),           # row 3
+        float(match.group("Ip")),           # row 4
+        float(match.group("ZR")),           # row 5
+        float(match.group("ZX")),           # row 6
+        int(match.group("adc_vmin"), 16),   # row 7
+        int(match.group("adc_vmax"), 16),   # row 8
+        int(match.group("adc_imin"), 16),   # row 9
+        int(match.group("adc_imax"), 16),   # row 10
+        float(match.group("nv")),           # row 11
+        float(match.group("ni")),           # row 12
+        float(match.group("QZR")),          # row 13
+        float(match.group("QZX")),          # row 14
     ]
-    row[1] = row[1]/(2*3.14159*row[0]*0.00005 + 1)      # compensate Va for pole at 20kHz
-    row[5] = row[1]/row[3] * math.cos(0.01745*(row[2]-row[4]))
-    row[6] = row[1]/row[3] * math.sin(0.01745*(row[2]-row[4]))
+#    print(row)	#### XX
     data_rows.append(row)
 
 def process_line(line):
@@ -160,7 +163,7 @@ def process_line(line):
 def get_dataframe():
     return pd.DataFrame(
         data_rows,
-        columns=["Va", "Vp", "Ia", "Ip", "ZR", "ZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax"]
+        columns=["Va", "Vp", "Ia", "Ip", "QZR", "QZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax", "nv", "ni"]
     )
 
 class TelnetReader:
@@ -186,62 +189,76 @@ class TelnetReader:
         # Main loop: reads incoming lines forever
         try:
             while state["freq"] > state["stop_freq"]:
-                line = self.tn.read_until(b"\n")        # read line
-                if not line:
-                    break
-
-                decoded = line.decode("utf-8", errors="ignore").strip()
-                self.process_line(decoded)              # extract all info from line
+                if state["initializing"]>0:
+                  self.process_line("Va=")
+                else:
+                  line = self.tn.read_until(b"\n")        # read line
+                  if not line:
+                      break
+                  decoded = line.decode("utf-8", errors="ignore").strip()
+                  self.process_line(decoded)              # extract all info from line
 
         except KeyboardInterrupt:
             print("Interrupted by user.")
         print(data_rows)
-        if state["freq"]>0.00001:                      # check if scan endeed normally (e.g. no abort due to clipping)
-            #dump_csv("measurements.csv")
-            dump_into_database()
-            #append_line_to_file(0, "scan_freq.csv")
-            #append_line_to_file(1, "scan_Va.csv")
-            #append_line_to_file(2, "scan_Vp.csv")
-            #append_line_to_file(3, "scan_Ia.csv")
-            #append_line_to_file(4, "scan_Ip.csv")
-            #append_line_to_file(5, "scan_ZR.csv")
-            #append_line_to_file(6, "scan_ZX.csv")
-            #append_Nyquist_run("scan_Nyquist.csv")
-            #state["noise_scan"] = not state["noise_scan"]
+        if state["freq"]>0.00001:                      # check if scan ended normally (e.g. no abort due to clipping)
+            if True:			# XXXDB
+                dump_into_database()
+            else:
+                dump_csv("measurements.csv")
+                #append_line_to_file(0, "scan_freq.csv")
+                #append_line_to_file(1, "scan_Va.csv")
+                #append_line_to_file(2, "scan_Vp.csv")
+                #append_line_to_file(3, "scan_Ia.csv")
+                #append_line_to_file(4, "scan_Ip.csv")
+                #append_line_to_file(5, "scan_ZR.csv")
+                #append_line_to_file(6, "scan_ZX.csv")
+                #append_Nyquist_run("scan_Nyquist.csv")
+                #state["noise_scan"] = not state["noise_scan"]
         else:
             print("scan aborted\r")
         gc.collect()    # clean up internal memory (garbage collect)
 
     def process_line(self, line):
         global state
-#        print(f"RAW: {line}")
+        global Inoise_baseline
+        global n_clip_events
+        print(line)
         if not line.startswith("Va="):                      # skip lines that are not to be analyzed
             return
-        if state["remaining_receive_lines"] > 0 :
-            state["remaining_receive_lines"] -= 1           # we are waiting for settling - just skip the line
-        else:
-            # prepare new frequency measurement
-            if state["initializing"] == 1:
-                # since we just start a frequency scan, let's set the R, Max and Amplitude
-                response = f"\rr516\r"                      # set Resistance value for scaling HAL sensor
-                self.tn.write(response.encode("utf-8"))
-                response = f"m1600\r"                        # set max amplitude value
-                self.tn.write(response.encode("utf-8"))
-                # for noise scans the amplitude is zero, else the state["ampl"]
-                if state["noise_scan"]==True:
-                    response = f"a0\r"                    # zero amplitude for noise measurement
-                else:
-                    response = f"a{state["ampl"]:.1f}\r"  # set amplitude
-                self.tn.write(response.encode("utf-8"))
-                # there will be a lot of lines, but they will be skipped as they do not match the pattern
-            else:   # regular loop (not initializing)
+        # prepare new frequency measurement
+        if state["initializing"] == 1:
+            # since we just start a frequency scan, let's set the R, Max and Amplitude
+            response = f"\rr516\r"                      # set Resistance value for scaling HAL sensor
+            self.tn.write(response.encode("utf-8"))
+            response = f"m1600\r"                       # set max amplitude value
+            self.tn.write(response.encode("utf-8"))
+            # for noise scans the amplitude is zero, else the state["ampl"]
+            if state["noise_scan"]==True:
+                response = f"a0\r"                    # zero amplitude for noise measurement
+            else:
+                response = f"a{state["ampl"]:.1f}\r"  # set amplitude
+            self.tn.write(response.encode("utf-8"))
+            # there will be a lot of lines, but they will be skipped as they do not match the pattern
+            state["initializing"] = 2
+        else:   # regular loop (initializing is 0 (normal) or 2 (first sample))
+            if state["initializing"] == 0:
                 extract_to_dataframe(line)  # capture the measurement
-#                print(f"minmax: {data_rows[7]},{data_rows[8]},{data_rows[9]},{data_rows[10] }\n")
-                if data_rows and (data_rows[-1][7]<5 or data_rows[-1][8]>250 or data_rows[-1][9]<5 or data_rows[-1][10]>250):
-                    #response = f"a{state["ampl"]:.1f}\r"  # send ampl to trigger sweep
-                    #self.tn.write(response.encode("utf-8"))
-                    # there will be a lot of lines, but they will be skipped as they do not match the pattern
-                    state["freq"] = 0       # force ending of the scan, and write no data in the database
+                if state["freq"]>3.0:
+                    Inoise_baseline = 0.8*Inoise_baseline + 0.2*data_rows[-1][12]    # remember last baseline around 3Hz (assuming top-down scanning)
+                if data_rows and (state["freq"]>10) and (data_rows[-1][11]>state["allowed_noise_level"] or data_rows[-1][12]>state["allowed_noise_level"]):  # or (state["freq"]<3.0 and data_rows[-1][12]>1.25*Inoise_baseline)):
+                    # Too much noise: add to blacklist
+                    print("Too much noise - adding to blacklist")
+                    blacklist.append({"Freq": state["freq"], "NrToSkip": state["skip_scans_for_noisy_freqs"]})
+                    print(f"Blacklist: {blacklist}\r")
+                    del data_rows[-1]       # remove this last entry from the list (for DB it is okay, but for CSV things will shift)
+                else:
+                    if data_rows and (data_rows[-1][7]<5 or data_rows[-1][8]>250 or data_rows[-1][9]<5 or data_rows[-1][10]>250):   # XXXDB
+                        if (++n_clip_events > 2):       # react only after multiple clip events (solves startup issue)
+                            if data_rows[-1][9]>0:      # make exception (for our broken hardware?)
+                                state["freq"] = 0       # force ending of the scan, and write no data in the database
+                    else:
+                        n_clip_events=0
                 # calculate next freq
                 if state["freq"] > 1.0:
                     freq_multiplier = state["freq_step_multiply"]     # calc next freq
@@ -251,26 +268,17 @@ class TelnetReader:
                     else:
                         freq_multiplier = state["freq_step_multiply"] ** 4  # skip 3/4 steps to speed up
                 state["freq"] *= freq_multiplier
-                if (state["freq"]>40) and (state["freq"]<660) and ((state["freq"]%50<2.5) or (-state["freq"]%50<2.5)):
-                    state["freq"] *= freq_multiplier              # if near a 50Hz harmonic, skip to the next frequency
-                while (state["freq"]<0.6) and ((state["freq"]%state["interference_freq"]<state["interference_bandwidth"]) or (-state["freq"]<state["interference_freq"]<state["interference_bandwidth"])):
-                    state["freq"] *= freq_multiplier              # if near a 0.052Hz harmonic, skip to the next frequency
+                while (any(item["Freq"] == state["freq"] for item in blacklist)):
+                    print("skipping freq")
+                    state["freq"] *= freq_multiplier              # skip all frequencies that are blacklisted
+
             if state["freq"] <= state["stop_freq"]:
                 print("Reached stop frequency")
             else:
-                if state["freq"] > 0.01:
-                    response = f"\rf{state["freq"]:.1f}\r"        # new freq
-                else:
-                    response = f"\rf{state["freq"]:.3f}\r"  # new freq
+                # program the health monitor to go to the next frequency:
+                response = f"\rQ{state["freq"]:.3f}\r"  # new freq
                 self.tn.write(response.encode("utf-8"))
-                bandwidth = state["freq"]/20
-                if bandwidth > 0.5 :
-                    bandwidth = 0.5
-                response = f"b{bandwidth:.3f}\r"        # new freq
-                self.tn.write(response.encode("utf-8"))
-                state["remaining_receive_lines"] = 1 + 4/bandwidth
-                print(line)
-                print(state)
+#                print(state)
             state["initializing"] = 0
 
         # Example:
@@ -288,7 +296,7 @@ class TelnetReader:
 if __name__ == "__main__":
     reader = TelnetReader(host="localhost", port=2002)
 #    reader = TelnetReader(host="10.1.122.152", port=2002)
-#    reader = TelnetReader(host="192.168.1.196", port=2002)
+#    reader = TelnetReader(host="192.168.1.235", port=2002)
 
     try:
         reader.connect()
@@ -298,10 +306,14 @@ if __name__ == "__main__":
             load_yaml_config(state)            # Load configuration settings
 
             state["freq"]                    = state["start_freq"];
-            state["remaining_receive_lines"] = 0;
             state["initializing"]            = 1;
 
             reader.read_loop()
+            # decrement all freq's in blacklist:
+            for item in blacklist:
+                item["NrToSkip"] -= 1
+            blacklist = [item for item in blacklist if item["NrToSkip"] != 0]  # remove all zero items
+            print(f"Blacklist: {blacklist}")
 
     finally:
         reader.disconnect()