Fixed bug of first scanned frequency

.forgejo/workflows/backup-repo.yaml

@@ -1,39 +0,0 @@
-name: Git Backup to WebDAV
-on:
-  push:
-    # This ensures it runs on every branch push
-    branches:
-      - '**'
-jobs:
-  print-content:
-    runs-on: debian-latest
-    steps:
-      - name: checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: archive repository
-        run: |
-          PROJECT_NAME=$(echo "${{ github.repository }}" | cut -d'/' -f2)
-          BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//-/g')
-          TIMESTAMP=$(date +'%Y-%m-%d_%H-%M')
-
-          # Create the local variable
-          FINAL_NAME="${PROJECT_NAME}_${BRANCH_NAME}_${TIMESTAMP}.tar.gz"
-
-          # CRITICAL: Save it for the next step
-          echo "PROJECTNAME=$PROJECT_NAME" >> $GITHUB_ENV
-          echo "FILENAME=$FINAL_NAME" >> $GITHUB_ENV
-
-          # Create the archive
-          git archive --format=tar.gz -v -o "$FINAL_NAME" HEAD
-          echo "Archive created: $FINAL_NAME"
-
-      - name: Upload via Curl
-        run: |
-          ls -lh "$FILENAME"
-
-          curl -T "$FILENAME" \
-          -u "${{ secrets.WEBDAV_USER }}:${{ secrets.WEBDAV_PASSWORD }}" \
-          "${{ secrets.WEBDAV_URL }}/$PROJECTNAME/$FILENAME"

.gitignore

@@ -1 +1,2 @@
 .venv/
+config.yaml

config.yaml

@@ -1,12 +1,9 @@
 # Configuration file for scanner service
 # it is automatically re-loaded between scans
-start_freq: 4000
-stop_freq:  0.4
+start_freq: 1000
+stop_freq:  0.2
 freq_step_multiply: 0.95
-allowed_noise_level: 15
-skip_scans_for_noisy_freqs: 3
+allowed_noise_level: 9
+skip_scans_for_noisy_freqs: 5
 ampl: 1600
 noise_scan: False
-hw_version: "2026-05-28"          # aangepaste R27 en R28 voor boven 700A
-scan_comment: "normale full scan met verbeterde PCB (nu goed boven 700A)"
-

scanner.py

@@ -6,19 +6,17 @@ import re
 import telnetlib3
 import csv
 import time
-import pyodbc
+import pyodbc      # XXXDB
 import math
 from datetime import datetime
 import yaml
 import os
 
-VersionScripSoftware = "2026-05-28"
-VersionDspSoftware = "Unknown"
-
 data_rows = []   # global 2-D list
 state = {}
 blacklist = []
 Inoise_baseline = 0.1
+n_clip_events = 0
 
 PATTERN = re.compile(
     r"Va=(?P<Va>[\d.-]+)\s+Vp=(?P<Vp>[\d.-]+)\s*\|\s*"
@@ -28,9 +26,11 @@ PATTERN = re.compile(
     r"(?P<adc_vmin>[0-9a-fA-F]+)-(?P<adc_vmax>[0-9a-fA-F]+)\s+"
     r"(?P<adc_imin>[0-9a-fA-F]+)-(?P<adc_imax>[0-9a-fA-F]+).*?\|\s*"
     r"nv=(?P<nv>[\d.-]+)\s+ni=(?P<ni>[\d.-]+)\s*\|\s*"
+    r"QZR=(?P<QZR>[\d.-]+)\s+QZX=(?P<QZX>[\d.-]+)"
 )
 
-config_path = '/home/bart/python-scanner/config.yaml'
+config_path = '/home/bart/python-scanner/config.yaml'  # XXXDB
+#config_path = 'config.yaml'
 
 def load_yaml_config(state_dict):
     if os.path.exists(config_path):
@@ -55,31 +55,16 @@ conn_str = (
 )
 
 def dump_into_database():
-    print("dump into database\n")
     try:
         with pyodbc.connect(conn_str) as conn:
             cursor = conn.cursor()
             sweep_insert_time = datetime.now()
-            # Add scan details
-            cursor.execute("""
-                IF NOT EXISTS (
-                    SELECT 1 FROM ScanDetails WHERE StartTimeOfSweep = ?
-                )
-                INSERT INTO ScanDetails
-                (StartTimeOfSweep, VersionScanScript, VersionDspSoftware, VersionHardware, Comment)
-                VALUES (?, ?, ?, ?, ?)
-            """, sweep_insert_time,
-                 sweep_insert_time, VersionScripSoftware, VersionDspSoftware, state["hw_version"], state["scan_comment"])
-            print("commit1\n")
-            conn.commit()
-
-        with pyodbc.connect(conn_str) as conn:
-            cursor = conn.cursor()
+            print("dump into database\n")
             # Prepare the data: SQL Server expects the columns in order.
             if state["noise_scan"]==False:
                 # regular scan data
                 formatted_rows = [
-                    [sweep_insert_time, r[0], r[5], r[6], r[1], r[2], r[3], r[4], r[11], r[12]]
+                    [sweep_insert_time, r[0], r[13], r[14], r[1], r[2], r[3], r[4], r[11], r[12]]
                     for r in data_rows
                 ]
                 sql = """
@@ -103,7 +88,7 @@ def dump_into_database():
             cursor.fast_executemany = False  # needs less RAM
             cursor.executemany(sql, formatted_rows)
 
-            print("commit2\n")
+            print("commit\n")
             conn.commit()
             print(f"Successfully inserted {len(formatted_rows)} rows.")
 
@@ -113,7 +98,7 @@ def dump_into_database():
 def dump_csv(filename="data.csv"):
     df = pd.DataFrame(
         data_rows,
-        columns=["Freq", "Va", "Vp", "Ia", "Ip", "ZR", "ZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax"]
+        columns=["Freq", "Va", "Vp", "Ia", "Ip", "QZR", "QZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax", "nv", "ni"]
     )
     df.to_csv(filename, index=False)
 
@@ -163,10 +148,10 @@ def extract_to_dataframe(line):
         int(match.group("adc_imax"), 16),   # row 10
         float(match.group("nv")),           # row 11
         float(match.group("ni")),           # row 12
+        float(match.group("QZR")),          # row 13
+        float(match.group("QZX")),          # row 14
     ]
-    row[1] = row[1]/(2*3.14159*row[0]*0.00005 + 1)      # compensate Va for pole at 20kHz
-    row[5] = row[1]/row[3] * math.cos(0.01745*(row[2]-row[4]))
-    row[6] = row[1]/row[3] * math.sin(0.01745*(row[2]-row[4]))
+#    print(row)	#### XX
     data_rows.append(row)
 
 def process_line(line):
@@ -178,7 +163,7 @@ def process_line(line):
 def get_dataframe():
     return pd.DataFrame(
         data_rows,
-        columns=["Va", "Vp", "Ia", "Ip", "ZR", "ZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax"]
+        columns=["Va", "Vp", "Ia", "Ip", "QZR", "QZX", "adc_vmin", "adc_vmax", "adc_imin", "adc_imax", "nv", "ni"]
     )
 
 class TelnetReader:
@@ -204,12 +189,14 @@ class TelnetReader:
         # Main loop: reads incoming lines forever
         try:
             while state["freq"] > state["stop_freq"]:
-                line = self.tn.read_until(b"\n")        # read line
-                if not line:
-                    break
-
-                decoded = line.decode("utf-8", errors="ignore").strip()
-                self.process_line(decoded)              # extract all info from line
+                if state["initializing"]>0:
+                  self.process_line("Va=")
+                else:
+                  line = self.tn.read_until(b"\n")        # read line
+                  if not line:
+                      break
+                  decoded = line.decode("utf-8", errors="ignore").strip()
+                  self.process_line(decoded)              # extract all info from line
 
         except KeyboardInterrupt:
             print("Interrupted by user.")
@@ -235,34 +222,31 @@ class TelnetReader:
     def process_line(self, line):
         global state
         global Inoise_baseline
-        global VersionDspSoftware
-        if line.startswith("SW-Version"):
-            VersionDspSoftware = line.split(":", 1)[1].strip()    # remember the DSP software version from its reporting
-            print(f"DSP SW Version={VersionDspSoftware}\r\n")
+        global n_clip_events
+        print(line)
         if not line.startswith("Va="):                      # skip lines that are not to be analyzed
             return
-        if state["remaining_receive_lines"] > 0 :
-            state["remaining_receive_lines"] -= 1           # we are waiting for settling - just skip the line
-        else:
-            # prepare new frequency measurement
-            if state["initializing"] == 1:
-                # since we just start a frequency scan, let's set the R, Max and Amplitude
-                response = f"\rr331\r"                      # set Resistance value for scaling HAL sensor
-                self.tn.write(response.encode("utf-8"))
-                response = f"m1600\r"                        # set max amplitude value
-                self.tn.write(response.encode("utf-8"))
-                # for noise scans the amplitude is zero, else the state["ampl"]
-                if state["noise_scan"]==True:
-                    response = f"a0\r"                    # zero amplitude for noise measurement
-                else:
-                    response = f"a{state["ampl"]:.1f}\r"  # set amplitude
-                self.tn.write(response.encode("utf-8"))
-                # there will be a lot of lines, but they will be skipped as they do not match the pattern
-            else:   # regular loop (not initializing)
+        # prepare new frequency measurement
+        if state["initializing"] == 1:
+            # since we just start a frequency scan, let's set the R, Max and Amplitude
+            response = f"\rr516\r"                      # set Resistance value for scaling HAL sensor
+            self.tn.write(response.encode("utf-8"))
+            response = f"m1600\r"                       # set max amplitude value
+            self.tn.write(response.encode("utf-8"))
+            # for noise scans the amplitude is zero, else the state["ampl"]
+            if state["noise_scan"]==True:
+                response = f"a0\r"                    # zero amplitude for noise measurement
+            else:
+                response = f"a{state["ampl"]:.1f}\r"  # set amplitude
+            self.tn.write(response.encode("utf-8"))
+            # there will be a lot of lines, but they will be skipped as they do not match the pattern
+            state["initializing"] = 2
+        else:   # regular loop (initializing is 0 (normal) or 2 (first sample))
+            if state["initializing"] == 0:
                 extract_to_dataframe(line)  # capture the measurement
                 if state["freq"]>3.0:
                     Inoise_baseline = 0.8*Inoise_baseline + 0.2*data_rows[-1][12]    # remember last baseline around 3Hz (assuming top-down scanning)
-                if data_rows and (data_rows[-1][11]>state["allowed_noise_level"] or data_rows[-1][12]>state["allowed_noise_level"] or (state["freq"]<3.0 and data_rows[-1][12]>1.25*Inoise_baseline)):
+                if data_rows and (state["freq"]>10) and (data_rows[-1][11]>state["allowed_noise_level"] or data_rows[-1][12]>state["allowed_noise_level"]):  # or (state["freq"]<3.0 and data_rows[-1][12]>1.25*Inoise_baseline)):
                     # Too much noise: add to blacklist
                     print("Too much noise - adding to blacklist")
                     blacklist.append({"Freq": state["freq"], "NrToSkip": state["skip_scans_for_noisy_freqs"]})
@@ -270,7 +254,11 @@ class TelnetReader:
                     del data_rows[-1]       # remove this last entry from the list (for DB it is okay, but for CSV things will shift)
                 else:
                     if data_rows and (data_rows[-1][7]<5 or data_rows[-1][8]>250 or data_rows[-1][9]<5 or data_rows[-1][10]>250):   # XXXDB
-                        state["freq"] = 0       # force ending of the scan, and write no data in the database
+                        if (++n_clip_events > 2):       # react only after multiple clip events (solves startup issue)
+                            if data_rows[-1][9]>0:      # make exception (for our broken hardware?)
+                                state["freq"] = 0       # force ending of the scan, and write no data in the database
+                    else:
+                        n_clip_events=0
                 # calculate next freq
                 if state["freq"] > 1.0:
                     freq_multiplier = state["freq_step_multiply"]     # calc next freq
@@ -280,26 +268,17 @@ class TelnetReader:
                     else:
                         freq_multiplier = state["freq_step_multiply"] ** 4  # skip 3/4 steps to speed up
                 state["freq"] *= freq_multiplier
-                if (state["freq"]>40) and (state["freq"]<660) and ((state["freq"]%50<2.5) or (-state["freq"]%50<2.5)):
-                    state["freq"] *= freq_multiplier              # if near a 50Hz harmonic, skip to the next frequency
-#                while (state["freq"]<0.6) and ((state["freq"]%state["interference_freq"]<state["interference_bandwidth"]) or (-state["freq"]<state["interference_freq"]<state["interference_bandwidth"])):
-#                    state["freq"] *= freq_multiplier              # if near a 0.052Hz harmonic, skip to the next frequency
+                while (any(item["Freq"] == state["freq"] for item in blacklist)):
+                    print("skipping freq")
+                    state["freq"] *= freq_multiplier              # skip all frequencies that are blacklisted
+
             if state["freq"] <= state["stop_freq"]:
                 print("Reached stop frequency")
             else:
-                if state["freq"] > 0.01:
-                    response = f"\rf{state["freq"]:.1f}\r"        # new freq
-                else:
-                    response = f"\rf{state["freq"]:.3f}\r"  # new freq
+                # program the health monitor to go to the next frequency:
+                response = f"\rQ{state["freq"]:.3f}\r"  # new freq
                 self.tn.write(response.encode("utf-8"))
-                bandwidth = state["freq"]/20
-                if bandwidth > 0.5 :
-                    bandwidth = 0.5
-                response = f"b{bandwidth:.3f}\r"        # new freq
-                self.tn.write(response.encode("utf-8"))
-                state["remaining_receive_lines"] = 1 + 4/bandwidth
-                print(line)
-                print(state)
+#                print(state)
             state["initializing"] = 0
 
         # Example:
@@ -317,7 +296,7 @@ class TelnetReader:
 if __name__ == "__main__":
     reader = TelnetReader(host="localhost", port=2002)
 #    reader = TelnetReader(host="10.1.122.152", port=2002)
-#    reader = TelnetReader(host="192.168.1.196", port=2002)
+#    reader = TelnetReader(host="192.168.1.235", port=2002)
 
     try:
         reader.connect()
@@ -327,7 +306,6 @@ if __name__ == "__main__":
             load_yaml_config(state)            # Load configuration settings
 
             state["freq"]                    = state["start_freq"];
-            state["remaining_receive_lines"] = 0;
             state["initializing"]            = 1;
 
             reader.read_loop()

start

@@ -1,3 +1 @@
-sudo systemctl restart ser2net.service
-sleep 2
 systemctl --user start python-scanner