diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50d714c..40e47a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,14 +54,20 @@ jobs: if: runner.os == 'Windows' run: >- python -m PyInstaller --noconfirm --clean --windowed --onefile - --name QAFIT-windows --icon dataFiles/qcfit-icon.ico + --name QAFIT-windows --icon dataFiles/qafit-icon.ico --add-data "dataFiles;dataFiles" qafit_script.py + - name: Generate macOS application icon + if: runner.os == 'macOS' + run: >- + python -c "from PIL import Image; image = Image.open('dataFiles/qafit-icon.ico'); image.save('qafit.icns', format='ICNS')" + - name: Build macOS application if: runner.os == 'macOS' run: >- python -m PyInstaller --noconfirm --clean --windowed --onedir - --name QAFIT-macos --add-data "dataFiles:dataFiles" qafit_script.py + --name QAFIT-macos --icon qafit.icns + --add-data "dataFiles:dataFiles" qafit_script.py - name: Package macOS application if: runner.os == 'macOS' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..b3eb55f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,31 @@ +name: Run Python tests + +on: + push: + branches: + - "**" + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + pytest: + name: pytest (Python 3.10) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install test dependencies + run: python -m pip install numpy==1.24.4 pandas==1.5.3 Pillow==9.5.0 "pytest>=8,<10" + + - name: Run tests + run: python -m pytest tests -v diff --git a/SERI-QC_with_QAFIT/README.md b/SERI-QC_with_QAFIT/README.md index c9d3a44..5eb9c25 100644 --- a/SERI-QC_with_QAFIT/README.md +++ b/SERI-QC_with_QAFIT/README.md @@ -16,6 +16,36 @@ Advanced features: - displays the points those fails by 3-component filtering - data point list saved as *.csv and *.txt +(As of Oct 7 2026, QAFIT upgraded from QCFIT) +- wider Ktmax and Knmax readouts in all three main-window panels accommodate three-digit values +- the main window title shows the selected station's code and description +- new QA0 files use aligned air-mass headers and `Kn-Kt S-PP S-PP;` groups, with two-digit Kn and three-digit Kt fields; unconfigured months retain `00-000 0-00 0-00;` defaults +- Save writes the same monthly row format; existing QA0 files without a final semicolon remain readable +- malformed QA0 files show a `QA0 File Error` dialog with the file path and the invalid field or boundary row; opening and saving stop without overwriting the file +- boundary-value errors identify the month, air-mass range, field, and offending value; separator errors are reported separately +- Help -> About displays the entire disclaimer image, resized proportionally to fit its panel when the window is resized +- QAFIT excludes points with unrounded Kn <= 0.01 (1%) from all K-space plots and automatic curve fitting, in every air-mass range and plane; the Y-axis still starts at zero +- QAFIT filters input immediately after SZA calculation: only rows with calculated SZA < 80 degrees and positive extraterrestrial irradiance proceed to K-space processing, fitting, and graphing +- curve positions are one-based: position 1 has no horizontal shift, and each subsequent position adds 2.5 percentage points; plotted boundaries and Configure adjustments use the same shift as automatic fitting +- fitting area tables include all 8 left curves and 7 right curves at positions 1 through 30; both initial fitting and Configure refitting look up areas by curve and position + +Application icons: + +QAFIT uses Tk photo icons generated from `dataFiles/qafit-icon.ico` for its windows, including dialogs and plot windows. Native window-icon visibility depends on the desktop environment; macOS does not show ordinary title-bar icons. + +The macOS build workflow converts the same artwork to ICNS and embeds it in the `.app` bundle for Finder and Dock. Rebuild and launch the packaged app to see the new bundle icon; running the Python script directly does not set the Finder icon of the Python executable. The Windows build embeds the ICO in the executable. + +Testing: + +From the repository root, using the application's Python environment: + +```bash +python -m pip install "pytest>=8,<10" +python -m pytest tests -v +``` + +The GitHub Actions `Run Python tests` workflow runs the tests on pushes, pull requests, and manual dispatches using Python 3.10. QA0 samples are embedded in the tests; external data files and GUI access are not required. + About SERI-QC: SERI-QC is a mathematical software package that assesses the quality of solar radiation data. Flags are set to inform the user of any departure of the data from expected values. These flags indicate the magnitude and direction of such departures. Flags only indicate that data do or do not fall within expected ranges. This does not mean that the data are or are not valid. Check Maxwell, 1993 for description of flags. diff --git a/SERI-QC_with_QAFIT/curveArea.py b/SERI-QC_with_QAFIT/curveArea.py index f961130..1ee08f0 100644 --- a/SERI-QC_with_QAFIT/curveArea.py +++ b/SERI-QC_with_QAFIT/curveArea.py @@ -18,17 +18,15 @@ def getArea(shape): y = [int(i) for i in y] if shape == "left": curve = ut.curveLeft - end = 7 elif shape == "right": curve = ut.curveRight - end = 6 else: print("Shape should be or ") return cols = {"y": y} - for i in range(1, end): - for j in range(1, 21): + for i in range(1, len(curve) + 1): + for j in range(1, 31): val = [] toAdd = 0 shift = (j - 1) * 2.5 @@ -210,6 +208,5 @@ def getArea(shape): - diff --git a/SERI-QC_with_QAFIT/dataFiles/blankQA0 b/SERI-QC_with_QAFIT/dataFiles/blankQA0 index f74745b..ad7aae7 100644 --- a/SERI-QC_with_QAFIT/dataFiles/blankQA0 +++ b/SERI-QC_with_QAFIT/dataFiles/blankQA0 @@ -4,22 +4,21 @@ --- Elevation: --- Time Zone: - ---Low Airmass-- ---Med Airmass-- --High Airmass-- - -MAX- -LFT RGT- -MAX- -LFT RGT- -MAX- -LFT RGT- - KN KT S P S P KN KT S P S P KN KT S P S P - -JAN: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -FEB: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -MAR: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -APR: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -MAY: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -JUN: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -JUL: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -AUG: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -SEP: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -OCT: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -NOV: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 -DEC: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00 + ---Low Airmass--- ---Med Airmass--- ---High Airmass-- + -Max- -Lft Rgt- -Max- -Lft Rgt- -Max- -Lft Rgt- + Kn Kt S P S P Kn Kt S P S P Kn Kt S P S P +JAN: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +FEB: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +MAR: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +APR: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +MAY: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +JUN: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +JUL: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +AUG: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +SEP: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +OCT: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +NOV: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +DEC: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; DEFAULT CONFIGURATION Integration (minutes): diff --git a/SERI-QC_with_QAFIT/dataFiles/disclaimer_qafit.png b/SERI-QC_with_QAFIT/dataFiles/disclaimer_qafit.png new file mode 100644 index 0000000..491e177 Binary files /dev/null and b/SERI-QC_with_QAFIT/dataFiles/disclaimer_qafit.png differ diff --git a/SERI-QC_with_QAFIT/dataFiles/disclaimer.png b/SERI-QC_with_QAFIT/dataFiles/disclaimer_qcfit.png similarity index 100% rename from SERI-QC_with_QAFIT/dataFiles/disclaimer.png rename to SERI-QC_with_QAFIT/dataFiles/disclaimer_qcfit.png diff --git a/SERI-QC_with_QAFIT/dataFiles/qafit-icon.ico b/SERI-QC_with_QAFIT/dataFiles/qafit-icon.ico new file mode 100644 index 0000000..7cd1138 Binary files /dev/null and b/SERI-QC_with_QAFIT/dataFiles/qafit-icon.ico differ diff --git a/SERI-QC_with_QAFIT/qafit_script.py b/SERI-QC_with_QAFIT/qafit_script.py index 3a8d459..6fb66e0 100644 --- a/SERI-QC_with_QAFIT/qafit_script.py +++ b/SERI-QC_with_QAFIT/qafit_script.py @@ -37,6 +37,16 @@ import math +def setWindowIcon(window): + with Image.open(resource_path('dataFiles/qafit-icon.ico')) as source: + icons = [ + ImageTk.PhotoImage(source.resize((size, size), Image.Resampling.LANCZOS), master=window) + for size in (32, 128) + ] + window.iconphoto(False, *icons) + window._qafitIconImages = icons + + def getFilePath(): global data root = tk.Tk() @@ -126,11 +136,10 @@ def getKspace(ip): ip.loc[(ip['goAhead'] == 1.0) & (ip['KT1'] < -10000.0), 'KT1'] = -10000.0 ip.loc[(ip['goAhead'] == 1.0) & (ip['KN1'] < -10000.0), 'KN1'] = -10000.0 - ip.loc[(ip['goAhead'] == 1.0) & (ip['XT'] < miss), 'KT'] = (ip['KT']).map(int) - ip.loc[(ip['goAhead'] == 1.0) & (ip['XD'] < miss), 'KD'] = (ip['KD']).map(int) - ip.loc[(ip['goAhead'] == 1.0) & (ip['XN'] < miss), 'KN'] = (ip['KN']).map(int) - ip.loc[(ip['goAhead'] == 1.0) & (ip['KT1'] < miss), 'KT1'] = (ip['KT1']).map(int) - ip.loc[(ip['goAhead'] == 1.0) & (ip['KN1'] < miss), 'KN1'] = (ip['KN1']).map(int) + for column, source in (('KT', 'XT'), ('KD', 'XD'), ('KN', 'XN'), + ('KT1', 'KT1'), ('KN1', 'KN1')): + mask = (ip['goAhead'] == 1.0) & (ip[source] < miss) + ip[column] = ip[column].map(int).where(mask, ip[column]) def hitOpen(): @@ -345,7 +354,7 @@ def nsCancel(): nsOKButton.place(relx=0.50, rely=0.90, relwidth=0.20, relheight=0.05) nsCancelButton = tk.Button(nsFrame, text="Cancel", command=nsCancel) nsCancelButton.place(relx=0.75, rely=0.90, relwidth=0.20, relheight=0.05) - newSiteRoot.iconbitmap(resource_path('dataFiles/qcfit-icon.ico')) + setWindowIcon(newSiteRoot) newSiteRoot.protocol("WM_DELETE_WINDOW", nsCancel) newSiteRoot.mainloop() @@ -354,8 +363,12 @@ def browseQA0(): global data, boundary, siteInfo qa0Path = getQA0FilePath() if len(qa0Path) > 1: + try: + existQA0Data = utills.readQA0(qa0Path) + except (utills.QA0FormatError, OSError, UnicodeError) as exc: + messagebox.showerror("QA0 File Error", f"Cannot open {qa0Path}:\n{exc}") + return data["qa0Path"] = qa0Path - existQA0Data = utills.readFile(qa0Path) index = existQA0Data.find("Site Identifier:") string = existQA0Data[index + len("Site Identifier:"):] siteID = string.split("\n") @@ -429,6 +442,11 @@ def browseQCF(): def okOS(): global ip, data, plotData, pdLow, pdMed, pdHigh, boundary, iterCount + try: + qa0dat = utills.readQA0(data["qa0Path"]) + except (utills.QA0FormatError, OSError, UnicodeError) as exc: + messagebox.showerror("QA0 File Error", f"Cannot open {data['qa0Path']}:\n{exc}") + return if iterCount > 0: iterCount = 0 answer = messagebox.askyesno("Save Changes", "Do you want to save changes?") @@ -437,7 +455,6 @@ def okOS(): save(monthToSave) resetStats() # read QA0 file - qa0dat = utills.readFile(data["qa0Path"]) boundary = utills.getBoundariesTable(qa0dat) temp = boundary data["integration"] = int(integVar.get()) @@ -460,6 +477,7 @@ def okOS(): script.calculateZenith(ip, repFreq, measFreq, avg, data["latitude"], data["longitude"], data["elevation"], pressure, temperature) + ip = ip[(ip["solzen"] < 80) & (ip["ETR"] > 0)].copy() getKspace(ip) script.calculateAirmass(ip) @@ -576,7 +594,7 @@ def cancelOS(): cancelOSButton.place(relx=0.77, rely=0.79, relwidth=0.20, relheight=0.1) openRoot.protocol("WM_DELETE_WINDOW", mainRoot.deiconify) - openRoot.iconbitmap(resource_path('dataFiles/qcfit-icon.ico')) + setWindowIcon(openRoot) openRoot.mainloop() @@ -585,7 +603,11 @@ def updateSiteInfo(*args): if optVar.get() == "select a site": return else: - existQA0Data = utills.readFile(siteInfo[optVar.get()]) + try: + existQA0Data = utills.readQA0(siteInfo[optVar.get()]) + except (utills.QA0FormatError, OSError, UnicodeError) as exc: + messagebox.showerror("QA0 File Error", f"Cannot open {siteInfo[optVar.get()]}:\n{exc}") + return latitude = utills.pickValue(existQA0Data, 'Latitude:') longitude = utills.pickValue(existQA0Data, 'Longitude:') elevation = float(utills.pickValue(existQA0Data, 'Elevation:')) @@ -620,8 +642,13 @@ def hitEdit(): save(monthToSave) qa0Path = getQA0FilePath() - if len(qa0Path) > 1: - QA0Data = utills.readFile(qa0Path) + if len(qa0Path) <= 1: + return + try: + QA0Data = utills.readQA0(qa0Path) + except (utills.QA0FormatError, OSError, UnicodeError) as exc: + messagebox.showerror("QA0 File Error", f"Cannot open {qa0Path}:\n{exc}") + return index = QA0Data.find("Site Identifier:") string = QA0Data[index + len("Site Identifier:"):] @@ -629,7 +656,7 @@ def hitEdit(): siteinf = siteinf[0].split(",") siteid = siteinf[0] - siteDesc = siteinf[1] + siteDesc = ",".join(siteinf[1:]) latitude = utills.pickValue(QA0Data, 'Latitude:') longitude = utills.pickValue(QA0Data, 'Longitude:') elevation = float(utills.pickValue(QA0Data, 'Elevation:')) @@ -835,7 +862,7 @@ def editQA0Cancel(): editQA0OKButton.place(relx=0.50, rely=0.90, relwidth=0.20, relheight=0.05) editQA0CancelButton = tk.Button(editQA0Frame, text="Cancel", command=editQA0Cancel) editQA0CancelButton.place(relx=0.75, rely=0.90, relwidth=0.20, relheight=0.05) - editQA0Root.iconbitmap(resource_path('dataFiles/qcfit-icon.ico')) + setWindowIcon(editQA0Root) editQA0Root.protocol("WM_DELETE_WINDOW", editQA0Cancel) editQA0Root.mainloop() @@ -880,13 +907,11 @@ def curvefitting(): ip.loc[(ip["KT"] > ip["KN1"]), "activeKtKd"] = 1 ip.loc[(ip["KT1"] > ip["KN"]), "activeKnKd"] = 1 - # ignore negative and values more than 100 - ip.loc[(ip["KT"] > 100), "activeKtKn"] = 0 - ip.loc[(ip["KT"] > 100), "activeKtKd"] = 0 - ip.loc[(ip["KT"] > 100), "activeKnKd"] = 0 - ip.loc[(ip["KN"] > 100), "activeKtKn"] = 0 - ip.loc[(ip["KN"] > 100), "activeKtKd"] = 0 - ip.loc[(ip["KN"] > 100), "activeKnKd"] = 0 + # ignore negative values (allow KT up to 130 and KN up to 99) + ip.loc[(ip["KT"] > 130), "activeKtKn"] = 0 + ip.loc[(ip["KT"] > 130), "activeKtKd"] = 0 + ip.loc[(ip["KN"] > 99), "activeKtKn"] = 0 + ip.loc[(ip["KN"] > 99), "activeKnKd"] = 0 ip.loc[(ip["KT"] < 0), "activeKtKn"] = 0 ip.loc[(ip["KT"] < 0), "activeKtKd"] = 0 ip.loc[(ip["KT"] < 0), "activeKnKd"] = 0 @@ -899,7 +924,8 @@ def curvefitting(): ip["bin"] = "" ip["residual"] = ip["XT"] - ip["XN"] - ip["XD"] - plotData = ip[ip["monthName"] == monVar.get()].copy() + # Use the unrounded Kn ratio; KN is truncated to integer percent. + plotData = ip[(ip["monthName"] == monVar.get()) & (ip["XN"] > 0.01)].copy() # seperate data based on airmasses @@ -933,6 +959,14 @@ def curvefitting(): for am, am_acro in zip(amass, amass_acro): data[am + "AM"]["knMax"] = int(list(month_content[am_acro + "_MAX_KN"])[0]) data[am + "AM"]["ktMax"] = int(list(month_content[am_acro + "_MAX_KT"])[0]) + # Store original values for reset functionality + data[am + "AM"]["knMax_original"] = int(list(month_content[am_acro + "_MAX_KN"])[0]) + data[am + "AM"]["ktMax_original"] = int(list(month_content[am_acro + "_MAX_KT"])[0]) + # Also store original shape and position values + data[am + "AM"]["shapeLeft_original"] = None + data[am + "AM"]["posLeft_original"] = None + data[am + "AM"]["shapeRight_original"] = None + data[am + "AM"]["posRight_original"] = None for df, am in zip([pdLow, pdMed, pdHigh], amass): # reduce maximus according to the airmass @@ -947,7 +981,7 @@ def curvefitting(): tempList = list(set(df[k])) if len(tempList) > 0: - tempList = [i for i in tempList if 0 < i < 100] + tempList = [i for i in tempList if 0 < i < 1000] maxVal = int(max(tempList)) else: maxVal = "NA" @@ -959,6 +993,10 @@ def curvefitting(): # now update the data dictionary data[am + "AM"]["ktMax"] = ktMax data[am + "AM"]["knMax"] = knMax + + # UPDATE original values when QAFIT calculates them (was zero before) + data[am + "AM"]["ktMax_original"] = ktMax + data[am + "AM"]["knMax_original"] = knMax def getBoundaries(): @@ -1090,8 +1128,8 @@ def getBoundaries(): scoreL = 1000 scoreR = 1000 - for curve in range(0, 6): - for position in range(1, 21): + for curve in range(0, 8): + for position in range(1, 31): shift = ((position - 1) * 2.5) dataCountL = 0 dataCountR = 0 @@ -1102,7 +1140,7 @@ def getBoundaries(): pointsL = zeroesL + pointsL[len(zeroesL):] # Find the right curve with shift - if curve < 5: + if curve < 7: pointsR = ut.curveRight[curve] pointsR = [(point) + shift for point in pointsR] @@ -1121,7 +1159,7 @@ def getBoundaries(): """if curve == 1: if position == 14: print("")""" - if curve < 5: + if curve < 7: activeCountR = list(df[df["KN"] == y]["KT"]) activeCountR = len([x for x in activeCountR if x > (pointsR[y])]) """# for debugging purpose @@ -1136,17 +1174,17 @@ def getBoundaries(): pixCountL = np.sum(abs(np.array(pointsL[:knMax + 1]) - np.array(activeLBound))) pcL[dim, curve, position - 1] = pixCountL - pixCountL = pixCountL / (areaLeft.iloc[int(knMax) - 1][(curve * 20) + position]) + pixCountL = pixCountL / areaLeft.loc[int(knMax) - 1, f"{curve + 1}-{position}"] dcL[dim, curve, position - 1] = dataCountL lcount = dataCountL dataCountL = dataCountL / len(df) scoreNowL = pixCountL + dataCountL - if curve < 5: + if curve < 7: pixCountR = np.sum(abs(np.array(pointsR[:knMax + 1]) - np.array(activeRBound))) pcR[dim, curve, position - 1] = pixCountR - pixCountR = pixCountR / (areaRight.iloc[int(knMax) - 1][(curve * 20) + position]) + pixCountR = pixCountR / areaRight.loc[int(knMax) - 1, f"{curve + 1}-{position}"] dcR[dim, curve, position - 1] = dataCountR rcount = dataCountR dataCountR = dataCountR / len(df) @@ -1159,7 +1197,7 @@ def getBoundaries(): shapeL = curve + 1 positionL = position outL = lcount - if curve < 5: + if curve < 7: if scoreNowR < scoreR: scoreR = scoreNowR shapeR = curve + 1 @@ -1236,7 +1274,7 @@ def getBoundaries(): leftB = list(ut.curveLeft[shapeL - 1]) leftB = leftB[0:int(knMax)] # get all the points of curve upto Ymax - leftB = [(i + (2.5 * positionL)) for i in leftB] + leftB = [(i + (2.5 * (positionL - 1))) for i in leftB] if leftB[-1] != int(ktMax): leftB.append(int(ktMax)) @@ -1252,7 +1290,7 @@ def getBoundaries(): rightB = list(ut.curveRight[shapeR - 1]) rightB = rightB[0:int(knMax)] # get all the points of curve upto Ymax - rightB = [(i + (2.5 * positionR)) for i in rightB] + rightB = [(i + (2.5 * (positionR - 1))) for i in rightB] rightB = [i for i in rightB if i <= int(ktMax)] if rightB[-1] != int(ktMax): rightB.append(int(ktMax)) @@ -1285,6 +1323,16 @@ def getBoundaries(): data[amass + "AM"]["Err_L"] = round((((outL) / data[amass + "AM"]["Active"]) * 100), 2) data[amass + "AM"]["Err_R"] = round((((outR) / data[amass + "AM"]["Active"]) * 100), 2) + # Save original shape and position values (only once when main plot first shown) + if data[amass + "AM"]["shapeLeft_original"] is None: + data[amass + "AM"]["shapeLeft_original"] = shapeL + data[amass + "AM"]["posLeft_original"] = positionL + data[amass + "AM"]["shapeRight_original"] = shapeR + data[amass + "AM"]["posRight_original"] = positionR + print(f"Saved original configuration for {amass.upper()}: " + f"KnMax={knMax}, KtMax={ktMax}, " + f"shapeL={shapeL}, posL={positionL}, shapeR={shapeR}, posR={positionR}") + #################### # now work on plot #################### @@ -1299,7 +1347,7 @@ def getBoundaries(): a.clear() line = np.linspace(0, 100, 101) a.set_xlim([-2, 130]) - a.set_ylim([-2, 100]) + a.set_ylim([0, 100]) a.plot(line, line, color="black") canvas = FigureCanvasTkAgg(f, master=location) canvas.get_tk_widget().place(relx=0, rely=0, relwidth=1, relheight=1) @@ -1320,7 +1368,7 @@ def getBoundaries(): def plotGraph(dframe, amass, location, xAxis, yAxis, flag): - if len(dframe) > 1: + if len(dframe) > 0: col = dframe.columns line = np.linspace(0, 100, 101) @@ -1391,7 +1439,7 @@ def plotGraph(dframe, amass, location, xAxis, yAxis, flag): a.scatter(xActOut, yActOut, color='firebrick', marker=".") a.set_xlim([-2, 130]) - a.set_ylim([-2, 100]) + a.set_ylim([0, 100]) a.plot(line, line, color="black") a.tick_params(axis='x', colors='black') a.tick_params(axis='y', colors='black') @@ -1536,7 +1584,7 @@ def plotForEdit(amass, dframe, flag, x, y, titleString): plt.tick_params(axis='y', colors='black') plt.yticks(rotation=90) plt.xlim(-2, 130) - plt.ylim(-2, 100) + plt.ylim(0, 100) ax.scatter(xInact, yInact, color='gray', marker=".") if len(xActb10) > 0: @@ -1573,7 +1621,7 @@ def plotForEdit(amass, dframe, flag, x, y, titleString): mngr.window.setGeometry = (50, 100, 640, 545) # mngr.window.setGeometry = (250, 120, 1280, 1024) # mngr.window.setGeometry - mngr.window.wm_iconbitmap(resource_path("dataFiles/qcfit-icon.ico")) + setWindowIcon(mngr.window) plt.show(block=False) if data["editWinFlag"] == 1: @@ -1610,44 +1658,130 @@ def handleGraphCross(): editGraph.withdraw() def reset(dframe, amass, x, y): + """ + Reset boundaries and spinbox values to their original state from the data dictionary. + Uses the original Kn/Kt max values, shape, and position values saved when the main plot was first shown. + """ global data plt.close('all') outL = 0 outR = 0 + line = list(np.linspace(0, 100, 101)) - ktMax = int(data[amass + "AM"]["knMax"]) + # Get ORIGINAL Kn, Kt, shape, and position values (saved when main plot first shown) + Kn = int(data[amass + "AM"]["knMax_original"]) + Kt = int(data[amass + "AM"]["ktMax_original"]) + shapeL = data[amass + 'AM']['shapeLeft_original'] + posL = data[amass + 'AM']['posLeft_original'] + shapeR = data[amass + 'AM']['shapeRight_original'] + posR = data[amass + 'AM']['posRight_original'] + + # Also update the current values back to originals + data[amass + "AM"]["knMax"] = Kn + data[amass + "AM"]["ktMax"] = Kt + data[amass + 'AM']['shapeLeft'] = shapeL + data[amass + 'AM']['posLeft'] = posL + data[amass + 'AM']['shapeRight'] = shapeR + data[amass + 'AM']['posRight'] = posR + + print("\n" + "="*60) + print(f"RESET FUNCTION CALLED for {amass.upper()} airmass") + print("="*60) + print(f"Original Kn Max: {Kn}") + print(f"Original Kt Max: {Kt}") + print(f"Original Shape Left: {shapeL}, Position Left: {posL}") + print(f"Original Shape Right: {shapeR}, Position Right: {posR}") + + # Reset spinbox values to original values from data dictionary + knVal.delete(0, 'end') + knVal.insert(0, Kn) + ktVal.delete(0, 'end') + ktVal.insert(0, Kt) + print(f"Spinbox values reset to: Kn={Kn}, Kt={Kt}") + + # Reset UI control variables for shape and position + cLVar.set(shapeL) + cRVar.set(shapeR) + pLVar.set(posL) + pRVar.set(posR) + print(f"UI controls reset to: shapeL={shapeL}, posL={posL}, shapeR={shapeR}, posR={posR}") + + # Recalculate boundary curves using original values + print(f"\nRecalculating boundary curves with original values...") + + # Prepare left boundary curve + yLB = line[0:Kn + 1] + [Kn for point in line if point > Kn] + shiftL = (posL - 1) * 2.5 + xLB = utills.curveLeft[shapeL - 1] + xLB = [point + shiftL for point in xLB] - plotForEdit(amass, dframe, flag, x, y, titleString) - # ax.scatter(xAct, yAct, color='blue') - # ax.scatter(xInact, yInact, color='orange') - # ax.plot(line, line, color="black") # line no 0 - # ax.plot(data[amass + "AM"]["xLB"], data[amass + "AM"]["yLB"], color="green") # line no 1 - # ax.plot(data[amass + "AM"]["xRB"], data[amass + "AM"]["yRB"], color="green") # line no 2 + # copy first value at 0th index for boundary alignment + xLB = [xLB[0], *xLB] - # update data dictionary - for yp in range(0, ktMax): - # activeCountL = list(df[df["KN"] == y][x]) - # activeCountL = len([x for x in activeCountL if x < (ut.curveLeft[shapeL][y] + shiftL)]) - # outL = outL + activeCountL + # Trim boundary to Kn points and extend remainder with Kt value + xLB = xLB[0:Kn + 1] + xLB += [Kt] * (len(line) - len(xLB)) + + # Remove negative boundary values (set to 0) + zeroesL = [0 for point in xLB if point < 0] + xLB = zeroesL + xLB[len(zeroesL):] + + # Prepare right boundary curve (similar to left boundary) + yRB = line[0:Kn + 1] + [Kn for point in line if point > Kn] + shiftR = (posR - 1) * 2.5 + xRB = utills.curveRight[shapeR - 1] + xRB = [point + shiftR for point in xRB] + # copy first value at 0th index for boundary alignment + xRB = [xRB[0], *xRB] + # Ensure right boundary does not exceed Kt value + xRB = xRB[0:Kt + 1] + xRB = [i for i in xRB if i <= Kt] + + # Extend right boundary to match line length (with safety check) + if len(xRB) > 0: + xRB += [xRB[-1]] * (len(line) - len(xRB)) + else: + # Handle edge case where xRB is empty + xRB = [0] * len(line) + + print(f"Boundary data AFTER recalculation:") + print(f"Left Boundary (xLB) first 10 values: {xLB[:10]}") + print(f"Right Boundary (xRB) first 10 values: {xRB[:10]}") + + # Store recalculated/initial boundaries in data dictionary + data[amass + 'AM']['xLB'] = xLB + data[amass + 'AM']['yLB'] = yLB + data[amass + "AM"]["lBound"] = xLB + + data[amass + 'AM']['xRB'] = xRB + data[amass + 'AM']['yRB'] = yRB + data[amass + "AM"]["rBound"] = xRB + # Plot with reset boundaries + plotForEdit(amass, dframe, flag, x, y, titleString) + print("Configuration panel plot updated with reset boundaries") + + # Count data points outside left and right boundaries + for yp in range(0, Kn + 1): countL = list(dframe[dframe[y] == yp][x]) countL = [xp for xp in countL if xp < int(data[amass + "AM"]["xLB"][yp])] outL = outL + len(countL) countR = list(dframe[dframe[y] == yp][x]) countR = [xp for xp in countR if xp > int(data[amass + "AM"]["xRB"][yp])] outR = outR + len(countR) - # print("at y = ", y, " ", "out left ", outL, "listL", - # list(filterData[filterData["KN"] == y][x]), "curvePoint", data[amass+"AM"]["xLB"][y], " out right ", - # outR) - # count if points are above configured kn max + # Count if points are above configured kn max upCount = dframe[dframe[flag] == 1] - upCount = upCount[upCount[y] > int(knVal.get())] + upCount = upCount[upCount[y] > Kn] outUp = upCount.__len__() - # updating data dictionary + print(f"\nOutlier counts:") + print(f"Points outside Left boundary (outL): {outL}") + print(f"Points outside Right boundary (outR): {outR}") + print(f"Points above Kn max (outUp): {outUp}") + + # Update statistics in data dictionary data[amass + "AM"]["In"] = len(dframe[dframe[flag] == 1]) - (outL + outR + outUp) - # data[amass + "AM"]["Out"] = outL + outR + outUp data[amass + "AM"]["Active"] = len(dframe[dframe[flag] == 1]) if data[amass + "AM"]["Active"] > 1: data[amass + "AM"]["Out"] = str(outL + outR + outUp) + " (" + str(round( @@ -1659,10 +1793,32 @@ def reset(dframe, amass, x, y): if data[amass + "AM"]["Active"] > 1: data[amass + "AM"]["Err_L"] = round((((outL) / data[amass + "AM"]["Active"]) * 100), 2) data[amass + "AM"]["Err_R"] = round((((outR) / data[amass + "AM"]["Active"]) * 100), 2) + + print(f"\nStatistics after reset:") + print(f"In: {data[amass + 'AM']['In']}") + print(f"Out: {data[amass + 'AM']['Out']}") + print(f"Active: {data[amass + 'AM']['Active']}") + print(f"Ignored: {data[amass + 'AM']['Ignored']}") + print(f"Total: {data[amass + 'AM']['Total']}") + print(f"Err_L: {data[amass + 'AM']['Err_L']}%") + print(f"Err_R: {data[amass + 'AM']['Err_R']}%") + setStats() - # update main canvas plot + # Update the UI labels to display the reset statistics + InVal.configure(text=data[amass + "AM"]["In"]) + OutVal.configure(text=data[amass + "AM"]["Out"]) + ActiveVal.configure(text=data[amass + "AM"]["Active"]) + IgnoredVal.configure(text=data[amass + "AM"]["Ignored"]) + TotalVal.configure(text=data[amass + "AM"]["Total"]) + ErrLVal.configure(text=data[amass + "AM"]["Err_L"]) + ErrRVal.configure(text=data[amass + "AM"]["Err_R"]) + print("Configuration panel UI labels updated") + + # Update main canvas plot plotGraph(dframe, amass, locationDict[amass], data["xAxis"], data["yAxis"], data["flag"]) + print("Main canvas plot updated") + print("="*60 + "\n") def listPoints(amass, df, x, y): """if data["listPageFlag"] == 1: @@ -1814,58 +1970,74 @@ def exclude(amass, dframe, x, y): # tempData = tempData[tempData["KT"] > data["x1"]] def spinCalculation(currAmass, x, y): + """ + Updates Kn Max and Kt Max values for the specified airmass and recalculates boundaries. + + Args: + currAmass: Current airmass ('low', 'med', or 'high') + x: Column name for X-axis data (typically 'KT') + y: Column name for Y-axis data (typically 'KN') + """ global data, pdLow, pdMed, pdHigh line = list(np.linspace(0, 100, 101)) + # Calculate the change in Kn and Kt values from the spin box inputs knChange = int(knVal.get()) - int(data[currAmass + "AM"]["knMax"]) ktChange = int(ktVal.get()) - int(data[currAmass + "AM"]["ktMax"]) # if ktVal.get()==int(data[currAmass + "AM"]["xRB"][ktVal.get]): # return + # Iterate through all airmasses but only update the current one (currAmass) for amass, dframe in zip(["low", "med", "high"], [pdLow, pdMed, pdHigh]): - if data[amass + "AM"]["knMax"] != 'NA': - if 1 <= (int(data[amass + "AM"]["knMax"]) + knChange) <= 99: - if 1 <= (int(data[amass + "AM"]["ktMax"]) + ktChange) <= 130: + # IMPORTANT: Only process the currently selected airmass to keep values independent + if amass == currAmass and data[amass + "AM"]["knMax"] != 'NA': + # Validate that new values are within acceptable ranges (Kn: 0-99, Kt: 0-130) + if 0 <= (int(data[amass + "AM"]["knMax"]) + knChange) <= 99: + if 0 <= (int(data[amass + "AM"]["ktMax"]) + ktChange) <= 130: + # Update the Kn Max and Kt Max values for the current airmass only data[amass + "AM"]["knMax"] = int(data[amass + "AM"]["knMax"]) + knChange data[amass + "AM"]["ktMax"] = int(data[amass + "AM"]["ktMax"]) + ktChange - # renaming for ease + # Cache values for easier reference Kn = int(data[amass + "AM"]["knMax"]) Kt = int(data[amass + "AM"]["ktMax"]) - # Prepare left boundary + # Prepare left boundary curve + # Create Y-axis line up to Kn, then fill rest with Kn value yLB = line[0:Kn + 1] + [Kn for point in line if point > Kn] - shiftL = data[amass + 'AM']['posLeft'] * 2.5 + # Apply position shift to the left boundary curve + shiftL = (data[amass + 'AM']['posLeft'] - 1) * 2.5 xLB = utills.curveLeft[data[amass + 'AM']['shapeLeft'] - 1] xLB = [point + shiftL for point in xLB] - # copy first value at 0th index + # copy first value at 0th index for boundary alignment xLB = [xLB[0], *xLB] - # replace the values greater than knmax-th value in curve with knmax-th value + # Trim boundary to Kn points and extend remainder with Kt value xLB = xLB[0:Kn + 1] xLB += [Kt] * (len(line) - len(xLB)) - # Replace negative values with 0 + # Remove negative boundary values (set to 0) zeroesL = [0 for point in xLB if point < 0] xLB = zeroesL + xLB[len(zeroesL):] - # Prepare right boundary + # Prepare right boundary curve (similar to left boundary) yRB = line[0:Kn + 1] + [Kn for point in line if point > Kn] - shiftR = data[amass + 'AM']['posRight'] * 2.5 + shiftR = (data[amass + 'AM']['posRight'] - 1) * 2.5 xRB = utills.curveRight[data[amass + 'AM']['shapeRight'] - 1] xRB = [point + shiftR for point in xRB] - # copy first value at 0th index + # copy first value at 0th index for boundary alignment xRB = [xRB[0], *xRB] - # replace the values greater than knmax-th value in curve with knmax-th value + # Ensure right boundary does not exceed Kt value xRB = xRB[0:Kt + 1] xRB = [i for i in xRB if i <= Kt] + # Extend right boundary to match line length xRB += [xRB[-1]] * (len(line) - len(xRB)) - # update data dictionary with new boundaries and bounds(lBound and rBound) + # Store calculated boundaries in data dictionary data[amass + 'AM']['xLB'] = xLB data[amass + 'AM']['yLB'] = yLB data[amass + "AM"]["lBound"] = xLB @@ -1874,55 +2046,59 @@ def spinCalculation(currAmass, x, y): data[amass + 'AM']['yRB'] = yRB data[amass + "AM"]["rBound"] = xRB + # Get dataframe excluding marked outliers filtFrame = dframe[dframe[amass + "Exclude"] == 0] - # now clear old boundaries and plot new one. + # Refresh plot for the edited airmass if amass == currAmass: plt.close('all') plotForEdit(amass, filtFrame, flag, x, y, titleString) - # update main canvas plot + # Update main canvas plot to reflect boundary changes plotGraph(filtFrame, amass, locationDict[amass], data["xAxis"], data["yAxis"], data["flag"]) + + # Count data points outside left and right boundaries outL = 0 outR = 0 for yp in range(0, Kn + 1): - # activeCountL = list(df[df["KN"] == y]["KT"]) - # activeCountL = len([x for x in activeCountL if x < (ut.curveLeft[shapeL][y] + shiftL)]) - # outL = outL + activeCountL - + # Count points below left boundary for each Kn level countL = list(filtFrame[filtFrame[y] == yp][x]) countL = [xp for xp in countL if xp < data[amass + "AM"]["xLB"][yp]] outL = outL + len(countL) + + # Count points above right boundary for each Kn level countR = list(filtFrame[filtFrame[y] == yp][x]) countR = [xp for xp in countR if xp > data[amass + "AM"]["xRB"][yp]] outR = outR + len(countR) - # print("at y = ", y, " ", "out left ", outL, "listL", - # list(filterData[filterData["KN"] == y]["KT"]), "curvePoint", data[amass+"AM"]["xLB"][y], " out right ", - # outR) - # count if points are above configured kn max + # Count points above the configured Kn max upCount = filtFrame[filtFrame[flag] == 1] upCount = upCount[upCount[y] > Kn] outUp = upCount.__len__() - # updating data dictionary + # Update statistics in data dictionary + # "In" = valid points within boundaries data[amass + "AM"]["In"] = len(filtFrame[filtFrame[flag] == 1]) - (outL + outR + outUp) - # data[amass + "AM"]["Out"] = outL + outR + outUp + # "Active" = total flagged points data[amass + "AM"]["Active"] = len(filtFrame[filtFrame[flag] == 1]) + # "Out" = total outliers with percentage if data[amass + "AM"]["Active"] > 1: data[amass + "AM"]["Out"] = str(outL + outR + outUp) + " (" + str(round( ((outL + outR + outUp) / data[amass + "AM"]["Active"]) * 100, 2)) + " %)" else: data[amass + "AM"]["Out"] = outL + outR + outUp + # "Ignored" = total flagged (excluded) points data[amass + "AM"]["Ignored"] = len(filtFrame[filtFrame[flag] == 0]) + # "Total" = total data points data[amass + "AM"]["Total"] = len(filtFrame) + # Calculate error percentages for left and right boundaries if data[amass + "AM"]["Active"] > 1: data[amass + "AM"]["Err_L"] = round((((outL) / data[amass + "AM"]["Active"]) * 100), 2) data[amass + "AM"]["Err_R"] = round((((outR) / data[amass + "AM"]["Active"]) * 100), 2) setStats() - # Now update the current frame values + # Update the UI labels to display the current airmass statistics InVal.configure(text=data[currAmass + "AM"]["In"]) OutVal.configure(text=data[currAmass + "AM"]["Out"]) ActiveVal.configure(text=data[currAmass + "AM"]["Active"]) @@ -1952,7 +2128,7 @@ def reval(currAmass, x, y, *args): # Prepare left boundary yLB = line[0:Kn + 1] + [Kn for point in line if point > Kn] - shiftL = data[amass + 'AM']['posLeft'] * 2.5 + shiftL = (data[amass + 'AM']['posLeft'] - 1) * 2.5 xLB = utills.curveLeft[data[amass + 'AM']['shapeLeft'] - 1] xLB = [point + shiftL for point in xLB] @@ -1969,7 +2145,7 @@ def reval(currAmass, x, y, *args): # Prepare right boundary yRB = line[0:Kn + 1] + [Kn for point in line if point > Kn] - shiftR = data[amass + 'AM']['posRight'] * 2.5 + shiftR = (data[amass + 'AM']['posRight'] - 1) * 2.5 xRB = utills.curveRight[data[amass + 'AM']['shapeRight'] - 1] xRB = [point + shiftR for point in xRB] # copy first value at 0th index @@ -1989,6 +2165,27 @@ def reval(currAmass, x, y, *args): data[amass + 'AM']['yRB'] = yRB data[amass + "AM"]["rBound"] = xRB + # Store initial boundary curves ONLY if not already stored (first time graphConfig is opened) + # Check if initial curves are still at default [0] value (not yet set) + if data[amass + 'AM']['xLB_initial'] == [0]: + print(f"Storing initial curves for {amass.upper()} airmass") + print(f"Kn={Kn}, Kt={Kt}") + print(f"xLB first 10: {xLB[:10]}") + print(f"xRB first 10: {xRB[:10]}") + data[amass + 'AM']['xLB_initial'] = list(xLB) + data[amass + 'AM']['yLB_initial'] = list(yLB) + data[amass + 'AM']['xRB_initial'] = list(xRB) + data[amass + 'AM']['yRB_initial'] = list(yRB) + + # UPDATE original Kn/Kt values if they were loaded as zero from QA0 + # This ensures reset uses the QAFIT-calculated values, not the zero values + if int(data[amass + "AM"]["knMax_original"]) == 0 and int(data[amass + "AM"]["ktMax_original"]) == 0: + print(f"Updating knMax_original and ktMax_original from zero to calculated values") + data[amass + "AM"]["knMax_original"] = Kn + data[amass + "AM"]["ktMax_original"] = Kt + print(f"Original values now: Kn={Kn}, Kt={Kt}") + + filtFrame = dframe[dframe[amass + "Exclude"] == 0] # now clear old boundaries and plot new one. @@ -2206,8 +2403,8 @@ def errorCalculation(amass, scale, x, y): activeLBound = data[amass + "AM"]["activeLBound"] activeRBound = data[amass + "AM"]["activeRBound"] - for curve in range(0, 6): - for position in range(1, 21): + for curve in range(0, 8): + for position in range(1, 31): shift = ((position - 1) * 2.5) dataCountL = 0 dataCountR = 0 @@ -2218,7 +2415,7 @@ def errorCalculation(amass, scale, x, y): pointsL = zeroesL + pointsL[len(zeroesL):] # Find the right curve with shift - if curve < 5: + if curve < 7: pointsR = ut.curveRight[curve] pointsR = [(point) + shift for point in pointsR] @@ -2229,7 +2426,7 @@ def errorCalculation(amass, scale, x, y): activeCountL = len([xp for xp in activeCountL if xp < (pointsL[yp])]) dataCountL = dataCountL + activeCountL - if curve < 5: + if curve < 7: activeCountR = list(df[df[y] == yp][x]) activeCountR = len([xp for xp in activeCountR if xp > (pointsR[yp])]) dataCountR = dataCountR + activeCountR @@ -2238,13 +2435,13 @@ def errorCalculation(amass, scale, x, y): pcL[dim, curve, position - 1] = pixCountL # / scale dcL[dim, curve, position - 1] = dataCountL # * scale - if curve < 5: + if curve < 7: pixCountR = np.sum(abs(np.array(pointsR[:knMax + 1]) - np.array(activeRBound))) pcR[dim, curve, position - 1] = pixCountR # / scale dcR[dim, curve, position - 1] = dataCountR # * scale pixCountL = pcL[dim, curve, position - 1] / scale - pixCountL = pixCountL / (areaLeft.iloc[int(knMax) - 1][(curve * 20) + position]) + pixCountL = pixCountL / areaLeft.loc[int(knMax) - 1, f"{curve + 1}-{position}"] dataCountL = dcL[dim, curve, position - 1] * scale lcount = dcL[dim, curve, position - 1] @@ -2259,9 +2456,9 @@ def errorCalculation(amass, scale, x, y): positionL = position outL = lcount - if curve < 5: + if curve < 7: pixCountR = pcR[dim, curve, position - 1] / scale - pixCountR = pixCountR / (areaRight.iloc[int(knMax) - 1][(curve * 20) + position]) + pixCountR = pixCountR / areaRight.loc[int(knMax) - 1, f"{curve + 1}-{position}"] dataCountR = dcR[dim, curve, position - 1] * scale rcount = dcR[dim, curve, position - 1] dataCountR = dataCountR / len(df) @@ -2282,7 +2479,7 @@ def errorCalculation(amass, scale, x, y): leftB = list(ut.curveLeft[shapeL - 1]) leftB = leftB[0:int(knMax)] # get all the points of curve upto Ymax - leftB = [(i + (2.5 * positionL)) for i in leftB] + leftB = [(i + (2.5 * (positionL - 1))) for i in leftB] if leftB[-1] != int(ktMax): leftB.append(int(ktMax)) @@ -2298,7 +2495,7 @@ def errorCalculation(amass, scale, x, y): rightB = list(ut.curveRight[shapeR - 1]) rightB = rightB[0:int(knMax)] # get all the points of curve upto Ymax - rightB = [(i + (2.5 * positionR)) for i in rightB] + rightB = [(i + (2.5 * (positionR - 1))) for i in rightB] rightB = [i for i in rightB if i <= int(ktMax)] if rightB[-1] != int(ktMax): rightB.append(int(ktMax)) @@ -2331,7 +2528,7 @@ def errorCalculation(amass, scale, x, y): # Prepare left boundary yLB = list(line[0:knMax + 1]) + [knMax for point in line if point > knMax] - shiftL = data[amass + 'AM']['posLeft'] * 2.5 + shiftL = (data[amass + 'AM']['posLeft'] - 1) * 2.5 xLB = utills.curveLeft[data[amass + 'AM']['shapeLeft'] - 1] xLB = [point + shiftL for point in xLB] @@ -2348,7 +2545,7 @@ def errorCalculation(amass, scale, x, y): # Prepare right boundary yRB = list(line[0:knMax + 1]) + [knMax for point in line if point > knMax] - shiftR = data[amass + 'AM']['posRight'] * 2.5 + shiftR = (data[amass + 'AM']['posRight'] - 1) * 2.5 xRB = utills.curveRight[data[amass + 'AM']['shapeRight'] - 1] xRB = [point + shiftR for point in xRB] # copy first value at 0th index @@ -2467,7 +2664,7 @@ def close(): ktLabel = tk.Label(kCanvas, text="Kt Max * 100", font=(None, 12)) ktLabel.place(relx=0.51, rely=0.2, relwidth=0.30, relheight=0.6) ktVal = tk.Spinbox(kCanvas, from_=minKt, to=ktMaxSpinVal, textvariable=ktSpinVar, - command=partial(spinCalculation, amass)) + command=partial(spinCalculation, amass, x, y)) ktVal.place(relx=0.82, rely=0.1, relwidth=0.17, relheight=0.8) ktVal.delete(0, 'end') ktVal.insert(0, int(data[amass + "AM"]["ktMax"])) @@ -2482,7 +2679,7 @@ def close(): # list for option menus curveL = np.arange(1, 9) # [1, 2, 3, 4, 5, 6, 7, 8] curveR = np.arange(1, 8) # [1, 2, 3, 4, 5, 6, 7] - posit = np.arange(1, 41) # [1, ..., 40] + posit = np.arange(1, 31) # [1, ..., 30] cLabel = tk.Label(boundCanvas, text="Curve", font=(None, 12), anchor="w") cLabel.place(relx=0.01, rely=0.3566, relwidth=0.32, relheight=0.2866) @@ -2564,7 +2761,7 @@ def close(): close.place(relx=0.72, rely=0.88, relwidth=0.18, relheight=0.07) editGraph.protocol("WM_DELETE_WINDOW", handleGraphCross) - editGraph.iconbitmap(resource_path('dataFiles/qcfit-icon.ico')) + setWindowIcon(editGraph) editGraph.mainloop() @@ -2798,7 +2995,7 @@ def plotByYear(*args): thisYearY = thisYearData[yAxis] a.scatter(thisYearX, thisYearY, color='cyan', marker=".") a.set_xlim([-2, 130]) - a.set_ylim([-2, 100]) + a.set_ylim([0, 100]) a.plot(line, line, color="black") a.plot(xLB, yLB, color="green") a.plot(xRB, yRB, color="green") @@ -2859,7 +3056,7 @@ def freqPlot(): dat["bin"] = 0 dat["freqStatus"] = dat["lowExclude"] + dat["medExclude"] + dat["highExclude"] - fData = dat[dat["month"] == ((monVar.get()).upper())[:3]][dat["freqStatus"] == 0] + fData = dat.loc[(dat["month"] == ((monVar.get()).upper())[:3]) & (dat["freqStatus"] == 0)].copy() if data["plane"] == 1: flag = data["flag"] = "activeKtKn" elif data["plane"] == 2: @@ -2894,7 +3091,7 @@ def freqZoomed(event): loMed = pd.concat([pdLow, pdMed], ignore_index=True) dat = pd.concat([loMed, pdHigh], ignore_index=True) dat["bin"] = 0 - fData = dat[dat["month"] == ((monVar.get()).upper())[:3]] + fData = dat.loc[dat["month"] == ((monVar.get()).upper())[:3]].copy() if data["plane"] == 1: flag = data["flag"] = "activeKtKn" @@ -2934,7 +3131,7 @@ def freqZoomed(event): plt.show(block=False) plt.xlim(-15, 15) - pLow = fData[fData[flag] == 1][fData["NAM"] == 1] + pLow = fData.loc[(fData[flag] == 1) & (fData["NAM"] == 1)] if len(pLow) > 0: lowMean = list(pLow["bin"]) lowMean = [int(i) for i in lowMean] @@ -2942,7 +3139,7 @@ def freqZoomed(event): lowMean = lowMean * 20 else: lowMean = [0] * 20 - pMed = fData[fData[flag] == 1][fData["NAM"] == 2] + pMed = fData.loc[(fData[flag] == 1) & (fData["NAM"] == 2)] if len(pMed) > 0: medMean = list(pMed["bin"]) medMean = [int(i) for i in medMean] @@ -2950,7 +3147,7 @@ def freqZoomed(event): medMean = medMean * 30 else: medMean = [0] * 30 - pHigh = fData[fData[flag] == 1][fData["NAM"] == 3] + pHigh = fData.loc[(fData[flag] == 1) & (fData["NAM"] == 3)] if len(pHigh) > 0: highMean = list(pHigh["bin"]) highMean = [int(i) for i in highMean] @@ -2964,7 +3161,7 @@ def freqZoomed(event): plt.hist((highMean), label="High Airmass", color='blue') plt.legend() mngr = plt.get_current_fig_manager() - mngr.window.wm_iconbitmap(resource_path("dataFiles/qcfit-icon.ico")) + setWindowIcon(mngr.window) def density(*args): @@ -3078,7 +3275,8 @@ def density(*args): densityData["freq%"] = densityData["frequency"] / sum(densityData["frequency"]) densityData = densityData.sort_values(by=['density']) densityData["cumulative"] = densityData['freq%'].cumsum() - densityData.iloc[len(densityData) - 1, 4] = 1 + if not densityData.empty: + densityData.iloc[len(densityData) - 1, 4] = 1 # Allocating bins densityData.loc[(densityData["cumulative"] <= 1), "densBin"] = 5 densityData.loc[(densityData["cumulative"] < .8), "densBin"] = 4 @@ -3336,7 +3534,7 @@ def density(*args): a.scatter(xActb51, yActb51, color='khaki', marker=".") a.set_xlim([-2, 130]) - a.set_ylim([-2, 100]) + a.set_ylim([0, 100]) a.plot(line, line, color="black") a.plot(xLB, yLB, color="green") a.plot(xRB, yRB, color="green") @@ -3412,13 +3610,19 @@ def save(month): if iterCount == 0: messagebox.showerror("Error", "Sorry, No data available to save.") return - fileQC = open(data["qa0Path"], "r") - content = fileQC.read().splitlines() - fileQC.close() + try: + content = utills.readQA0(data["qa0Path"]).splitlines() + except (utills.QA0FormatError, OSError, UnicodeError) as exc: + messagebox.showerror("QA0 File Error", f"Cannot save {data['qa0Path']}:\n{exc}") + return # update QA0 data - position = monList.index(month) + 10 # locating month - string = content[position] + _mon = month[:3].upper() + positions = [i for i, line in enumerate(content) if line.lstrip().startswith(_mon + ":")] + if len(positions) != 1: + messagebox.showerror("Error", f"Expected one {_mon} boundary row in the QA0 file.") + return + position = positions[0] # find knMax and KtMax __kn = [data["lowAM"]["knMax"], data["medAM"]["knMax"], data["highAM"]["knMax"]] @@ -3426,9 +3630,6 @@ def save(month): __kn = pd.to_numeric(pd.Series(__kn), errors="coerce").fillna(0).astype(int).tolist() __kt = pd.to_numeric(pd.Series(__kt), errors="coerce").fillna(0).astype(int).tolist() - # first writing Knmax and left curve shape and positions - _mon, _, _lam_max, _lam_left, _lam_right, _, _mam_max, _mam_left, _mam_right, _, _ham_max, _ham_left, _ham_right = re.split(r"[:,;, ]", string) - # replace Kn-Kt max values _lam_max = f'{__kn[0]:02}-{__kt[0]:03}' _mam_max = f'{__kn[1]:02}-{__kt[1]:03}' @@ -3444,7 +3645,7 @@ def save(month): _ham_right = f'{data["highAM"]["shapeRight"]}-{data["highAM"]["posRight"]:02}' # flesh out the final string with all the updated values - string = f"{_mon}: {_lam_max} {_lam_left} {_lam_right}; {_mam_max} {_mam_left} {_mam_right}; {_ham_max} {_ham_left} {_ham_right}" + string = f"{_mon}: {_lam_max} {_lam_left} {_lam_right}; {_mam_max} {_mam_left} {_mam_right}; {_ham_max} {_ham_left} {_ham_right};" content[position] = string content = "\n".join(content) fid = open(data["qa0Path"], 'w') @@ -3530,13 +3731,30 @@ def exitProgram(): def help(): helpRoot = tk.Toplevel() - helpRoot.wm_geometry("800x720+500+250") - logo = ImageTk.PhotoImage(Image.open(resource_path('dataFiles/disclaimer.png'))) + helpRoot.wm_geometry("1000x720+500+250") + logoSource = Image.open(resource_path('dataFiles/disclaimer_qafit.png')) + logoPreview = logoSource.copy() + logoPreview.thumbnail((496, 716), Image.Resampling.LANCZOS) + logo = ImageTk.PhotoImage(logoPreview) logoFrame = tk.Frame(helpRoot, bd=2) logoFrame.place(relx=0, rely=0, relwidth=0.5, relheight=1) - labelImg = tk.Label(logoFrame, image=logo) + labelImg = tk.Label(logoFrame, image=logo, bd=0, padx=0, pady=0) labelImg.place(relx=0, rely=0, relwidth=1, relheight=1) + logoState = {"size": None, "image": logo} + + def resizeDisclaimer(event): + size = (event.width, event.height) + if size[0] <= 0 or size[1] <= 0 or size == logoState["size"]: + return + + resizedImage = logoSource.copy() + resizedImage.thumbnail(size, Image.Resampling.LANCZOS) + logoState["image"] = ImageTk.PhotoImage(resizedImage) + logoState["size"] = size + labelImg.configure(image=logoState["image"]) + + labelImg.bind("", resizeDisclaimer) text = "DISCLAIMER FOR \n SOFTWARE SUBMITTED TO OSTI\n\nThis Software is provided by the National Renewable Energy\n" \ "Laboratory (\"NREL\"), which is operated by the Alliance for\n" \ " Sustainable Energy, LLC (\"ALLIANCE\") for the " \ @@ -3579,7 +3797,7 @@ def help(): textLabel = tk.Label(helpRoot, text=text) textLabel.place(relx=0.5, rely=0.0, relwidth=0.50, relheight=1) - helpRoot.iconbitmap(resource_path('dataFiles/qcfit-icon.ico')) + setWindowIcon(helpRoot) helpRoot.mainloop() @@ -3623,14 +3841,14 @@ def callSave(): 'listPageFlag': 0, 'flag': 'activeKtKn', 'xAxis': 'KT', 'yAxis': 'KN', 'x1': 0, 'x2': 0, 'y1': 0, 'y2': 0, 'density': {'low': {'lower': "", 'upper': ""}, 'med': {'lower': "", 'upper': ""}, 'high': {'lower': "", 'upper': ""}}, - 'lowAM': {'ktMax': 0, 'knMax': 0, 'kdMax': 0, 'shapeLeft': 0, 'shapeRight': 0, 'posLeft': 0, 'posRight': 0, - 'lBound': [0], 'rBound': [0], 'In': 0, 'Out': 0, 'Active': 0, 'Ignored': 0, 'Total': 0, 'Err_L': 0, + 'lowAM': {'ktMax': 0, 'knMax': 0, 'ktMax_original': 0, 'knMax_original': 0, 'kdMax': 0, 'shapeLeft': 0, 'shapeRight': 0, 'posLeft': 0, 'posRight': 0, + 'lBound': [0], 'rBound': [0], 'xLB_initial': [0], 'xRB_initial': [0], 'yLB_initial': [0], 'yRB_initial': [0], 'In': 0, 'Out': 0, 'Active': 0, 'Ignored': 0, 'Total': 0, 'Err_L': 0, 'Err_R': 0, 'activeLBound': [0], 'activeRBound': [0], 'scale': 1}, - 'medAM': {'ktMax': 0, 'knMax': 0, 'kdMax': 0, 'shapeLeft': 0, 'shapeRight': 0, 'posLeft': 0, 'posRight': 0, - 'lBound': [0], 'rBound': [0], 'In': 0, 'Out': 0, 'Active': 0, 'Ignored': 0, 'Total': 0, 'Err_L': 0, + 'medAM': {'ktMax': 0, 'knMax': 0, 'ktMax_original': 0, 'knMax_original': 0, 'kdMax': 0, 'shapeLeft': 0, 'shapeRight': 0, 'posLeft': 0, 'posRight': 0, + 'lBound': [0], 'rBound': [0], 'xLB_initial': [0], 'xRB_initial': [0], 'yLB_initial': [0], 'yRB_initial': [0], 'In': 0, 'Out': 0, 'Active': 0, 'Ignored': 0, 'Total': 0, 'Err_L': 0, 'Err_R': 0, 'activeLBound': [0], 'activeRBound': [0], 'scale': 1}, - 'highAM': {'ktMax': 0, 'knMax': 0, 'kdMax': 0, 'shapeLeft': 0, 'shapeRight': 0, 'posLeft': 0, 'posRight': 0, - 'lBound': [0], 'rBound': [0], 'In': 0, 'Out': 0, 'Active': 0, 'Ignored': 0, 'Total': 0, 'Err_L': 0, + 'highAM': {'ktMax': 0, 'knMax': 0, 'ktMax_original': 0, 'knMax_original': 0, 'kdMax': 0, 'shapeLeft': 0, 'shapeRight': 0, 'posLeft': 0, 'posRight': 0, + 'lBound': [0], 'rBound': [0], 'xLB_initial': [0], 'xRB_initial': [0], 'yLB_initial': [0], 'yRB_initial': [0], 'In': 0, 'Out': 0, 'Active': 0, 'Ignored': 0, 'Total': 0, 'Err_L': 0, 'Err_R': 0, 'activeLBound': [0], 'activeRBound': [0], 'scale': 1}} siteInfo = {} @@ -3643,12 +3861,12 @@ def callSave(): lowFilter = pd.DataFrame() medFilter = pd.DataFrame() highFilter = pd.DataFrame() -dcL = np.zeros([3, 6, 20]) -pcL = np.zeros([3, 6, 20]) -dcR = np.zeros([3, 5, 20]) -pcR = np.zeros([3, 5, 20]) -scoreLeft = np.zeros([3, 5, 20]) -scoreRight = np.zeros([3, 5, 20]) +dcL = np.zeros([3, 8, 30]) +pcL = np.zeros([3, 8, 30]) +dcR = np.zeros([3, 7, 30]) +pcR = np.zeros([3, 7, 30]) +scoreLeft = np.zeros([3, 8, 30]) +scoreRight = np.zeros([3, 7, 30]) iterCount = 0 dFlag = 0 filterDict = {'low': lowFilter, 'med': medFilter, 'high': highFilter} @@ -4167,5 +4385,5 @@ def resizeLogo(event): disable(integrationFrame.winfo_children()) mainRoot.protocol("WM_DELETE_WINDOW", exitProgram) -mainRoot.iconbitmap(resource_path('dataFiles/qcfit-icon.ico')) +setWindowIcon(mainRoot) mainRoot.mainloop() diff --git a/SERI-QC_with_QAFIT/seriqc_script.py b/SERI-QC_with_QAFIT/seriqc_script.py index 79b2d67..b127625 100644 --- a/SERI-QC_with_QAFIT/seriqc_script.py +++ b/SERI-QC_with_QAFIT/seriqc_script.py @@ -760,7 +760,11 @@ def main(): ################################# - qcData = utills.readFile(path_QA0) + try: + qcData = utills.readQA0(path_QA0) + except (utills.QA0FormatError, OSError, UnicodeError) as exc: + messagebox.showerror("QA0 File Error", f"Cannot open {path_QA0}:\n{exc}") + return ipData = pd.read_csv(path_csv) diff --git a/SERI-QC_with_QAFIT/utills.py b/SERI-QC_with_QAFIT/utills.py index db02042..e812dfc 100644 --- a/SERI-QC_with_QAFIT/utills.py +++ b/SERI-QC_with_QAFIT/utills.py @@ -9,12 +9,51 @@ import pandas as pd import numpy as np import os +import re +import math + + +class QA0FormatError(ValueError): + pass + + +def readQA0(name): + text = readFile(name) + for field in ('Site Identifier', 'Latitude', 'Longitude', 'Time Zone', 'Integration (minutes)'): + values = re.findall(r'^[ \t-]*' + re.escape(field) + r':[ \t]*(.*)$', text, re.MULTILINE) + if len(values) != 1 or not values[0].strip(): + raise QA0FormatError(f"Expected one non-empty '{field}:' field.") + if field != 'Site Identifier': + try: + value = float(values[0]) + except ValueError as exc: + raise QA0FormatError(f"'{field}:' must contain a number.") from exc + if not math.isfinite(value): + raise QA0FormatError(f"'{field}:' must contain a finite number.") + if field == 'Integration (minutes)' and (value <= 0 or not value.is_integer()): + raise QA0FormatError("'Integration (minutes):' must be a positive whole number.") + for field in ('Elevation', '3-Component Filter'): + values = re.findall(r'^[ \t-]*' + re.escape(field) + r':[ \t]*(.*)$', text, re.MULTILINE) + if len(values) > 1: + raise QA0FormatError(f"Duplicate '{field}:' field.") + if values and values[0].strip(): + try: + value = float(values[0]) + except ValueError as exc: + raise QA0FormatError(f"'{field}:' must contain a number.") from exc + if not math.isfinite(value): + raise QA0FormatError(f"'{field}:' must contain a finite number.") + planes = re.findall(r'^[ \t]*Plane:[ \t]*(.*)$', text, re.MULTILINE) + if len(planes) > 1 or (planes and planes[0].strip() and + not re.fullmatch(r'[012](?:\s+K[tn]-K[nd])?', planes[0].strip())): + raise QA0FormatError("'Plane:' must start with 0, 1, or 2 (for example, '0 Kt-Kn').") + getBoundariesTable(text) + return text def readFile(name): # pass the *.qa0 file name - f = (open(name, 'r')) - val = (f.read()) - return val + with open(name, 'r') as file: + return file.read() def readList(name): # pass the *.txt file name @@ -49,26 +88,66 @@ def pickValue(data, Name): def getBoundariesTable(data): - start = data.find('JAN') - stop = data.find('DEFAULT') - data = data[start:stop] - data = data.replace(':', ',') - data = data.replace('-', ',') - data = data.replace('/', ',') - data = data.replace(' ', ',') - data = data.replace(';', ',') - data = data.replace(',,', ',') - data = data.split('\n')[0:12] - - table = pd.DataFrame({'data': data}) - + months = 'JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC'.split() + lines = data.splitlines() + starts = [i for i, line in enumerate(lines) if line.lstrip().startswith('JAN:')] + stops = [i for i, line in enumerate(lines) if line.strip() == 'DEFAULT CONFIGURATION'] + if len(starts) != 1 or len(stops) != 1 or starts[0] >= stops[0]: + raise QA0FormatError("Expected a JAN: boundary row followed by DEFAULT CONFIGURATION.") + rows = [(i + 1, line.strip()) for i, line in enumerate(lines) + if starts[0] <= i < stops[0] and line.strip()] + if len(rows) != 12: + raise QA0FormatError(f"Expected 12 monthly boundary rows (JAN through DEC); found {len(rows)}.") + records = [] + for month, (line_number, line) in zip(months, rows): + prefix = f"Line {line_number}: invalid {month} boundary row." + label, separator, values = line.partition(':') + if not separator or label != month: + raise QA0FormatError(f"Line {line_number}: expected {month}:, found {label!r}.") + groups = values.strip().removesuffix(';').split(';') + if len(groups) != 3: + raise QA0FormatError( + f"{prefix} Separate the low, medium, and high air-mass sections " + f"with semicolons; found {len(groups)} sections instead of 3." + ) + record = [month] + for air_mass, group in zip(('low', 'medium', 'high'), groups): + context = f"{prefix} {air_mass.capitalize()} air mass:" + pairs = group.split() + if len(pairs) != 3: + raise QA0FormatError( + f"{context} expected maximum values, left curve, and right curve " + f"(for example, '95-123 2-03 4-05'); found {len(pairs)} entries." + ) + for pair, fields in zip(pairs, ( + (('Kn max', 2), ('Kt max', 3)), + (('left curve shape', 1), ('left curve position', 2)), + (('right curve shape', 1), ('right curve position', 2)), + )): + parts = pair.split('-') + if len(parts) != 2: + raise QA0FormatError( + f"{context} separate {fields[0][0]} and {fields[1][0]} " + f"with a hyphen; found {pair!r}." + ) + for value, (field, width) in zip(parts, fields): + if not re.fullmatch(r'\d+', value): + raise QA0FormatError( + f"{context} {field} must be a non-negative whole number; " + f"found {value!r}." + ) + if len(value) > width: + raise QA0FormatError( + f"{context} {field} must have at most {width} digits; " + f"found {value!r}." + ) + record.append(value) + records.append(record) columns = ['month', 'LA_MAX_KN', 'LA_MAX_KT', 'LA_Left_S', 'LA_Left_P', 'LA_Right_S', 'LA_Right_P', 'MA_MAX_KN', 'MA_MAX_KT', 'MA_Left_S', 'MA_Left_P', 'MA_Right_S', 'MA_Right_P', 'HA_MAX_KN', 'HA_MAX_KT', 'HA_Left_S', 'HA_Left_P', 'HA_Right_S', 'HA_Right_P'] - table[columns] = table['data'].str.split(',', expand=True) - table = table.iloc[:, 1:] - return table + return pd.DataFrame(records, columns=columns) def maxX(i, j): diff --git a/tests/test_qa0_format.py b/tests/test_qa0_format.py new file mode 100644 index 0000000..49ba7e7 --- /dev/null +++ b/tests/test_qa0_format.py @@ -0,0 +1,166 @@ +"""QA0 validator tests with embedded samples and mocked file I/O. + +Run with the application's Python environment: + python tests/test_qa0_format.py -v + +These tests exercise the real validator, not GUI dialogs. +""" + +from pathlib import Path +import sys +import unittest +from unittest.mock import mock_open, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'SERI-QC_with_QAFIT')) +import utills + + +VALID_QA0 = """ Site Identifier: TEST, Sample station + --- Latitude: 39.74 + --- Longitude: -105.18 + --- Elevation: 1827.0 + --- Time Zone: -7.0 + + ---Low Airmass--- ---Med Airmass--- ---High Airmass-- + -Max- -Lft Rgt- -Max- -Lft Rgt- -Max- -Lft Rgt- + Kn Kt S P S P Kn Kt S P S P Kn Kt S P S P +JAN: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +FEB: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +MAR: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +APR: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +MAY: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +JUN: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +JUL: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +AUG: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +SEP: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +OCT: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +NOV: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; +DEC: 00-000 0-00 0-00; 00-000 0-00 0-00; 00-000 0-00 0-00; + +DEFAULT CONFIGURATION + Integration (minutes): 1 + Data Folder: + Plane: 0 Kt-Kn + 3-Component Filter: 1.0 + Comments: +""" + + +class QA0FormatTests(unittest.TestCase): + def setUp(self): + self.valid = VALID_QA0 + + def read_text(self, text): + with patch('builtins.open', mock_open(read_data=text)): + return utills.readQA0('station.qa0') + + def test_legacy_format_and_optional_final_semicolon(self): + rows = [line for line in self.valid.splitlines() + if line[:3] in 'JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC'.split()] + self.assertEqual(len(rows), 12) + self.assertTrue(all(line.endswith(';') for line in rows)) + table = utills.getBoundariesTable(self.read_text(self.valid)) + self.assertEqual(table.shape, (12, 19)) + self.assertEqual(table.iloc[0]['month'], 'JAN') + self.assertEqual(table.iloc[-1]['month'], 'DEC') + text = self.valid.replace(';\n', '\n') + self.assertNotEqual(text, self.valid) + self.assertTrue(table.equals(utills.getBoundariesTable(self.read_text(text)))) + + def test_new_template_and_three_digit_kt(self): + text = self.valid.replace( + ' ---Low Airmass-- ---Med Airmass-- --High Airmass--\n' + ' -MAX- -LFT RGT- -MAX- -LFT RGT- -MAX- -LFT RGT-\n' + ' KN KT S P S P KN KT S P S P KN KT S P S P\n\n', + ' ---Low Airmass--- ---Med Airmass--- ---High Airmass--\n' + ' -Max- -Lft Rgt- -Max- -Lft Rgt- -Max- -Lft Rgt-\n' + ' Kn Kt S P S P Kn Kt S P S P Kn Kt S P S P\n', + ).replace('00-000', '95-123') + table = utills.getBoundariesTable(self.read_text(text)) + self.assertEqual(table.iloc[0]['LA_MAX_KT'], '123') + self.assertEqual(table.iloc[-1]['HA_MAX_KN'], '95') + + def test_invalid_boundary_rows(self): + cases = { + 'missing month': self.valid.replace('FEB:', 'BAD:'), + 'duplicate month': self.valid.replace('FEB:', 'JAN:'), + 'missing value': self.valid.replace('00-000', '00-', 1), + 'non-numeric value': self.valid.replace('00-000', '00-abc', 1), + 'missing separator': self.valid.replace('0-00; ', '0-00 ', 1), + 'extra value': self.valid.replace('00-000', '00-000 7', 1), + 'missing configuration': self.valid.replace('DEFAULT CONFIGURATION', ''), + 'missing row': '\n'.join(line for line in self.valid.splitlines() + if not line.startswith('DEC:')), + } + for name, text in cases.items(): + with self.subTest(name=name), self.assertRaises(utills.QA0FormatError): + self.read_text(text) + + def test_invalid_metadata(self): + cases = ( + self.valid.replace('Site Identifier:', 'Station:'), + self.valid.replace('Latitude:', 'Latitude: invalid'), + self.valid.replace('Longitude:', 'Longitude: nan'), + self.valid.replace('Elevation:', 'Elevation: invalid'), + self.valid.replace('Integration (minutes):', 'Integration (minutes): -'), + self.valid.replace('Plane: 0', 'Plane: 4'), + self.valid + '\nLatitude: 10\n', + ) + for text in cases: + with self.subTest(text=text), self.assertRaises(utills.QA0FormatError): + self.read_text(text) + + def test_error_identifies_line(self): + text = self.valid.replace('00-000', '00-bad', 1) + with self.assertRaisesRegex(utills.QA0FormatError, r'Line \d+: invalid JAN'): + self.read_text(text) + + def test_error_identifies_invalid_field(self): + cases = ( + ('00-ABC 0-00 0-00', 'Kt max', "'ABC'"), + ('XY-000 0-00 0-00', 'Kn max', "'XY'"), + ('00-000 X-00 0-00', 'left curve shape', "'X'"), + ('00-000 0-XY 0-00', 'left curve position', "'XY'"), + ('00-000 0-00 X-00', 'right curve shape', "'X'"), + ('00-000 0-00 0-XY', 'right curve position', "'XY'"), + ('00-1234 0-00 0-00', 'Kt max must have at most 3 digits', "'1234'"), + ) + for air_mass, index in (('Low', 0), ('Medium', 1), ('High', 2)): + for replacement, field, value in cases: + with self.subTest(air_mass=air_mass, field=field): + lines = self.valid.splitlines() + row = next(i for i, line in enumerate(lines) if line.startswith('JAN:')) + groups = lines[row].partition(':')[2].strip().removesuffix(';').split(';') + groups[index] = replacement + lines[row] = 'JAN: ' + '; '.join(group.strip() for group in groups) + ';' + with self.assertRaises(utills.QA0FormatError) as caught: + self.read_text('\n'.join(lines)) + message = str(caught.exception) + self.assertIn(f'Line {row + 1}: invalid JAN', message) + self.assertIn(f'{air_mass} air mass:', message) + self.assertIn(field, message) + self.assertIn(value, message) + self.assertNotIn('semicolon', message) + + def test_read_uses_requested_path(self): + reader = mock_open(read_data=self.valid) + with patch('builtins.open', reader): + self.assertEqual(utills.readQA0('station.qa0'), self.valid) + reader.assert_called_once_with('station.qa0', 'r') + + def test_invalid_input_is_not_written(self): + reader = mock_open(read_data=self.valid.replace('00-000', '00-bad', 1)) + with patch('builtins.open', reader): + with self.assertRaises(utills.QA0FormatError): + utills.readQA0('invalid.qa0') + reader.assert_called_once_with('invalid.qa0', 'r') + reader().write.assert_not_called() + + def test_file_read_error_is_propagated(self): + with patch('builtins.open', side_effect=OSError('Cannot read QA0')): + with self.assertRaisesRegex(OSError, 'Cannot read QA0'): + utills.readQA0('unreadable.qa0') + + +if __name__ == '__main__': + unittest.main()