# ============================================================================#
# File: main_gui_logic.py #
# Author: Pfesesani V. van Zyl #
# ============================================================================#
# Standard library imports
# --------------------------------------------------------------------------- #
from ast import Pass
from datetime import datetime
from PyQt5 import QtWidgets # from PyQt6 import QtWidgets
import numpy as np
import sys
import os
import matplotlib.pyplot as pl
from matplotlib.backends.backend_qtagg import (
FigureCanvasQTAgg as FigureCanvas, NavigationToolbar2QT as NavigationToolbar)
import pandas as pd
import webbrowser
from astropy.time import Time
import glob
# Local imports
# --------------------------------------------------------------------------- #
sys.path.append("src/")
from common.msgConfiguration import msg_wrapper
from common.sqlite_db import SQLiteDB
# from common.fits_file_reader import FitsFileReader
from common.observation import Observation
# from common.driftScans import DriftScans
import common.fitting as fit
from gui.main_window import Ui_MainWindow
from gui.edit_driftscan_window1 import Ui_DriftscanWindow
from gui.edit_timeseries_window import Ui_TimeSeriesWindow
from gui.view_plots_window import Ui_PlotViewer
from common.file_handler import FileHandler
import common.calibrate as cp
from gui.canvasManager import CanvasManager
from gui.secondaryCanvasManager import SecondaryCanvasManager
from gui.timeseries_canvas import TimeCanvas
# from common.fitting import f as fx
import common.fitting as fit
import common.miscellaneousFunctions as misc
# =========================================================================== #
[docs]
class Main(QtWidgets.QMainWindow, Ui_MainWindow):
""" The main class that handles all gui operations."""
def __init__(self, log):
super(Main, self).__init__()
self.log = log
self.filePath=""
msg_wrapper("debug", self.log.debug,"GUI initiated")
# Setup the user interface
self.setupUi(self)
self.statusbar.showMessage("Ready.")
# initiate file handling
self.file=FileHandler(self.log)
# setup holder for deleted values
self.deleted=[]
if self.filePath=="":
# set button properties
btnCol='white'
btnTextCol='black'
# Connect buttons to window
self.btn_edit_driftscan.clicked.connect(self.open_drift_window)
self.btn_edit_timeseries.clicked.connect(self.open_timeseries_window)
self.btn_view_plots.clicked.connect(self.open_plots_window)
# self.btn_edit_driftscan.setStyleSheet(f'QPushButton {{background-color: {btnCol}; color: {btnTextCol};}}')
# self.btn_edit_timeseries.setStyleSheet(f'QPushButton {{background-color: {btnCol}; color: {btnTextCol};}}')
# self.btn_view_plots.setStyleSheet(f'QPushButton {{background-color: {btnCol}; color: {btnTextCol};}}')
else:
self.open_drift_window()
# self.open_file()
# === Windows setup ===
[docs]
def open_drift_window(self):
""" Connect the edit drift scan window to the main window. """
msg_wrapper("debug", self.log.debug, "Initiating drift scan editing window")
# Initiate Canvas
self.Canvas = CanvasManager(log=self.log)
self.ntb = NavigationToolbar(self.Canvas, self)
self.drift_window = QtWidgets.QMainWindow()
self.drift_ui = Ui_DriftscanWindow()
self.drift_ui.setupUi(self.drift_window)
# initiate Secondary canvas
self.secondaryCanvas = SecondaryCanvasManager(log=self.log)
# Layouts
plotLayout = self.drift_ui.PlotLayout
otherPlotLayout = self.drift_ui.otherPlotsLayout
# Add Canvas/es to the gui
plotLayout.addWidget(self.ntb)
plotLayout.addWidget(self.Canvas)
otherPlotLayout.addWidget(self.secondaryCanvas)
# Connect buttons to actions performed by user
self.connect_buttons()
# Print welcome message
self.write("DRAN GUI loaded successfully.",'info')
self.write("Open a file to get started.",'info')
# Set status, indicates whether fit/plot has been modified/not
self.status = [0, 0, 0, 0, 0, 0]
self.drift_window.show()
[docs]
def open_timeseries_window(self):
""" Connect the edit timeseries window. """
msg_wrapper("debug", self.log.debug, "Initiating timeseries editing window")
# Initiate Canvas
self.Canvas = TimeCanvas(log=self.log)
self.ntb = NavigationToolbar(self.Canvas,self) # add nav toolbar
self.time_window = QtWidgets.QMainWindow()
self.time_ui = Ui_TimeSeriesWindow()
self.time_ui.setupUi(self.time_window)
# Layouts
plotLayout = self.time_ui.PlotLayout
# Add canvas to the gui
plotLayout.addWidget(self.ntb)
plotLayout.addWidget(self.Canvas)
# connect buttons to actions performed by users
self.time_ui.BtnResetPoint.setVisible(False)
self.time_ui.BtnFit.setVisible(True)
self.time_ui.BtnQuit.setText("Update db")
self.time_ui.EdtSplKnots.setVisible(False)
self.time_ui.LblSplKnots.setVisible(False)
self.time_ui.BtnUpdateDB.setVisible(False)
self.time_ui.BtnOpenDB.clicked.connect(self.open_db)
self.time_ui.comboBoxColsYerr.setVisible(True)
self.time_ui.Lblxlim.setVisible(False)
self.time_ui.Lblylim.setVisible(False)
self.time_ui.EdtxlimMin.setVisible(False)
self.time_ui.EdtxlimMax.setVisible(False)
self.time_ui.EdtylimMax.setVisible(False)
self.time_ui.EdtylimMin.setVisible(False)
self.time_ui.BtnFilter.setEnabled(False)
self.time_ui.BtnRefreshDB.setVisible(True)
self.time_ui.BtnSaveDB.setVisible(False)
self.time_ui.EdtFilter.setEnabled(False)
self.time_ui.EdtEndDate.setVisible(False)
self.time_ui.EdtStartDate.setVisible(False)
self.time_ui.LblEndDate.setVisible(False)
self.time_ui.LblStartDate.setVisible(False)
# Setup index change options for combo boxes
self.time_ui.comboBoxTables.currentIndexChanged.connect(self.on_table_name_changed)
self.time_ui.comboBoxFitTypes.currentIndexChanged.connect(self.on_fit_changed)
# connect buttons
self.time_ui.BtnRefreshDB.clicked.connect(self.refresh_list) #TODO: make this operational
self.time_window.show()
[docs]
def open_plots_window(self):
""" Connect the plots window to the main window. """
msg_wrapper("debug", self.log.debug, "Initiating plot viewer window")
# Initiate viewer
self.plot_window = QtWidgets.QMainWindow()
self.plot_ui = Ui_PlotViewer()
self.plot_ui.setupUi(self.plot_window)
# fix all else
self.plot_ui.btnDelete.setEnabled(False)
self.plot_ui.btnRefreshPlotList.setEnabled(False)
self.plot_ui.btnShow.setEnabled(False)
self.plot_ui.comboBox.setEnabled(False)
self.plot_ui.comboBoxFilter.setEnabled(False)
self.plot_ui.comboBoxOptions.setEnabled(False)
self.plot_ui.txtBoxEnd.setEnabled(False)
self.plot_ui.txtBoxStart.setEnabled(False)
# Enable open db only
self.plot_ui.btnOpen.clicked.connect(self.open_db_path)
# Setup index change options for combo boxes
self.plot_ui.comboBox.currentIndexChanged.connect(self.on_combo_changed)
self.plot_ui.comboBoxFilter.currentIndexChanged.connect(self.on_filter_changed)
# # Connect buttons to actions performed by user
self.plot_ui.btnRefreshPlotList.clicked.connect(self.add_items_to_combobox)
self.plot_ui.btnShow.clicked.connect(self.show_plot_browser)
self.plot_ui.btnDelete.clicked.connect(self.delete_obs)
self.plot_window.show()
# checked start --------
# ---------------------------
[docs]
def write(self,msg,logType=""):
""" Write to screen and gui """
if logType=="info":
msg_wrapper("info", self.log.info, msg)
else:
msg_wrapper("debug", self.log.debug, msg)
[docs]
def open_file_name_dialog(self, ext):
""" Open file. Had to use this
to avoid segmentation fault: 11 error
"""
msg_wrapper("debug", self.log.debug, "Open file name dialog")
fileName, _ = QtWidgets.QFileDialog.getOpenFileName(
self, "Open File", "", "Fits Files ("+ext+");;Fits Files ("+ext+")")
if fileName:
return fileName
else:
return
[docs]
def set_fit_parmeters(self,bf=0,pf=0,ps=0):
"""
Initialize fit status parameters. These are parameters that tell us
wether a certain part of the scan has been fit or not. The status of
the fit can be either 0 (data not fit yet) or 1 (data fitting has
been done) for the current scan. A smoothing parameter is also included
to let us know if smoothing has been done on the original data.
By default, all the parameters are set to zero.
"""
msg_wrapper("debug", self.log.debug, "Setting fit status")
self.base_is_fit = bf # Baseline has not been corrected
self.peak_is_fit = pf # Peak fit has not been done
self.plot_is_smoothed = ps # rmscut/smoothing has not been done
[docs]
def reset_plot(self):
""" Reset figure to default settings. """
self.write("Reset figure to default settings",'info')
# Get the plot index, set fitting done to zero and rest any previous fits
plotIndex = self.get_plot_index()
self.fit_done = 0
self.reset_previous_fits()
# Reset x and y to factory settings of current plot
self.reset_xy(plotIndex)
print('here')
# Reset the plot
title = self.set_title()
self.Canvas.plot_figure(self.x, self.y, title, "Raw data")
self.Canvas.plot_residual()
# Reset fit and flag parameters
self.set_fit_parmeters()
self.set_flags()
self.reset_peak(plotIndex)
[docs]
def get_plot_index(self):
""" Get the current plot index. Deliberately set to ind+1 to account for offset."""
msg_wrapper("debug", self.log.debug,
f'Getting plot index {self.drift_ui.ComboBoxPlotType.currentIndex()}\n')
return int(self.drift_ui.ComboBoxPlotType.currentIndex())+1
[docs]
def reset_previous_fits(self):
""" Reset previous fits. """
msg_wrapper("debug", self.log.debug, "Reset previous fits")
if "D" in self.data["FRONTEND"]:
self.reset_dual_plot_fit_a()
self.reset_dual_plot_fit_b()
if "S" in self.data["FRONTEND"]:
self.reset_plot_fit()
[docs]
def reset_dual_plot_fit_a(self):
""" Reset all previously fit parameters of the A beam to zreo. """
self.peakFitA = np.nan
self.peakRmsA = np.nan
self.s2nA = np.nan
self.rmsbA = np.nan
self.rmsaA = np.nan
self.base_slopeA = np.nan
self.base_rmsA = np.nan
self.peakLocA=np.nan
[docs]
def reset_dual_plot_fit_b(self):
""" Reset all previously fit parameters of the B beam to zero. """
self.peakFitB = np.nan
self.peakRmsB = np.nan
self.s2nB = np.nan
self.rmsbB = np.nan
self.rmsaB = np.nan
self.base_slopeB = np.nan
self.base_rmsB = np.nan
self.peakLocB=np.nan
[docs]
def reset_plot_fit(self):
""" Reset all previously fit parameters to zero. """
self.peakFit = np.nan
self.peakRms = np.nan
self.s2n = np.nan
self.baseSlope = np.nan
self.base_rms = np.nan
self.peakLoc = np.nan
[docs]
def reset_xy(self, ind):
""" Set X and Y values to current plot. """
# TODO: check this does what it says it does
msg_wrapper("debug", self.log.debug, "Setting X and Y")
# for single beam
try:
self.x = self.scans[self.scanKeys[0]]
except Exception as e:
print(e)
self.x = np.zeros_like(1000)
# for dual beam - check this again
# try:
# ind=ind+2
# for i in range(2, len(self.lenScans)):
# if ind == i:
# self.y = self.scans[self.scanKeys[ind]]
# except Exception:
# self.y = np.zeros_like(1000)
[docs]
def set_title(self):
""" Set the plot title. """
plotTitle = "Plot of " + \
self.data['OBJECT']+" @ " + \
str(self.data['CENTFREQ'])+" MHz"
return plotTitle
[docs]
def set_flags(self, flag=100):
""" Set base flags.
FLAGS = 100 # Source was manually edited/reduced by user
This becomes the default for all flags if you save to DB
while using the GUI.
"""
msg_wrapper("debug", self.log.debug, "set flags")
plotIndex = self.get_plot_index()
beam = self.data['FRONTEND']
if beam == "13.0S":
if (self.scanKeys[plotIndex]) == "OLTA":
self.data['OLFLAG'] = flag
if (self.scanKeys[plotIndex]) == "ORTA":
self.data['ORFLAG'] = flag
else:
if (self.scanKeys[plotIndex]) == "SLTA":
try:
self.data["SLFLAG"] = flag
except:
pass
elif (self.scanKeys[plotIndex]) == "NLTA":
try:
self.data["NLFLAG"] = flag
except:
pass
elif (self.scanKeys[plotIndex]) == "OLTA":
try:
self.data["OLFLAG"] = flag
except:
pass
elif (self.scanKeys[plotIndex]) == "SRTA":
try:
self.data["SRFLAG"] = flag
except:
pass
elif (self.scanKeys[plotIndex]) == "NRTA":
try:
self.data["NRFLAG"] = flag
except:
pass
elif (self.scanKeys[plotIndex]) == "ORTA":
try:
self.data["ORFLAG"] = flag
except:
pass
[docs]
def reset_peak(self,plotIndex):
""" Reset previous peak fits. """
msg_wrapper("debug", self.log.debug, 'reset peak fit')
if self.data['FRONTEND'] == "13.0S":
if plotIndex == 0:
msg_wrapper("debug", self.log.debug, "Data for LCP")
self.reset_dict("OLTA", "")
else:
msg_wrapper("debug", self.log.debug, "Data for RCP")
self.reset_dict("ORTA", "")
elif self.data["FRONTEND"] == "02.5S" or self.data["FRONTEND"] == "01.3S" or self.data["FRONTEND"] == "04.5S":
msg_wrapper("debug", self.log.debug, "Fit stats for " +
self.scanKeys[plotIndex]+" data")
if plotIndex == 0:
self.reset_dict("SLTA", "NLTA")
elif plotIndex == 1:
self.reset_dict("NLTA", "OLTA")
elif plotIndex == 2:
self.reset_dict("OLTA", "SRTA")
elif plotIndex == 3:
self.reset_dict("SRTA", "NRTA")
elif plotIndex == 4:
self.reset_dict("NRTA", "ORTA")
elif plotIndex == 5:
self.reset_dict("ORTA", "")
elif "D" in self.data["FRONTEND"]:
msg_wrapper("debug", self.log.debug, "Fit stats for " +
self.scanKeys[plotIndex]+" data")
if plotIndex == 0:
self.reset_dict("ASLTA", "ANLTA")
elif plotIndex == 1:
self.reset_dict("ANLTA", "AOLTA")
elif plotIndex == 2:
self.reset_dict("AOLTA", "ASRTA")
elif plotIndex == 3:
self.reset_dict("ASRTA", "ANRTA")
elif plotIndex == 4:
self.reset_dict("ANRTA", "AORTA")
elif plotIndex == 5:
self.reset_dict("AORTA", "")
[docs]
def reset_dict(self, filter_key1, filter_key2=""):
""" Reset the data dictionary containing the
driftscan keys
"""
ld = list((self.data).items())
keys = list((self.data).keys())
print(ld)
print(keys)
print(filter_key1)
sys.exit()
# values = list((self.data).values())
idx_of_key1 = int(keys.index(filter_key1))
try:
idx_of_key2 = int(keys.index(filter_key2))
except:
idx_of_key2 = len(ld)
if self.data["FRONTEND"] == "02.5S" or self.data["FRONTEND"] == "01.3S" or "D" in self.data["FRONTEND"]:
if filter_key1 == "SLTA":
for i in range(len(ld)):
if i >= idx_of_key1 and i < idx_of_key2:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
else:
pass
elif filter_key1 == "NLTA":
for i in range(len(ld)):
if i >= idx_of_key1 and i < idx_of_key2:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
else:
pass
elif filter_key1 == "OLTA":
for i in range(len(ld)):
if i >= idx_of_key1 and i < idx_of_key2:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
else:
pass
elif filter_key1 == "SRTA":
for i in range(len(ld)):
if i >= idx_of_key1 and i < idx_of_key2:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
else:
pass
elif filter_key1 == "NRTA":
for i in range(len(ld)):
if i >= idx_of_key1 and i < idx_of_key2:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
else:
pass
elif filter_key1 == "ORTA":
for i in range(len(ld)):
if i >= idx_of_key1:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
else:
pass
else:
for i in range(len(ld)):
if i >= idx_of_key1:
if "FLAG" in keys[i]:
self.data[keys[i]] = 100
else:
self.data[keys[i]] = 0
# === Combobox change setup ===
[docs]
def on_fit_combobox_changed(self):
""" Handle events related to the combobox option change. """
msg_wrapper("debug", self.log.debug, "Fitting ComboBox option changed")
# Check the selected fit type
self.write("Now fitting: " +
self.drift_ui.ComboBoxFitLoc.currentText(),'info')
# Set the fit type
if self.drift_ui.ComboBoxFitLoc.currentText() == "Base":
self.drift_ui.ComboBoxFitOrder.setCurrentIndex(0)
elif self.drift_ui.ComboBoxFitLoc.currentText() == "Peak":
self.drift_ui.ComboBoxFitOrder.setCurrentIndex(1)
[docs]
def on_filter_type_combobox_changed(self):
"""
Handle events related to the filter type combobox option change.
"""
msg_wrapper("debug", self.log.debug, "Combo Box Filter Type Changed")
if self.drift_ui.ComboBoxFilterType.currentText() == "Please select:":
self.write("Smoothing the data ",'info')
# Enable filtering window
self.drift_ui.EdtFilteringWindow.setEnabled(False)
self.drift_ui.BtnFilterData.setEnabled(False)
elif self.drift_ui.ComboBoxFilterType.currentText() == "rms cuts":
self.write("Performing Rms cuts",'info')
# Enable filtering window
self.drift_ui.EdtFilteringWindow.setEnabled(False)
self.drift_ui.BtnFilterData.setEnabled(True)
else:
self.drift_ui.EdtFilteringWindow.setEnabled(True)
self.drift_ui.BtnFilterData.setEnabled(True)
[docs]
def on_fit_type_combobox_changed(self):
""" Handle events related to the combobox option change. """
msg_wrapper("debug", self.log.debug,
"Fitting ComboBox option changed ")
fit_type = self.get_fit_type()
# Check the selected fit type
self.write(" Now fitting: " + fit_type,'info')
# Set the fit type
if fit_type == "Polynomial":
# Enable peak fitting buttons
self.drift_ui.ComboBoxFitOrder.setEnabled(True)
self.drift_ui.ComboBoxFitLoc.setEnabled(True)
[docs]
def on_checkbox_changed(self):
""" Update the cursor depending on checkbox status (on/off)"""
# Check state of checkbox
if self.drift_ui.checkBox.isChecked() == True:
self.write("Checked",'info')
self.Canvas.cursorState = "on"
else:
self.write("Not checked",'info')
self.Canvas.cursorState = "off"
# === Enabling, disabling and populating widgets ===
[docs]
def enable_widgets(self):
""" Enable the widgets. """
msg_wrapper("debug", self.log.debug,"Enabling widgets")
self.drift_ui.ComboBoxPlotType.setEnabled(True)
self.drift_ui.ComboBoxFitLoc.setEnabled(True)
self.drift_ui.ComboBoxFitOrder.setEnabled(True)
self.drift_ui.ComboBoxFitType.setEnabled(True)
self.drift_ui.ComboBoxFilterType.setEnabled(True)
self.drift_ui.EdtFilteringWindow.setEnabled(True)
# Enable buttons
self.drift_ui.BtnChoosePlot.setEnabled(True)
self.drift_ui.BtnViewFit.setEnabled(True)
self.drift_ui.BtnFitData.setEnabled(True)
self.drift_ui.BtnClearSelection.setEnabled(True)
self.drift_ui.BtnResetPlot.setEnabled(True)
self.drift_ui.BtnSave.setEnabled(True)
#self.drift_ui.BtnResetFit.setEnabled(True)
self.drift_ui.BtnSaveToDb.setEnabled(True)
self.drift_ui.BtnFilterData.setEnabled(True)
self.drift_ui.BtnCalc.setEnabled(True)
# enable target buttons
self.drift_ui.BtnPopulatePSS.setEnabled(True)
self.drift_ui.BtnSavePSS.setVisible(False)
self.drift_ui.BtnResetPSS.setVisible(False)
# Enable checkboxes
self.drift_ui.checkBox.setEnabled(True)
self.cursor_state = "off"
[docs]
def populate_widgets(self):
""" Populate the gui widgets. """
msg_wrapper("debug", self.log.debug,"Populate widgets")
self.drift_ui.EdtFilename.setText(self.data["FILENAME"])
self.drift_ui.EdtCurDate.setText(self.data["CURDATETIME"])
self.drift_ui.EdtObsDate.setText(self.data["OBSDATE"])
self.drift_ui.EdtObsTime.setText(self.data["OBSTIME"])
self.drift_ui.EdtObjectType.setText(self.data["OBSERVER"])
self.drift_ui.EdtObjectName.setText(self.data["OBJECT"])
self.drift_ui.EdtMjd.setText(str(f'{self.data["MJD"]:.1f}'))
self.drift_ui.EdtFreq.setText(str(self.data["CENTFREQ"]))
self.drift_ui.EdtTsysL.setText(str(f'{self.data["TSYS1"]:.1f}'))
self.drift_ui.EdtTsysR.setText(str(f'{self.data["TSYS2"]:.1f}'))
self.drift_ui.EdtTemp.setText(str(f'{self.data["TAMBIENT"]:.1f}'))
self.drift_ui.EdtPres.setText(str(f'{self.data["PRESSURE"]:.1f}'))
self.drift_ui.EdtHum.setText(str(f'{self.data["HUMIDITY"]:.1f}'))
self.drift_ui.EdtFnbw.setText(str(f'{self.data["FNBW"]:.2f}'))
self.drift_ui.EdtHa.setText(str(f'{self.data["HA"]:.1f}'))
self.drift_ui.EdtZa.setText(str(f'{self.data["ZA"]:.1f}'))
self.drift_ui.EdtHpbw.setText(str(f'{self.data["HPBW"]:.2f}'))
[docs]
def setup_flags(self):
""" Setup flag parameters, default =100. These
flags tell us whether a scan has been fit manually
or not. see set_flags()
"""
msg_wrapper("debug", self.log.debug,"Setting flags")
if ("13.0S" in self.data["FRONTEND"]):
self.data["OLFLAG"] = 100
self.data["ORFLAG"] = 100
else:
self.data["SLFLAG"] = 100
self.data["NLFLAG"] = 100
self.data["OLFLAG"] = 100
self.data["SRFLAG"] = 100
self.data["NRFLAG"] = 100
self.data["ORFLAG"] = 100
[docs]
def open_file(self):
""" Open file for processing. """
self.write("Opening file",'info')
# hide pss's
self.drift_ui.pss_values_groupbox.setVisible(False)
# Initialize fit parameters
self.set_fit_parmeters()
self.filePath = self.open_file_name_dialog("*.fits")
# self.filePath= '/Users/pfesesanivanzyl/data/Calibrators/HydraA/2280/2021d365_00h28m13s_Cont_mike_HYDRA_A.fits'
if self.filePath == None:
self.write("You need to select a file to open",'info')
self.write("Please select a file",'info')
pass
else:
try:
self.reset_plot()
self.reset_peak()
except:
pass
self.drift_ui.pss_values_groupbox.setVisible(False)
# Get data from file
self.obs=Observation(FILEPATH=self.filePath, theoFit='',autoFit='',log=self.log, dbCols={})
self.obs.get_data_only()
self.data, self.scans = {},{}
self.scanKeys, self.dataKeys = [], []
for k,v in self.obs.__dict__.items():
# print(k,v)
if 'theo' in k or 'auto' in k or 'INFOHEADER' in k or 'log' in k or 'HDULIST' in k or 'CARDS' in k or 'plotDir' in k or 'RAW' in k or 'J2000' in k:
pass
elif 'TA_' in k or 'OFFSET' in k:
# if 'OFFSET' in k:
# self.scans['OFFSET']=v
# else:
print('--- ',k,v)
if k=='ON_TA_LCP' or k=='ON_TA_RCP':
if 'L' in k:
key='OLTA'
else:
key='ORTA'
self.scans[key]=v['value']
self.scanKeys.append(key)
self.data[key]=v['value']
elif k=='HPN_TA_LCP' or k=='HPN_TA_RCP':
if 'L' in k:
key='NLTA'
else:
key='NRTA'
self.scans[key]=v['value']
self.scanKeys.append(key)
self.data[key]=v['value']
elif k=='HPS_TA_LCP' or k=='HPS_TA_RCP':
if 'L' in k:
key='SLTA'
else:
key='SRTA'
self.scans[key]=v['value']
self.scanKeys.append(key)
self.data[key]=v['value']
else:
self.scans[k]=v['value']
self.scanKeys.append(k)
# self.data[k]=v['value']
else:
# print(k,v)
try:
self.data[k] = v['value']
except:
self.data[k] = v
self.dataKeys.append(k)
print('\n')
print('Data keys: ',self.data.keys(), '\n')
print('Scan keys: ',self.scans.keys(),'\n')
# sys.exit()
# print(self.data['FILEPATH'],'\n')
# self.lenData= len(self.data)
self.lenScans= len(self.scans)
# sys.exit()
# # self.data = fileReader.data
# # self.scans = fileReader.scans
self.Canvas.delete_canvas()
# set the figure layout
if "D" in self.data["FRONTEND"]:
self.Canvas.init_dual_fig_canvas()
self.Canvas.clear_dual_canvas()
self.secondaryCanvas.init_canvas()
self.secondaryCanvas.clear_canvas()
else:
self.Canvas.init_fig_canvas()
self.Canvas.clear_canvas()
self.secondaryCanvas.init_canvas()
self.secondaryCanvas.clear_canvas()
# # Setup pss and flux calculation buttons
# if (self.data["OBJECTTYPE"] == "CAL"):
# self.drift_ui.BtnCalc.setText("Calc PSS")
# self.drift_ui.BtnCalc.clicked.connect(self.calc_pss_)
# # remove pss estimation buttons and boxes
# self.drift_ui.pss_values_groupbox.setVisible(False)
# #self.disable_target_widgdets()
# elif self.data["OBJECTTYPE"] == "TAR":
# #self.drift_ui.BtnCalc.
# self.drift_ui.BtnCalc.setText("Calc Flux")
# self.drift_ui.BtnCalc.clicked.connect(self.calc)
# self.drift_ui.pss_values_groupbox.setVisible(False)
# #self.enable_target_buttons()
self.write("Opened file: "+self.filePath,'info')
print("\n")
# sys.exit()
# Setup environment variables
# self.set_beam_names()
self.populate_widgets()
self.enable_widgets()
# sys.exit()
if self.lenScans == 3:
# for single beam scans
ind=0
self.x = self.scans[self.scanKeys[ind]]
self.y = self.scans[self.scanKeys[ind+1]]
y1 = self.scans[self.scanKeys[ind+2]]
self.secondaryCanvas.plot_figure(self.x, self.y, y1, label1=self.scanKeys[ind+1],label2=self.scanKeys[ind+2])
else:
# for dual beam scans
ind=0
self.x = self.scans[self.scanKeys[ind]]
self.y = self.scans[self.scanKeys[ind+1]]
y1 = self.scans[self.scanKeys[ind+2]]
self.secondaryCanvas.plot_figure(self.x, self.y, y1, label1=self.scanKeys[ind+1],label2=self.scanKeys[ind+2])
# Add items to combo boxes
self.drift_ui.ComboBoxPlotType.clear()
self.drift_ui.ComboBoxPlotType.clear()
self.drift_ui.ComboBoxPlotType.addItems(self.scanKeys[1:])
# sys.exit()
# allProperties = list(self.data.keys())#[3:]
#self.drift_ui.ComboBoxScanProperty.setEnabled(True)
#self.drift_ui.ComboBoxScanProperty.clear()
#self.drift_ui.ComboBoxScanProperty.addItems(allProperties)
#self.drift_ui.newPropertyValue.setEnabled(True)
#self.drift_ui.BtnModify.setEnabled(True)
self.fit_types = ["Polynomial"]#, "Gaussian"]
self.drift_ui.ComboBoxFitType.clear()
self.drift_ui.ComboBoxFitType.clear()
self.drift_ui.ComboBoxFitType.addItems(self.fit_types)
self.smoothing_types = ["please select:", "rms cuts", "flat",
"hanning", "bartlett", "blackman"]
self.drift_ui.ComboBoxFilterType.clear()
self.drift_ui.ComboBoxFilterType.clear()
self.drift_ui.ComboBoxFilterType.addItems(self.smoothing_types)
# Setup index change options for combo boxes
self.drift_ui.ComboBoxFitLoc.currentIndexChanged.connect(
self.on_fit_combobox_changed)
self.drift_ui.ComboBoxFilterType.currentIndexChanged.connect(
self.on_filter_type_combobox_changed)
self.drift_ui.ComboBoxFitType.currentIndexChanged.connect(
self.on_fit_type_combobox_changed)
self.drift_ui.checkBox.stateChanged.connect(
self.on_checkbox_changed)
# SET FLAGS
self.setup_flags()
# set x,y, plot first image
# print(ind,self.scanKeys[ind+1])
# print(len(self.x),len(self.y))
# sys.exit()
# self.reset_xy(0)
# print(len(self.x),len(self.y))
# sys.exit()
self.Canvas.plot_figure(self.x, self.y, label=self.scanKeys[ind+1])
if "D" in self.data["FRONTEND"]:
self.Canvas.label_dual_res()
else:
self.Canvas.label_res()
self.write("Starting data reduction",'info')
[docs]
def update_smoothing_params(self):
""" Update that plot has been smoothed"""
#TODO: check this function points to the correct index
plotIndex = self.get_plot_index()
if "13.0S" == self.data["FRONTEND"]:
if plotIndex == 1:
self.data["OLRMSB"] = self.rmsb
self.data["OLRMSA"] = self.rmsa
elif plotIndex == 2:
self.data["ORRMSB"] = self.rmsb
self.data["ORRMSA"] = self.rmsa
elif "01.3S" == self.data["FRONTEND"] or "02.5S" == self.data["FRONTEND"] or "04.5S" == self.data["FRONTEND"]:
if plotIndex == 1:
self.data["SLRMSB"] = self.rmsb
self.data["SLRMSA"] = self.rmsa
elif plotIndex == 2:
self.data["NLRMSB"] = self.rmsb
self.data["NLRMSA"] = self.rmsa
elif plotIndex == 3:
self.data["OLRMSB"] = self.rmsb
self.data["OLRMSA"] = self.rmsa
elif plotIndex == 4:
self.data["SRRMSB"] = self.rmsb
self.data["SRRMSA"] = self.rmsa
elif plotIndex == 5:
self.data["NRRMSB"] = self.rmsb
self.data["NRRMSA"] = self.rmsa
elif plotIndex == 6:
self.data["ORRMSB"] = self.rmsb
self.data["ORRMSA"] = self.rmsa
else:
pass
else:
pass
[docs]
def filter_data(self):
""" Filter/smooth the data """
if self.drift_ui.ComboBoxFilterType.currentText() == "rms cuts":
# perform rms cuts on the data, remove points with calues > 3*rms
self.write("Performing rms cuts",'info')
# ================================================
# Clean the data using rms cuts
# ================================================
scanLen = len(self.x)
if self.data["CENTFREQ"] <= 3000:
noTotScans = 2
else:
noTotScans = 6
# spline the data
spl = fit.spline(self.x, self.y)
scanRes, self.rmsb, self.rmsa, self.x, self.y, self.res, finMaxSpl, finspl, pointsDeleted = fit.clean_data(
spl, self.x, self.y, scanLen,self.log)
self.write(f'RMS before: after -> {self.rmsb}: {self.rmsa}','info')
self.Canvas.clear_canvas() # Update figures
self.update_plot(self.x, self.y, "After rms cuts",
self.res, "smoothed data", "smoothed residual") # Update plot
#update rms before and after
self.update_smoothing_params()
self.plot_is_smoothed = 1
else:
msg_wrapper("debug", self.log.debug, "Smoothing window")
smooth_type = self.drift_ui.ComboBoxFilterType.currentText()
if smooth_type == "":
self.write("Please select a filter.")
self.plot_is_smoothed = 0
else:
# TODO: Select a better smoothing algorithm
# Get value from smoothing window
window = self.drift_ui.EdtFilteringWindow.text()
try:
window = int(window)
except Exception as e:
self.write("You need to enter a window width")
if type(window).__name__ == "int":
smoothed = fit.filter_scans(self.y, window, smooth_type)
self.x = np.arange(0, len(smoothed), 1)
self.y = smoothed
self.Canvas.clear_canvas()
self.update_plot(self.x, self.y, "After " +
smooth_type+" smoothing", [])
self.plot_is_smoothed = 1
# def sbKeys(self):
# sbKeys = ['FILENAME','FILEPATH','HDULENGTH','CURDATETIME','OBSDATE',
# 'OBSTIME','OBSDATETIME','OBJECT','LONGITUD','LAATITUDE',
# 'COORDSYS','EQUINOX','RADECSYS','OBSERVER','OBSLOCAL',
# 'PROJNAME','PROPOSAL','TELESCOP','UPGRADE','FOCUS','TILT',
# 'TAMBIENT','PRESSURE','HUMIDITY','WINDSPD','SCANDIR',
# 'POINTING','FEEDTYPE','BMOFFHA']
[docs]
def view_fit(self):
""" View current fit info"""
self.write("Viewing fit information...",'info')
plotIndex = self.get_plot_index()
print('*** ',plotIndex)
print(self.data.keys())
sys.exit()
if self.data['FRONTEND'] == "13.0S":
if plotIndex == 0:
msg_wrapper("debug", self.log.debug, "Data for LCP")
self.iter_dict("OLTA", "ORTA")
else:
msg_wrapper("debug", self.log.debug, "Data for RCP")
self.iter_dict("ORTA", "")
elif self.data["FRONTEND"] == "02.5S" or self.data["FRONTEND"] == "01.3S":
plotIndex = self.get_plot_index()
if plotIndex == 0:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("SLTA", "NLTA")
elif plotIndex == 1:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data", 'info')
self.iter_dict("NLTA", "OLTA")
elif plotIndex == 2:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("OLTA", "SRTA")
elif plotIndex == 3:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("SRTA", "NRTA")
elif plotIndex == 4:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("NRTA", "ORTA")
elif plotIndex == 5:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("ORTA", "")
elif "D" in self.data["FRONTEND"]:
#TODO: what about "B" scans?
plotIndex = self.get_plot_index()
if plotIndex == 0:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("ASLTA", "ANLTA")
elif plotIndex == 1:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("ANLTA", "AOLTA")
elif plotIndex == 2:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("AOLTA", "ASRTA")
elif plotIndex == 3:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("ASRTA", "ANRTA")
elif plotIndex == 4:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("ANRTA", "AORTA")
elif plotIndex == 5:
self.write("Fit stats for "+self.scanKeys[plotIndex]+" data",'info')
self.iter_dict("AORTA", "")
# def iter_dict(self, filter_key1, filter_key2=""):
# """
# Dictionary iterator to extract current fit information
# """
# ld = list((self.data).items())
# keys = list((self.data).keys())
# values = list((self.data).values())
# idx_of_key1 = int(keys.index(filter_key1))
# try:
# idx_of_key2 = int(keys.index(filter_key2))
# except:
# idx_of_key2 = len(ld)
# self.write("Fit stats: \n"+"-"*30,'info')
# if self.data["FRONTEND"] == "02.5S" or self.data["FRONTEND"] == "01.3S" or self.data["FRONTEND"] == "04.5S":
# if filter_key1 == "SLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "NLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "OLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "SRTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "NRTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "ORTA":
# for i in range(len(ld)):
# if i >= idx_of_key1:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif self.data["FRONTEND"] == "13.0S":
# if filter_key1 == "OLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2 and idx_of_key2 != "":
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# for i in range(len(ld)):
# if i >= idx_of_key1:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# elif "D" in self.data["FRONTEND"]:
# if filter_key1 == "ASLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "ANLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "AOLTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "ASRTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "ANRTA":
# for i in range(len(ld)):
# if i >= idx_of_key1 and i < idx_of_key2:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# elif filter_key1 == "AORTA":
# for i in range(len(ld)):
# if i >= idx_of_key1:
# self.write(str(keys[i]) + ": " + str(values[i]),'info')
# else:
# pass
# OPERATIONS FOR TIMESERIES
[docs]
def open_db(self):
""" Open the database. """
# get db file
# self.dbFile = '/Users/pfesesanivanzyl/dran/HART26DATA.db'
# self.dbFile = '/Users/pfesesanivanzyl/software/working/HartSoftware/dran/CALDB.db'
self.dbFile = self.open_file_name_dialog("*.db")
# get db file
# self.dbFile = self.open_file_name_dialog("*.db")
if self.dbFile == None:
print("No file selected")
else:
self.time_ui.EdtDB.setText(self.dbFile)
self.time_ui.EdtDB.setEnabled(True)
# open the database and get tables
msg_wrapper("debug", self.log.debug,f'\nOpening database: {self.dbFile}')
self.db = SQLiteDB(dbPath=self.dbFile, log=self.log)
self.db.create_db()
# populate columns
self.dbName=self.dbFile.split('/')[-1]
self.tables=sorted(self.db.get_table_names(self.dbName))
# self.time_ui.comboBoxTables.
# print(self.tables)
# sys.exit()
# self.db.close_db()
# self.populate_cols()
#self.time_ui.comboBoxColsX.setEnabled(True)
#self.time_ui.comboBoxColsY.setEnabled(True)
self.enable_time_buttons()
self.connect_time_buttons()
self.populate_cols()
self.db.close_db()
# print('Closing database: ',self.dbFile)
[docs]
def on_table_name_changed(self):
""" update combobox when table name changes. """
self.populate_cols()
[docs]
def populate_cols(self):
""" Populate x and y columns with data from tables. """
# TODO: Double check this makes sense
# open the database and get tables
# print(self.dbFile)
self.db = SQLiteDB(dbPath=self.dbFile, log=self.log)
self.db.create_db()
self.db.close_db()
# get column names from currently selected table
self.table = self.time_ui.comboBoxTables.currentText()
# print('Reading table: ', self.table)
# sys.exit()
# self.db.close_db()
print('table before issue: ', self.table)
# print(self.tables)
if not self.table:
print("Issue: ")
# force the combobox to accept items
#self.time_ui.comboBoxTables.addItems(self.tables)
#self.time_ui.comboBoxTables.addItems(self.tables)
# self.time_ui.comboBoxTables.clear()
self.time_ui.comboBoxTables.clear()
self.time_ui.comboBoxTables.addItems(self.tables)
# print(self.tables)
# sys.exit()
# lt = len(self.tables)
# lc = self.time_ui.comboBoxTables.count()
# x = [self.time_ui.comboBoxTables.itemText(i) for i in range(self.time_ui.comboBoxTables.count())]
self.table = self.time_ui.comboBoxTables.currentText()
# print('lt, lc, x: ',lt,lc,x)
# print('curtxt now: ',self.time_ui.comboBoxTables.currentText())
# print(self.table)
# print(self.tables)
# sys.exit()
# print('Outs')
# sys.exit()
# print(self.table)
self.db = SQLiteDB(dbPath=self.dbFile, log=self.log)
self.db.create_db()
self.colInd, self.colName, self.colType = self.db.get_all_table_coloumns(self.table)
# self.db.close_db()
# print('table: ', self.table)
# print('cols: ',self.colName,'\n')
# self.db = SQLiteDB(databaseName=self.dbFile, log=self.log)
# self.db.create_db()
# select only numerical cols with a few exceptions
colDict={}
self.colNames=[]
for i in range(len(self.colInd)):
colDict[self.colInd[i]] = self.colName[i]
self.colNames.append(self.colName[i])
# get rows
self.rows = self.db.get_rows_of_cols(self.table, self.colNames)
# self.db.close_db()
print('creating dataframe from table rows')
self.df = pd.DataFrame(list(self.rows), columns=self.colNames)
self.df = self.df.sort_values('FILENAME')
self.orig_df=self.df # copy of df
# try:
try:
self.df["OBSDATE"]=pd.to_datetime(self.df["OBSDATE"],format="%Y-%m-%d")
except:
# print(self.df['OBSDATE'])
self.df["OBSDATE"] = pd.to_datetime(self.df["OBSDATE"]).dt.date
print(self.df['OBSDATE'])
# sys.exit()
# except:
# pass
# self.df.fillna(value=np.nan, inplace=True)
# self.time_ui.comboBoxColsX.clear()
# self.time_ui.comboBoxColsY.clear()
# self.time_ui.comboBoxColsYerr.clear()
# print(self.colNames)
# sys.exit()
plotCols=[]
for name in self.colNames:
#'id', 'FILENAME', 'FILEPATH', 'HDULENGTH', 'CURDATETIME', 'OBSDATE', 'OBSTIME',
# 'OBSDATETIME', 'OBJECT', 'LONGITUD', 'LATITUDE', 'COORDSYS', 'EQUINOX', 'RADECSYS',
# 'OBSERVER', 'OBSLOCAL', 'PROJNAME', 'PROPOSAL', 'TELESCOP', 'UPGRADE', 'FOCUS', 'TILT',
# 'TAMBIENT', 'PRESSURE', 'HUMIDITY', 'WINDSPD', 'SCANDIR', 'POINTING', 'FEEDTYPE',
# 'BMOFFHA', 'BMOFFDEC', 'HPBW', 'FNBW', 'SNBW', 'DICHROIC', 'PHASECAL', 'NOMTSYS', 'FRONTEND',
# 'CENTFREQ', 'BANDWDTH', 'INSTRUME', 'INSTFLAG', 'SCANDIST', 'SCANTIME', 'TCAL1', 'TCAL2',
# 'HZPERK1', 'HZKERR1', 'HZPERK2', 'HZKERR2', 'TSYS1', 'TSYSERR1', 'TSYS2', 'TSYSERR2',
# 'BEAMTYPE', 'LOGFREQ', 'ELEVATION', 'ZA', 'MJD', 'HA', 'PWV', 'SVP', 'AVP', 'DPT', 'WVD',
# 'ATMOSABS', 'OBSNAME', 'OLTA', 'OLTAERR', 'OLBRMS', 'OLSLOPE', 'OLMIDOFFSET', 'OLFLAG',
# 'OLPEAKLOC', 'OLBASELEFT', 'OLBASERIGHT', 'OLS2N', 'ORTA', 'ORTAERR', 'ORBRMS', 'ORSLOPE',
# 'ORMIDOFFSET', 'ORFLAG', 'ORPEAKLOC', 'ORBASELEFT', 'ORBASERIGHT', 'ORS2N', 'SRC'
if 'id' in name or 'LOGFREQ' in name or 'CURDATETIME' in name or \
'FILE' in name or 'OBSD' in name \
or 'MJD' in name or 'OBS' in name or 'OBJ' in name or 'id' == name \
or 'RAD' in name or 'TYPE' in name or 'PRO' in name or 'TELE' in\
name or 'UPGR' in name or 'INST' in name or \
'SCANDIR' in name or 'SRC' in name or 'COORDSYS' in name or 'LONGITUD' in name \
or 'LATITUDE' in name or 'POINTING' in name \
or 'DICHROIC' in name \
or 'PHASECAL' in name or 'HPBW' in name or 'FNBW' in name or 'SNBW' in name\
or 'FRONTEND' in name or 'BASE' in name:
# 'HZK' in name or 'HZP' in name:
#or 'ERR' in name or 'LD' in name or 'RD' in name or 'HDU' in name or 'EQUINOX' in name or 'SOURCE' in name or
#or 'HA' in name or 'BMOFFHA' in name or 'BMOFFDEC' in name or 'NOMTSYS' in name or 'BANDWDTH' in name
# or 'ELEVATION' in name
pass
else:
plotCols.append(name)
errcols=[]
for name in self.colNames:
if 'ERR' in name or 'LD' in name or 'RD' in name or 'DS' in name or 'DN' in name or 'DO' in name or 'DF' in name or 'DC' in name:
if 'WINDSPD' in name or 'ADOPTED' in name or 'DATE' in name:
pass
else:
errcols.append(name)
else:
pass
yerr=['None']
self.time_ui.comboBoxColsX.clear()
self.time_ui.comboBoxColsX.clear()
self.time_ui.comboBoxColsX.addItems(['OBSDATE','MJD','HA','ELEVATION'])
self.time_ui.comboBoxColsY.clear()
self.time_ui.comboBoxColsY.clear()
self.time_ui.comboBoxColsY.addItems(plotCols)
self.yErr=list(yerr)+list(errcols)
self.time_ui.comboBoxColsYerr.clear()
self.time_ui.comboBoxColsYerr.clear()
self.time_ui.comboBoxColsYerr.addItems(self.yErr)
self.db.close_db()
[docs]
def on_fit_changed(self):
""" Toggle labels and edit boxes on or off when fit type is changed."""
if self.time_ui.comboBoxFitTypes.currentText()=="Spline":
self.time_ui.LblSplKnots.setVisible(True)
self.time_ui.EdtSplKnots.setVisible(True)
self.time_ui.EdtEndDate.setVisible(False)
self.time_ui.EdtStartDate.setVisible(False)
self.time_ui.LblEndDate.setVisible(False)
self.time_ui.LblStartDate.setVisible(False)
else:
self.time_ui.LblSplKnots.setVisible(False)
self.time_ui.EdtSplKnots.setVisible(False)
self.time_ui.EdtEndDate.setVisible(True)
self.time_ui.EdtEndDate.setEnabled(True)
self.time_ui.EdtStartDate.setVisible(True)
self.time_ui.EdtStartDate.setEnabled(True)
self.time_ui.LblEndDate.setVisible(True)
self.time_ui.LblStartDate.setVisible(True)
[docs]
def refresh_list(self):
"""Refresh list if database updated"""
#TODO: enable this function
pass
[docs]
def enable_time_buttons(self):
""" Enable time buttons. """
self.time_ui.comboBoxTables.setEnabled(True)
self.time_ui.comboBoxColsX.setEnabled(True)
self.time_ui.comboBoxColsY.setEnabled(True)
self.time_ui.comboBoxColsYerr.setEnabled(True)
self.time_ui.EdtSplKnots.setEnabled(True)
self.time_ui.BtnPlot.setEnabled(True)
self.time_ui.comboBoxFilters.setEnabled(True)
self.time_ui.EdtFilter.setEnabled(True)
self.time_ui.BtnFilter.setEnabled(True)
self.time_ui.comboBoxFitTypes.setEnabled(True)
self.time_ui.comboBoxOrder.setEnabled(True)
self.time_ui.BtnFit.setEnabled(True)
# self.time_ui.BtnSavePlot.setEnabled(True)
self.time_ui.BtnDelPoint.setEnabled(True)
self.time_ui.BtnDelBoth.setEnabled(True)
self.time_ui.BtnResetPoint.setEnabled(True)
self.time_ui.BtnReset.setEnabled(True)
self.time_ui.BtnRefreshDB.setEnabled(True)
self.time_ui.BtnUpdateDB.setEnabled(True)
self.time_ui.BtnSaveDB.setEnabled(True)
self.time_ui.BtnQuit.setEnabled(True)
[docs]
def connect_time_buttons(self):
""" connect button key press to widgets. """
msg_wrapper("debug", self.log.debug,"Connecting buttons to main canvas")
self.time_ui.BtnPlot.clicked.connect(self.plot_cols)
self.time_ui.BtnFilter.clicked.connect(self.filter_timeseries_data)
self.time_ui.BtnFit.clicked.connect(self.fit_timeseries)
self.time_ui.BtnDelPoint.clicked.connect(self.update_point)#point)
self.time_ui.BtnDelBoth.clicked.connect(self.update_db_all)#point)
# self.time_ui.BtnResetPoints.clicked.connect()
# self.time_ui.BtnSavePlot.clicked.connect()
# self.time_ui.BtnQuit.clicked.connect()
self.time_ui.BtnQuit.clicked.connect(self.update_db)
self.time_ui.BtnReset.clicked.connect(self.reset_timeseries)
#self.time_ui.BtnOpenDB.clicked.connect(self.open_db)
self.time_ui.BtnRefreshDB.clicked.connect(self.refresh_db)
# self.time_ui.BtnUpdateDB.clicked.connect()
self.time_ui.BtnSaveDB.clicked.connect(self.save_time_db)
#self.time_ui.BtnFit.clicked.connect(self.fit_timeseries)
[docs]
def plot_cols(self,col="",xcol="",ycol="",yerr=""):
""" Plot database columns. """
self.db = SQLiteDB(dbPath=self.dbFile, log=self.log)
self.db.create_db()
# get column names from currently selected table
self.table = self.time_ui.comboBoxTables.currentText()
# print('Table: ', self.table)
# sys.exit()
if not self.table:
print("Issue: ")
self.time_ui.comboBoxTables.clear()
self.time_ui.comboBoxTables.clear()
print('Tables 1: ', self.tables)
self.time_ui.comboBoxTables.addItems(self.tables)
lt = len(self.tables)
lc = self.time_ui.comboBoxTables.count()
x = [self.time_ui.comboBoxTables.itemText(i) for i in range(self.time_ui.comboBoxTables.count())]
self.colInd, self.colName, self.colType = self.db.get_all_table_coloumns(self.table)
# select only numerical cols with a few exceptions
colDict={}
self.colNames=[]
for i in range(len(self.colInd)):
colDict[self.colInd[i]] = self.colName[i]
self.colNames.append(self.colName[i])
# get rows
self.rows = self.db.get_rows_of_cols(self.table, self.colNames)
# sys.exit()
# print([d for d in self.colNames if 'DTA' in d].replace('DTA','ERRTA'))
# sys.exit()
self.df = pd.DataFrame(list(self.rows), columns=self.colNames)
self.df=self.df.sort_values('FILENAME')
try:
self.df["OBSDATE"]=pd.to_datetime(self.df["OBSDATE"],format="%Y-%m-%d")
except:
pass
# get col names
if xcol!="" and ycol!="" and yerr!="":
xCol=xcol
yCol=ycol
yErr=yerr
else:
xCol = self.time_ui.comboBoxColsX.currentText()
yCol = self.time_ui.comboBoxColsY.currentText()
yErr = self.time_ui.comboBoxColsYerr.currentText()
# print('yCol: ', yErr)
if xCol == yCol:
print("\nYou cannot plot the same column on X and Y\n")
else:
# yErr = self.time_ui.comboBoxColsYerr.currentText()
# get col data
# self.db.set_table_name(self.table)
print(f"Plotting {xCol} vs {yCol} in table {self.table}")
if yErr=="None":
yerr=np.zeros(len(self.df))
else:
yerr=self.df[yErr]
# print(self.df[yErr])
# self.df[yErr][self.df[yErr] < 0] = np.nan
# print(self.df[yErr])
# sys.exit()
# yerr=self.df[yErr].astype(float)
# # condition : When True, yield x, otherwise yield y.
# yerr=np.where(yerr<0, np.nan, yerr)
# self.df[yErr][self.df[yErr] < 0] = np.nan
# print(self.df['OBSDATE'])
t=self.df[self.df[xCol].isnull()]
# print(t)
# print(t[['id','FILENAME','OBSDATE']])
# Fix the issue with OBSDATE NOT working properly
for i in range(len(t)):
fn=t['FILENAME'].iloc[i]
# print(i,fn)
date=fn[:8].split('d')
year=date[0]
day=date[1]
# print(date)
# converting to date
res = datetime.strptime(year + "-" + day, "%Y-%j").strftime("%Y-%m-%d")
print(res)
sys.exit()
# id=int(t['id'].iloc[i])
# print(id)
t.at[t.index[i],'OBSDATE'] = res
self.df.at[self.df.index[i],'OBSDATE'] = res
# print(t[['id','FILENAME','OBSDATE']])
#TODO: clean this up
# self.df[xCol].fillna(value=0, inplace=True)
# self.df[xCol].fillna(value=pd.NaT, inplace=True)
# self.df[yCol].fillna(value=np.nan, inplace=True)
# print(self.df)
# sys.exit()
if xCol == 'OBSDATE':
# convert to datetime
self.df[xCol] = pd.to_datetime(self.df[xCol])#, errors='coerce')
# print(self.df[xCol])
# sys.exit()
self.Canvas.plot_fig(self.df[xCol],self.df[yCol],xCol,yCol,data=self.df,yerr=yerr) #,data=self.)
# sys.exit()
# <class 'pandas._libs.tslibs.nattype.NaTType'>
self.db.close_db()
[docs]
def filter_timeseries_data(self):
""" Filter the data"""
#TODO: fix this - implement rms and binning cuts
# get current text from combobox and bins
filterText = self.time_ui.comboBoxFilters.currentText()
filterCut = self.time_ui.EdtFilter.text()
if filterText=="Type":
print(f'please select a filter type\n')
else:
if filterText == ">" or filterText == ">=" or filterText == "<" or filterText == "<=":
try:
cut=float(filterCut)
except:
print(f'{filterCut} is an invalid entry for filter {filterText}')
cut=""
if cut=="":
pass
else:
print(f'filter everything {filterText} {filterCut}')
# get x and y
if len(self.Canvas.x) == 0:
print(self.Canvas.x,self.Canvas.y)
print('\nYou need to plot the data first\n')
else:
x=np.array(self.Canvas.x)
y=np.array(self.Canvas.y).astype(float)
xCol=self.Canvas.xlab
yCol=self.Canvas.ylab
# print(self.df[['id','FILENAME','MJD']])
# print(f'cut: {cut}')
if filterText == ">":
cuts=np.where(y>float(cut))[0]
elif filterText == ">=":
cuts=np.where(y>=float(cut))[0]
elif filterText == "<":
cuts=np.where(y<float(cut))[0]
elif filterText == "<=":
cuts=np.where(y<=float(cut))[0]
else:
print('Invalid cut detected')
sys.exit()
if len(cuts)>0:
print(f'Dropping rows at index {cuts}')
#df_cuts=self.df[self.df[xCol]>]
self.df=self.df.drop(self.df.index[cuts])
#print(self.df)
self.deleted.append(cuts)
print(f'deleted: {self.deleted}')
# re-plot
print(self.df['SRC'])
print(self.df['FILENAME'][:8])
# sys.exit()
self.Canvas.plot_fig(self.df[xCol], self.df[yCol], xCol, yCol, data=self.df,title=f"Plot of {self.df['SRC'].iloc[-1]} - {xCol} vs {yCol}") #{self.df['FILENAME'][:8]}")
else:
print(f"No values found for condition {filterText} {filterCut} ")
elif filterText == "rms cuts":
print('RMS cuts not implemented yet.')
# print("Performing rms cuts")
# # ================================================
# # Clean the data using rms cuts
# # ================================================
# # get df
# xCol=self.Canvas.xlab
# yCol=self.Canvas.ylab
# # need to run this once else it destroys data
# self.df[yCol]=self.df[yCol].replace('nan',0)
# self.df[yCol].fillna(0,inplace=True)
# x=list(self.df[xCol])#np.array(self.Canvas.x)
# y=list(self.df[yCol])#np.array(self.Canvas.y).astype(float)
# scanLen = len(x)
# if self.df["CENTFREQ"][int(len(self.df)/2)] <= 3000:
# noTotScans = 2
# else:
# noTotScans = 6
# # spline the data
# spl = fit.spline(x, y)
# pl.plot(x,y)
# pl.plot(x,spl)
# pl.show()
# sys.exit()
# if len(spl)==0:
# pass
# else:
# # print('spl: ',spl)
# # print('y: ',y)
# #sys.exit()
# print(self.df['FILENAME'])
# scanResBefore, scanRmsBefore = fit.calc_residual(spl, y)
# finX, finY, scanRmsAfter, scanResAfter, finMaxSpl, finspl, pt, names = fit.clean_data_iterative_fitting(x, y, scanLen, scanResBefore, scanRmsBefore, self.log, x2=list(self.df["FILENAME"]))
# self.write(f'RMS before: after -> {scanRmsBefore:.3f}: {scanRmsAfter:.3f}','info')
# self.Canvas.clear_figure() # Update figures
# #self.df[xCol]=""
# self.df[yCol]=np.nan
# #for j in range(len(list(self.df["FILENAME"]))):
# err=self.time_ui.comboBoxColsYerr.currentText()
# for i in range(len(names)):
# self.df.loc[self.df['FILENAME']==names[i] ,yCol]=finY[i]
# #self.df.loc[self.df['FILENAME']==names[i] ,err]=finY[i]
# # #self.update_plot(self.x, self.y, "After rms cuts",
# # # self.res, "smoothed data", "smoothed residual") # Update plot
# # if self.time_ui.errorCheckBox.isChecked() == True:
# # self.Canvas.plot_fig_errs(self.df[xCol], self.df[yCol], xCol, yCol, data=self.df,errs=self.df[err])
# # else:
# # self.Canvas.plot_fig(self.df[xCol], self.df[yCol], xCol, yCol, data=self.df)
# # smallestRmsFound=True
# # self.df.fillna(0,inplace=True)
# # self.df=self.df.replace('nan',0)
# # for col in list(self.df.columns):
# # if "FILENAME" in col or "OBSDATE":# in col or ""
# # pass
# # else:
# # self.df[col] = pd.to_numeric(self.df[col],downcast=float)
# # out_df=self.remove_rfi(self.df,xCol,yCol)
# # #print(out_df.columns)
# # #if (len(out_df) == len(self.df)):
# # # print("Failed to remove RFI, probably data length too short")
# # #else:
# # # merge to original
# # self.df= pd.merge(self.df, out_df, how='left', on=['MJD'])
# # #self.rename_col(self.df,"_x")
# # self.rename_col(self.df,"_y")
# # # re-plot
# # self.Canvas.clear_figure()
# # self.Canvas.plot_fig(self.df[xCol], self.df[yCol], xCol, yCol, data=self.df)
# # rms3_ =
# # scanLen = len(self.Canvas.x)
# # if self.data["CENTFREQ"] <= 3000:
# # noTotScans = 2
# # else:
# # noTotScans = 6
# # # spline the data
# # spl = fit.spline(self.Canvas.x, self.Canvas.y)
# # scanRes, self.Canvas.x, self.Canvas.y, res, finspl, rmsb, rmsa = fit.cleanRmsCuts2(
# # spl, self.Canvas.x, self.Canvas.y, scanLen, self.log)
# # self.write(f'RMS before: after -> {rmsb}: {rmsa,}','info')
# # self.Canvas.clear_figure() # Update figures
# # # re-plot
# # xCol=self.Canvas.xlab
# # yCol=self.Canvas.ylab
# #self.Canvas.plot_fig(self.df[xCol] ,self.df[yCol], xCol, yCol, data=self.df)
elif filterText == "binning":
print("Binning not implemented yet\n")
[docs]
def fit_timeseries(self):
# get fit type and fit order
fitType = self.time_ui.comboBoxFitTypes.currentText()
fitOrder = self.time_ui.comboBoxOrder.currentText()
print(f"Fitting: {fitType}, order: {fitOrder}")
if fitOrder != "Order" and fitType!="Type":
xCol=self.Canvas.xlab
yCol=self.Canvas.ylab
print("col: ",xCol)
if xCol !="MJD":
print('Require x-axis as MJD')
else:
x=np.array(self.Canvas.x).astype(float)
y=np.array(self.Canvas.y).astype(float)
if fitType == "Polynomial":
try:
tstart=int(self.time_ui.EdtStartDate.text())
except:
tstart="nodate"
try:
tend=int(self.time_ui.EdtEndDate.text())
except:
tend="nodate"
if tstart == "nodate" and tend=="nodate":
print(f'\nstart; {x[0]}, end: {x[-1]}')
xm,model,res,rma,coeffs=fit.calc_residual_and_rms_fit(x,y,int(fitOrder))
ind=(np.where(model==max(model))[0])[0]
# print(ind)
# sys.exit()
print("\n","*"*20)
print(f'tstart: {x[0]}, tend: {x[-1]}')
print(f'ymin:{min(y):.3f}, ymax: {max(y):.3f}')
print('> model')
print(f'ymin:{min(model):.3f}, ymax: {max(model):.3f}, x @ ymax: {xm[ind]}')
print(f'rms of fit: {rma:.3f}')
print('coeffs: ',coeffs)
print("*"*20,"\n")
self.Canvas.plot_dual_fig(x,y,xm,model,'data','model','Plot of data vs polynomial model')
elif tstart == "nodate" and tend!="nodate":
print(f'start; {x[0]}, end: {tend}')
if tend < x[0]:
print(f"End date ({tend}) needs to be bigger than start date ({x[0]})")
else:
v=np.where(x <= tend)[0]
x1=x[v]
y1=y[v]
xm,model,res,rma,coeffs=fit.calc_residual_and_rms_fit(x1,y1,int(fitOrder))
ind=(np.where(model==max(model))[0])[0]
print("\n","*"*20)
print(f'tstart: {x[0]}, tend: {tend}')
print(f'ymin:{min(y):.3f}, ymax: {max(y):.3f}')
print('> model')
print(f'ymin:{min(model):.3f}, ymax: {max(model):.3f}, x @ ymax: {xm[ind]}')
print(f'rms of fit: {rma:.3f}')
print('coeffs: ',coeffs)
print("*"*20,"\n")
self.Canvas.plot_dual_fig(x,y,xm,model,'data','model','Plot of data vs polynomial model')
elif tstart != "nodate" and tend=="nodate":
print(f'start; {tstart}, end: {x[-1]}')
if tstart > x[-1]:
print(f"Start date ({tstart}) needs to be smaller than end date ({x[-1]})")
else:
v=np.where(x >= tstart)[0]
x1=x[v]
y1=y[v]
xm,model,res,rma,coeffs=fit.calc_residual_and_rms_fit(x1,y1,int(fitOrder))
ind=(np.where(model==max(model))[0])[0]
print("\n","*"*20)
print(f'tstart: {tstart}, tend: {x[-1]}')
print(f'ymin:{min(y):.3f}, ymax: {max(y):.3f}')
print('> model')
print(f'ymin:{min(model):.3f}, ymax: {max(model):.3f}, x @ ymax: {xm[ind]}')
print(f'rms of fit: {rma:.3f}')
print('coeffs: ',coeffs)
print("*"*20,"\n")
self.Canvas.plot_dual_fig(x,y,xm,model,'data','model','Plot of data vs polynomial model')
elif tstart != "nodate" and tend!="nodate":
print(f'start; {tstart}, end: {tend}')
if tstart > tend:
print(f"Start date ({tstart}) needs to be smaller than end date ({tend})")
else:
v=np.where((x >= tstart)&(x <= tend))[0]
x1=x[v]
y1=y[v]
xm,model,res,rma,coeffs=fit.calc_residual_and_rms_fit(x1,y1,int(fitOrder))
ind=(np.where(model==max(model))[0])[0]
print("\n","*"*20)
print(f'tstart: {tstart}, tend: {tend}')
print(f'ymin:{min(y):.3f}, ymax: {max(y):.3f}')
print('> model')
print(f'ymin:{min(model):.3f}, ymax: {max(model):.3f}, x @ ymax: {xm[ind]}')
print(f'rms of fit: {rma:.3f}')
print('coeffs: ',coeffs)
print("*"*20,"\n")
#print('coeffs: ',coeffs)
self.Canvas.plot_dual_fig(x,y,xm,model,'data','model','Plot of data vs polynomial model')
elif fitType == "Spline":
knots=int(self.time_ui.EdtSplKnots.text())
if knots<9:
knots=9
xm,model=fit.spline_fit(x, y, int(knots),int(fitOrder))
self.Canvas.plot_dual_fig(x,y,xm,model,'data','model','Plot of data vs spline model')
elif fitOrder == "Order" and fitType!="Type":
print("\nNo fit order selected\n")
elif fitOrder != "Order" and fitType=="Type":
print("\nNo fit type selected\n")
else:
print("\nNo fit parameters selected\n")
[docs]
def update_point(self):
#TODO: fix the format
# update db on change
# check last selected points
fit_points= self.Canvas.fit_points
click_index = self.Canvas.click_index
print('\n')
print("+"*30)
print(f"\nDeleting \n-point: {fit_points} \n-at index {click_index}")
#self.deleted[click_index].append([click_index, fit_points])
# print('\ndf length Before deleting: ',len(self.df))
xCol = self.Canvas.xlab
yCol = self.Canvas.ylab
print('\n','-'*30)
t = Time(fit_points[0][0])
#date=t
doy = t.strftime('%j')
#mjd=t.mjd
xp=self.df[xCol]
yp=self.df[yCol]
date_str=f'{str(fit_points[0][0])[:4]}d{str(doy)}'
i=int(click_index[0]) # point index to delete
# print('Deleting: ')
# find what i is in the db
id=self.df['id'].iloc[i]
mjd=self.df['MJD'].iloc[i]
FN=self.df['FILENAME'].iloc[i]
print('\n-mjd: ',mjd,'\n-file: ',FN) #date_str
print(f"\n-id: {self.df['id'].iloc[i]}, \n-index: {i}")
print(f'\n> Setting {yCol}: {self.df.iloc[i][yCol]}, to np.nan')
try:
self.df.at[i,yCol]=np.nan
except ValueError:
self.df.at[i,yCol]=0.0
print(f'\n+ New {yCol}: {self.df.iloc[i][yCol]}\n')
# change plot
tablename = self.time_ui.comboBoxTables.currentText()
xcol=self.time_ui.comboBoxColsX.currentText()
ycol=self.time_ui.comboBoxColsY.currentText()
#self.dbFile
# self.db = SQLiteDB(self.dbFile, log=self.log)
# self.db.create_db()
db = SQLiteDB(self.dbFile, log=self.log)
db.create_db()
try:
# srcType=self.df['OBJECTTYPE'].iloc[i]
self.write(f"Updating {self.dbFile} table: {tablename}",'debug') # , type: {srcType}
print('\n-BEAMTYPE: ',self.df['BEAMTYPE'][i]) #,'\n-srcType: ',srcType)
except:
pass
# update the database
cols=list(self.df.columns)
stmt=f"UPDATE '{tablename}' SET "
dct={}
print(list(self.df.columns))
dct['OBJECT'] = self.df['OBJECT'].iloc[i]
try:
dct['FRONTEND']=self.df['FRONTEND'].iloc[i]
except:
dct['BEAMTYPE']=self.df['BEAMTYPE'].iloc[i]
dct['FRONTEND']=self.df['BEAMTYPE'].iloc[i]
try:
dct['ATMOS_ABSORPTION_CORR']=self.df['ATMOS_ABSORPTION_CORR'].iloc[i]
dct['SIZE_CORRECTION_FACTOR']=self.df['SIZE_CORRECTION_FACTOR'].iloc[i]
except:
pass
bms=['A','B']
print("YCOL: ",ycol)
if ycol == "SLTA" or ycol=="ASLTA" or ycol=="BSLTA" or ycol=="SLTAERR" or ycol=="ASLTAERR" or ycol=="BSLTAERR" or ycol == "SRTA" or ycol=="ASRTA" or ycol=="BSRTA" or ycol == "SRTAERR" or ycol=="ASRTAERR" or ycol=="BSRTAERR":
if ycol[0]=="A":
k=0
elif ycol[0]=="B":
k=1
else:
k=""
if ycol=="SLTA" or ycol=="ASLTA" or ycol=="BSLTA" or ycol=="SLTAERR" or ycol=="ASLTAERR" or ycol=="BSLTAERR":
pol="L"
elif ycol == "SRTA" or ycol=="ASRTA" or ycol=="BSRTA" or ycol == "SRTAERR" or ycol=="ASRTAERR" or ycol=="BSRTAERR":
pol="R"
print()
# try:
hps=np.nan
err_hps=np.nan
b=''
try:
hpn=self.df[f'N{pol}TA'].iloc[i]
b='s'
except:
hpn=self.df[f'{bms[k]}N{pol}TA'].iloc[i]
b='d'
try:
err_hpn=self.df[f'N{pol}TAERR'].iloc[i]
b='s'
except:
err_hpn=self.df[f'{bms[k]}N{pol}TAERR'].iloc[i]
b='d'
try:
on=self.df[f'O{pol}TA'].iloc[i]
b='s'
except:
on=self.df[f'{bms[k]}O{pol}TA'].iloc[i]
b='d'
try:
err_on=self.df[f'O{pol}TAERR'].iloc[i]
b='s'
except:
err_on=self.df[f'{bms[k]}O{pol}TAERR'].iloc[i]
b='d'
if b=='':
print('something failed')
sys.exit()
elif b=='s':
print('dct: ',dct)
print(f"Using - 'N{pol}TA': {hpn}, 'N{pol}TAERR': {err_hpn}, 'O{pol}TA': {on}, 'O{pol}TAERR': {err_on} to recalculate ")
pc, corrTa, errCorrTa=cp.calibrate( 0, 0, hpn, err_hpn, on, err_on, dct)
stmt=stmt+f"S{pol}TA='{hps}', S{pol}TAERR='{err_hps}', S{pol}S2N='{np.nan}', O{pol}PC='{pc}', CO{pol}TA='{corrTa}', CO{pol}TAERR='{errCorrTa}', "
if pc==0.0:
print(f'- setting O{pol}PC=np.nan')
pc=np.nan
if corrTa==0.0:
corrTa=np.nan
print(f'- setting CO{pol}TA=np.nan')
if errCorrTa==0.0:
errCorrTa=np.nan
print(f'- setting CO{pol}TAERR=np.nan')
elif b=='d':
# dual-beam
print(f"Using - '{bms[k]}N{pol}TA': {hpn}, '{bms[k]}N{pol}TAERR': {err_hpn}, '{bms[k]}O{pol}TA': {on}, '{bms[k]}O{pol}TAERR': {err_on} to recalculate ")
pc, corrTa, errCorrTa=cp.calibrate(0, 0, hpn, err_hpn, on, err_on, dct)
stmt=stmt+f"{bms[k]}S{pol}TA='{np.nan}', {bms[k]}S{pol}TAERR='{np.nan}', {bms[k]}S{pol}S2N='{np.nan}', {bms[k]}O{pol}PC='{pc:.3f}', {bms[k]}CO{pol}TA='{corrTa:.3f}', {bms[k]}CO{pol}TAERR='{errCorrTa:.3f}', "
if pc==0.0:
pc=np.nan
print(f'- setting {bms[k]}O{pol}PC=np.nan')
if corrTa==0.0:
corrTa=np.nan
print(f'- setting {bms[k]}CO{pol}TA=np.nan')
if errCorrTa==0.0:
errCorrTa=np.nan
print(f'- setting {bms[k]}CO{pol}TAERR=np.nan')
elif ycol == "NLTA" or ycol=="ANLTA" or ycol=="BNLTA" or ycol == "NLTAERR" or ycol=="ANLTAERR" or ycol=="BNLTAERR" or ycol == "NRTA" or ycol=="ANRTA" or ycol=="BNRTA" or ycol == "NRTAERR" or ycol=="ANRTAERR" or ycol=="BNRTAERR":
if ycol[0]=="A":
k=0
elif ycol[0]=="B":
k=1
else:
k=""
if ycol=="NLTA" or ycol=="ANLTA" or ycol=="BNLTA" or ycol == "NLTAERR" or ycol=="ANLTAERR" or ycol=="BNLTAERR":
pol="L"
elif ycol == "NRTA" or ycol=="ANRTA" or ycol=="BNRTA" or ycol == "NRTAERR" or ycol=="ANRTAERR" or ycol=="BNRTAERR":
pol="R"
print()
try:
hps=self.df[f'S{pol}TA'].iloc[i]
err_hps=self.df[f'S{pol}TAERR'].iloc[i]
hpn=np.nan
err_hpn=np.nan
on=self.df[f'O{pol}TA'].iloc[i]
err_on=self.df[f'O{pol}TAERR'].iloc[i]
# single-beam
# if "TAR" in self.df['OBJECTTYPE'].iloc[i]:
print(f"Using - 'S{pol}TA': {hps}, 'S{pol}TAERR': {err_hps}, 'O{pol}TA': {on}, 'O{pol}TAERR': {err_on} to recalculate ")
pc, corrTa, errCorrTa=cp.calibrate(hps, err_hps, 0, 0, on, err_on, dct)
stmt=stmt+f"N{pol}TA='{hpn}', N{pol}TAERR='{err_hpn}', O{pol}PC='{pc}', CO{pol}TA='{corrTa}', CO{pol}TAERR='{errCorrTa}', "
if pc==0.0:
pc=np.nan
print(f'- setting O{pol}PC=np.nan')
if corrTa==0.0:
corrTa=np.nan
print(f'- setting CO{pol}TA=np.nan')
if errCorrTa==0.0:
errCorrTa=np.nan
print(f'- setting CO{pol}TAERR=np.nan')
except:
hps=self.df[f'{bms[k]}S{pol}TA'].iloc[i]
err_hps=self.df[f'{bms[k]}S{pol}TAERR'].iloc[i]
hpn=np.nan
err_hpn=np.nan
on=self.df[f'{bms[k]}O{pol}TA'].iloc[i]
err_on=self.df[f'{bms[k]}O{pol}TAERR'].iloc[i]
# dual-beam
# if "TAR" in self.df['OBJECTTYPE'].iloc[i]:
print(f"Using - '{bms[k]}S{pol}TA': {hps}, '{bms[k]}S{pol}TAERR': {err_hps}, '{bms[k]}O{pol}TA': {on}, '{bms[k]}O{pol}TAERR': {err_on} to recalculate ")
pc, corrTa, errCorrTa=cp.calibrate(hps, err_hps, 0, 0, on, err_on, dct)
stmt=stmt+f"{bms[k]}N{pol}TA='{np.nan}', {bms[k]}N{pol}TAERR='{np.nan}', {bms[k]}N{pol}S2N='{np.nan}', {bms[k]}O{pol}PC='{pc}', {bms[k]}CO{pol}TA='{corrTa}', {bms[k]}CO{pol}TAERR='{errCorrTa}', "
print(pc, corrTa, errCorrTa)
if pc==0.0:
pc=np.nan
print(f'- setting {bms[k]}O{pol}PC=np.nan')
if corrTa==0.0:
corrTa=np.nan
print(f'- setting {bms[k]}CO{pol}TA=np.nan')
if errCorrTa==0.0:
errCorrTa=np.nan
print(f'- setting {bms[k]}CO{pol}TAERR=np.nan')
elif ycol == "OLTA" or ycol == "COLTA" or ycol == "COLTAERR" or ycol == "ACOLTA" or ycol == "ACOLTAERR" or ycol == "BCOLTA" or ycol == "BCOLTAERR" or ycol == "OLTAERR" or ycol == "AOLTA" or ycol == "BOLTA" or ycol=="ORTA" or ycol=="ORTAERR" or ycol=="AORTA" or ycol=="BORTA" or ycol == "OLPSS" or ycol=="AOLPSS" or ycol == "BOLPSS" or ycol == "ORPSS" or ycol=="AORPSS" or ycol == "BORPSS" or ycol == "OLDPSS" or ycol=="AOLDPSS" or ycol == "BOLDPSS" or ycol == "ORDPSS" or ycol=="AORDPSS" or ycol == "BORDPSS" or ycol == "CORTA" or ycol == "CORTAERR" or ycol == "ACORTA" or ycol == "ACORTAERR" or ycol == "BCORTA" or ycol == "BCORTAERR" :
if ycol[0]=="A":
k=0
elif ycol[0]=="B":
k=1
else:
k=""
if ycol=="OLTA" or ycol=="OLTAERR" or ycol=="AOLTA" or ycol=="BOLTA" or ycol == "OLPSS" or ycol=="AOLPSS" or ycol == "BOLPSS" or ycol == "OLDPSS" or ycol=="AOLDPSS" or ycol == "BOLDPSS" or ycol == "COLTA" or ycol == "COLTAERR" or ycol == "ACOLTA" or ycol == "ACOLTAERR" or ycol == "BCOLTA" or ycol == "BCOLTAERR" :
pol="L"
elif ycol=="ORTA" or ycol=="ORTAERR" or ycol=="AORTA" or ycol=="BORTA" or ycol == "ORPSS" or ycol=="AORPSS" or ycol == "BORPSS" or ycol == "ORDPSS" or ycol=="AORDPSS" or ycol == "BORDPSS" or ycol == "CORTA" or ycol == "CORTAERR" or ycol == "ACORTA" or ycol == "ACORTAERR" or ycol == "BCORTA" or ycol == "BCORTAERR" :
pol="R"
print()
# try:
# single-beam
# if "TAR" in self.df['OBJECTTYPE'].iloc[i]:
frq=self.df['CENTFREQ'].iloc[i]
if k=="":
if int(frq)<=3000:
stmt=stmt+f"O{pol}TA='{np.nan}', O{pol}TAERR='{np.nan}', O{pol}FLAG='{np.nan}', "
else:
stmt=stmt+f"O{pol}FLAG='{np.nan}', O{pol}TA='{np.nan}', O{pol}TAERR='{np.nan}', O{pol}PC='{np.nan}', CO{pol}TA='{np.nan}', CO{pol}TAERR='{np.nan}', S{pol}TA='{np.nan}', S{pol}TAERR='{np.nan}', N{pol}TA='{np.nan}', N{pol}TAERR='{np.nan}', "
else:
stmt=stmt+f"O{pol}FLAG='{np.nan}', {bms[k]}N{pol}TA='{np.nan}', {bms[k]}N{pol}TAERR='{np.nan}', {bms[k]}S{pol}TA='{np.nan}', {bms[k]}S{pol}TAERR='{np.nan}', {bms[k]}O{pol}TA='{np.nan}', {bms[k]}O{pol}TAERR='{np.nan}', {bms[k]}CO{pol}TA='{np.nan}', {bms[k]}CO{pol}TAERR='{np.nan} ',{bms[k]}S{pol}S2N='{np.nan}', {bms[k]}O{pol}PC='{np.nan}', "
# except:
# elif "CAL" in self.df['OBJECTTYPE'].iloc[i]:
# frq=self.df['CENTFREQ'].iloc[i]
# if k=="":
# if int(frq)<=3000:
# # pss, errPSS, appEff = cp.calc_pss(flux, Ta, errTa)
# stmt=stmt+f"O{pol}TA='{np.nan}', O{pol}TAERR='{np.nan}', O{pol}PSS='{np.nan}', O{pol}DPSS='{np.nan}', O{pol}S2N='{np.nan}', "
# else:
# stmt=stmt+f"N{pol}TA='{np.nan}', N{pol}TAERR='{np.nan}', S{pol}TA='{np.nan}', S{pol}TAERR='{np.nan}', O{pol}TA='{np.nan}', O{pol}TAERR='{np.nan}', O{pol}PC='{np.nan}', CO{pol}TA='{np.nan}', CO{pol}TAERR='{np.nan}', O{pol}PSS='{np.nan}', O{pol}DPSS='{np.nan}', O{pol}APPEFF='{np.nan}', "
# else:
# stmt=stmt+f"{bms[k]}N{pol}TA='{np.nan}', {bms[k]}N{pol}TAERR='{np.nan}', {bms[k]}S{pol}TA='{np.nan}', {bms[k]}S{pol}TAERR='{np.nan}', {bms[k]}O{pol}TA='{np.nan}', {bms[k]}O{pol}TAERR='{np.nan}', {bms[k]}CO{pol}TA='{np.nan}', {bms[k]}CO{pol}TAERR='{np.nan} ',{bms[k]}S{pol}S2N='{np.nan}', {bms[k]}O{pol}PC='{np.nan}', {bms[k]}O{pol}PSS='{np.nan}', {bms[k]}O{pol}DPSS='{np.nan}', {bms[k]}O{pol}APPEFF='{np.nan}', "
else:
# only delete that entry
# if O
stmt=stmt+f"{ycol}='{np.nan}', "
stmt=stmt[:-3]+f"' WHERE FILENAME = '{FN}' ;"
print(stmt,'\n')
# stmt=stmt[:-3]+f"' WHERE FILENAME = '{FN}' ;"
# print(stmt)
# sys.exit()
# self.db.c.execute(stmt)
# self.db.commit_changes()
# self.db.close_db()
db.c.execute(stmt)
db.commit_changes()
db.close_db()
self.Canvas.clear_figure()
self.plot_cols_df()
print("+-"*30)
[docs]
def update_db_all(self):
#TODO: Fix the format
# update db on change
# check last selected points
fit_points= self.Canvas.fit_points
click_index = self.Canvas.click_index
print('\n',"+"*30)
print(f"\nDeleting \n-point: {fit_points} \n-at index {click_index}")
#self.deleted[click_index].append([click_index, fit_points])
# print('\ndf length Before deleting: ',len(self.df))
xCol = self.Canvas.xlab
yCol = self.Canvas.ylab
print('\n','-'*30)
i=int(click_index[0]) # point index to delete
# find what i is in the db
id=self.df['id'].iloc[i]
#print(f'i: {ii}, id: {id}')
# i=ii#d
FN=self.df['FILENAME'].iloc[i]
# if 'TAR' in self.df['OBJECTTYPE'].iloc[i]:
t = Time(fit_points[0][0])
print(t)
# sys.exit()
#date=t
doy = t.strftime('%j')
#mjd=t.mjd
xp=self.df[xCol]
yp=self.df[yCol]
date_str=f'{str(fit_points[0][0])[:4]}d{str(doy)}'
# for i in range(len(self.df)):
# if date_str in self.df['FILENAME'].iloc[i]:
#print(date_str, self.df['FILENAME'].iloc[i])
#if self.df[yCol].iloc[i]==float(fit_points[0][1]):
print('Deleting: ')
print('\n-date: ',date_str,'\n-file: ',FN)
print(f"\n-id: {self.df['id'].iloc[i]}, \n-index: {i}")
# print('match: ', self.df['id'].iloc[i])#, self.df.iloc[i])
#self.df.drop(id,inplace=True)
print(f'\n> Setting {yCol}: {self.df.iloc[i][yCol]}, to np.nan')
#print(self.df.iloc[i][yCol])
try:
self.df.at[i,yCol]=np.nan
except ValueError:
self.df.at[i,yCol]=0.0
#self.df=self.df[~(self.df['id']==id)]
#print(self.df.iloc[i][yCol])
print(f'\n+ New {yCol}: {self.df.iloc[i][yCol]}')
# change plot
tablename = self.time_ui.comboBoxTables.currentText()
xcol=self.time_ui.comboBoxColsX.currentText()
ycol=self.time_ui.comboBoxColsY.currentText()
#self.dbFile
db = SQLiteDB(self.dbFile, log=self.log)
db.create_db()
try:
# srcType=self.df['OBJECTTYPE'].iloc[i]
self.write(f"Updating {self.dbFile} table: {tablename}",'debug') # , type: {srcType}
print('\n-BEAMTYPE: ',self.df['BEAMTYPE'][i]) #,'\n-srcType: ',srcType)
except:
pass
# update the database
cols=list(self.df.columns)
stmt=f"UPDATE '{tablename}' SET "
for col in cols:
if 'id' in col or 'LOGFREQ' in col or 'CURDATETIME' in col or \
'FILE' in col or 'OBSD' in col \
or 'MJD' in col or 'OBS' in col or 'OBJ' in col or 'id' == col \
or 'RAD' in col or 'TYPE' in col or 'PRO' in col or 'TELE' in\
col or 'UPGR' in col or 'INST' in col or \
'SCANDIR' in col or 'SRC' in col or 'COORDSYS' in col or 'LONGITUD' in col \
or 'LATITUDE' in col or 'POINTING' in col \
or 'DICHROIC' in col \
or 'PHASECAL' in col or 'HPBW' in col or 'FNBW' in col or 'SNBW' in col\
or 'FRONTEND' in col or 'BASE' in col:
pass
else:
stmt=stmt+f"{col}='{np.nan}', "
#
stmt=stmt[:-3]+f"' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
# print("updated CALDB")
print(tablename,xcol,ycol)
print('After deleting: ',len(self.df))
print(self.df.iloc[i][yCol])
self.Canvas.clear_figure()
self.plot_cols_df()
[docs]
def update_db(self):
#TODO: check that this works
# update db on change
# check last selected points
fit_points= self.Canvas.fit_points
click_index = self.Canvas.click_index
print('\n',"+"*30)
print(f"\nDeleting \n-point: {fit_points} \n-at index {click_index}")
# print('\ndf length Before deleting: ',len(self.df))
xCol = self.Canvas.xlab
yCol = self.Canvas.ylab
print('\n','-'*30)
try:
t = Time(fit_points[0][0])
except:
print('An error occurred, please ensure you click on a point')
sys.exit()
doy = t.strftime('%j')
xp=self.df[xCol]
yp=self.df[yCol]
date_str=f'{str(fit_points[0][0])[:4]}d{str(doy)}'
ii=int(click_index[0]) # point index to delete
print('Deleting: ')
try:
self.df=self.df.replace(None,np.nan)
except:
pass
try:
self.df=self.df.replace("nan",np.nan)
except:
pass
# find what i is in the db
id=self.df['id'].iloc[ii]
i=ii#d
FN=self.df['FILENAME'].iloc[i]
print('\n-date: ',date_str,'\n-file: ',FN)
print(f"\n-id: {self.df['id'].iloc[i]}, \n-index: {ii}")
print(f'\n> Setting {yCol}: {self.df.iloc[i][yCol]}, to np.nan')
self.df.at[i,yCol]=np.nan
print(f'\n+ New {yCol}: {self.df.iloc[i][yCol]}')
tablename = self.time_ui.comboBoxTables.currentText()
xcol=self.time_ui.comboBoxColsX.currentText()
ycol=self.time_ui.comboBoxColsY.currentText()
db = self.db #SQLiteDB(databaseName="CALDB.db", log=self.log)
db.create_db()
srcType=self.df['OBJECTTYPE'].iloc[i]
if srcType == "CAL":
self.write(f"Updating {self.dbFile} table: {tablename}, type: {srcType}",'info')
print('\n-BEAMTYPE: ',self.df['BEAMTYPE'][i],'\n-srcType: ',srcType)
# update the database
if 'S' in str(self.df['BEAMTYPE'].iloc[i]):
print(self.df['BEAMTYPE'].iloc[i])
if "13.0S" == self.df['BEAMTYPE'].iloc[i]:
if "TSYS1" in ycol or "TSYS2" in ycol:
if 'TSYS1' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTA='{np.nan}', OLTAERR='{np.nan}', OLFLAG=200, OLPSS='{pss}', OLDPSS={errpss}, OLAPPEFF={appeff}, WHERE FILENAME = '{FN}' ;"
if 'TSYS2' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTA='{np.nan}', ORTAERR='{np.nan}', ORFLAG=200, ORPSS='{pss}', ORDPSS={errpss}, ORAPPEFF={appeff}, WHERE FILENAME = '{FN}' ;"
elif "OLTA" in ycol :
pss,errpss,appeff = np.nan,np.nan,np.nan
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',OLTAERR='{np.nan}', OLFLAG=201, OLPSS='{pss}', OLDPSS='{errpss}', OLAPPEFF='{appeff}' WHERE FILENAME = '{FN}' ;"
elif "ORTA" in ycol :
pss,errpss,appeff = np.nan,np.nan,np.nan
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',ORTAERR='{np.nan}', ORFLAG=202, ORPSS='{pss}', ORDPSS='{errpss}', ORAPPEFF='{appeff}' WHERE FILENAME = '{FN}' ;"
elif "OLPSS" in ycol:
print('\nprint updating OLPSS')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLDPSS='{np.nan}', OLFLAG=201, OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
elif "ORPSS" in ycol:
print('\nprint updating ORPSS')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=202, ORDPSS='{np.nan}', ORAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
print('\nDONE')
print("+"*30,"\n")
else:
if "TSYS1" in ycol or "TSYS2" in ycol or 'LS2N' in ycol or 'RS2N' in ycol or 'PC' in ycol :# or "SLTA" in ycol:
if 'TSYS1' in ycol :
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTA='{np.nan}', SLTA='{np.nan}', \
NLTA='{np.nan}', OLTAERR='{np.nan}', SLTAERR='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, \
OLS2N='{np.nan}',OLPSS='{np.nan}', OLDPSS='{np.nan}', OLPC='{np.nan}', \
OLAPPEFF='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'OLPC' in ycol :
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTA='{np.nan}', SLTA='{np.nan}', \
NLTA='{np.nan}', OLTAERR='{np.nan}', SLTAERR='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, \
OLS2N='{np.nan}',OLPSS='{np.nan}', OLDPSS='{np.nan}', OLPC='{np.nan}', \
OLAPPEFF='{np.nan}', TSYS1='{np.nan}',COLTA='{np.nan}', COLTAERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'ORPC' in ycol :
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTA='{np.nan}', SLTA='{np.nan}', \
NLTA='{np.nan}', OLTAERR='{np.nan}', SLTAERR='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, \
OLS2N='{np.nan}',OLPSS='{np.nan}', OLDPSS='{np.nan}', OLPC='{np.nan}', \
OLAPPEFF='{np.nan}', TSYS2='{np.nan}',COLTA='{np.nan}', COLTAERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'TSYS2' in ycol :
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTA='{np.nan}', SRTA='{np.nan}', NRTA='{np.nan}', \
ORTAERR='{np.nan}', SRTAERR='{np.nan}', NRTAERR='{np.nan}', ORFLAG=200, ORS2N='{np.nan}',ORPSS='{np.nan}', \
ORDPSS='{np.nan}', ORPC='{np.nan}', ORAPPEFF='{np.nan}', CORTA='{np.nan}', CORTAERR={np.nan} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'SLS2N' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLS2N='{np.nan}',NLS2N='{np.nan}',OLTA='{np.nan}', SLTA='{np.nan}', NLTA='{np.nan}', OLTAERR='{np.nan}', SLTAERR='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, TSYS1='{np.nan}', OLPSS='{np.nan}', OLDPSS='{np.nan}', OLPC='{np.nan}', OLAPPEFF='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'NLS2N' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLS2N='{np.nan}',SLS2N='{np.nan}',OLTA='{np.nan}', SLTA='{np.nan}', NLTA='{np.nan}', OLTAERR='{np.nan}', SLTAERR='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, TSYS1='{np.nan}', OLPSS='{np.nan}', OLDPSS='{np.nan}', OLPC='{np.nan}', OLAPPEFF='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'OLS2N' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SLS2N='{np.nan}',NLS2N='{np.nan}',OLTA='{np.nan}', SLTA='{np.nan}', NLTA='{np.nan}', OLTAERR='{np.nan}', SLTAERR='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, TSYS1='{np.nan}', OLPSS='{np.nan}', OLDPSS='{np.nan}', OLPC='{np.nan}', OLAPPEFF='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'SRS2N' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORS2N='{np.nan}',NRS2N='{np.nan}',ORTA='{np.nan}', SRTA='{np.nan}', NRTA='{np.nan}', ORTAERR='{np.nan}', SRTAERR='{np.nan}', NRTAERR='{np.nan}', ORFLAG=200, TSYS2='{np.nan}', ORPSS='{np.nan}', ORDPSS='{np.nan}', ORPC='{np.nan}', ORAPPEFF='{np.nan}', CORTA='{np.nan}', CORTAERR={np.nan} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'NRS2N' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORS2N='{np.nan}',SRS2N='{np.nan}',ORTA='{np.nan}', SRTA='{np.nan}', NRTA='{np.nan}', ORTAERR='{np.nan}', SRTAERR='{np.nan}', NRTAERR='{np.nan}', ORFLAG=200, TSYS2='{np.nan}', ORPSS='{np.nan}', ORDPSS='{np.nan}', ORPC='{np.nan}', ORAPPEFF='{np.nan}', CORTA='{np.nan}', CORTAERR={np.nan} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
if 'ORS2N' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SRS2N='{np.nan}',NRS2N='{np.nan}',ORTA='{np.nan}', SRTA='{np.nan}', NRTA='{np.nan}', ORTAERR='{np.nan}', SRTAERR='{np.nan}', NRTAERR='{np.nan}', ORFLAG=200, TSYS2='{np.nan}', ORPSS='{np.nan}', ORDPSS='{np.nan}', ORPC='{np.nan}', ORAPPEFF='{np.nan}', CORTA='{np.nan}', CORTAERR={np.nan} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
elif "OLTA" in ycol or "NLTA" in ycol or "SLTA" in ycol:
if 'NLTA' in ycol:
if "13.0S" not in str(self.df['BEAMTYPE'].iloc[i]):
onta=self.df['OLTA'].iloc[i]
ontaerr=self.df['OLTAERR'].iloc[i]
sta=self.df['SLTA'].iloc[i]
staerr=self.df['SLTAERR'].iloc[i]
if "01.3S" in str(self.df['BEAMTYPE'].iloc[i]):
print('\nupdating 22ghz olpss - nlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['TOTAL_PLANET_FLUX_D'].iloc[i],self.df)
elif "02.5S" in str(self.df['BEAMTYPE'].iloc[i]):
print('\nupdating 12ghz olpss - nlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
#print(str(self.df['BEAMTYPE'].iloc[i]),"0.25S","0.25S"==str(self.df['BEAMTYPE'].iloc[i]))
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, OLPSS='{pss}', OLDPSS={errpss}, OLPC={pc}, OLAPPEFF={appeff}, COLTA={ota}, COLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
elif 'SLTA' in ycol:
if "13.0S" not in str(self.df['BEAMTYPE'].iloc[i]):
onta=self.df['OLTA'].iloc[i]
ontaerr=self.df['OLTAERR'].iloc[i]
nta=self.df['NLTA'].iloc[i]
ntaerr=self.df['NLTAERR'].iloc[i]
if "01.3S" in str(self.df['BEAMTYPE'].iloc[i]):
print('\nupdating 22ghz olpss - slta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0,nta,ntaerr,onta,ontaerr,self.df['TOTAL_PLANET_FLUX_D'].iloc[i],self.df)
elif "02.5S" in str(self.df['BEAMTYPE'].iloc[i]):
print('\nupdating 12ghz olpss - slta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0,nta,ntaerr,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',SLTAERR='{np.nan}', OLFLAG=200, OLPSS='{pss}', OLDPSS={errpss}, OLPC={pc}, OLAPPEFF={appeff},COLTA={ota},COLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
else:
print('print updating OLTA')
try:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTAERR='{np.nan}', OLPC='{np.nan}', OLFLAG=200, OLPSS='{np.nan}', OLDPSS='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}',OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
except:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',OLTAERR='{np.nan}',OLFLAG=200, OLPSS='{np.nan}',OLDPSS='{np.nan}',OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"\nupdated {self.dbFile} ")
print(tablename,xcol,ycol)
print('\nDONE')
print("+"*30,"\n")
elif "ORTA" in ycol or "NRTA" in ycol or "SRTA" in ycol:
if 'NRTA' in ycol:
if "13.0S" not in str(self.df['BEAMTYPE'].iloc[i]):
onta=self.df['ORTA'].iloc[i]
ontaerr=self.df['ORTAERR'].iloc[i]
sta=self.df['SRTA'].iloc[i]
staerr=self.df['SRTAERR'].iloc[i]
if "01.3S" in str(self.df['BEAMTYPE'].iloc[i]):
print('updating 22ghz orpss - nrta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['TOTAL_PLANET_FLUX_D'].iloc[i],self.df)
elif "02.5S" in str(self.df['BEAMTYPE'].iloc[i]):
print('updating 12ghz orpss - nrta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTAERR='{np.nan}',ORFLAG=200, ORPSS='{pss}', ORDPSS={errpss}, ORPC={pc}, ORAPPEFF={appeff},CORTA={ota},CORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print("updated CALDB")
print(tablename,xcol,ycol)
print('DONE')
elif 'SRTA' in ycol:
if "13.0S" not in str(self.df['BEAMTYPE'].iloc[i]):
onta=self.df['ORTA'].iloc[i]
ontaerr=self.df['ORTAERR'].iloc[i]
nta=self.df['NRTA'].iloc[i]
ntaerr=self.df['NRTAERR'].iloc[i]
if "01.3S" in str(self.df['BEAMTYPE'].iloc[i]):
print('updating 22ghz orpss - srta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0,nta,ntaerr,onta,ontaerr,self.df['TOTAL_PLANET_FLUX_D'].iloc[i],self.df)
elif "02.5S" in str(self.df['BEAMTYPE'].iloc[i]):
print('updating 12ghz orpss - srta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0,nta,ntaerr,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SRTAERR='{np.nan}', ORFLAG=200, ORPSS='{pss}', ORDPSS={errpss}, ORPC={pc}, ORAPPEFF={appeff},CORTA={ota},CORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print("updated CALDB")
print(tablename,xcol,ycol)
print('DONE')
else:
print('\nprint updating ORTA')
try:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTAERR='{np.nan}', ORFLAG=200, ORPC='{np.nan}', ORPSS='{np.nan}', ORDPSS='{np.nan}', CORTA='{np.nan}', CORTAERR='{np.nan}', ORAPPEFF={appeff} WHERE FILENAME = '{FN}' ;"
except:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, ORPSS='{np.nan}', ORDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print("updated CALDB")
print(tablename,xcol,ycol)
elif "OLPSS" in ycol:
print('\nprint updating OLPSS')
# try:
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, COLTA='{np.nan}', COLTAERR='{np.nan}', OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
# except:
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, OLTA='{np.nan}' WHERE FILENAME = '{FN}' ;"
# try:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLDPSS='{np.nan}', OLFLAG=201, OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
# except:
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',OLTAERR='{np.nan}',OLFLAG=200, OLPSS='{np.nan}',OLDPSS='{np.nan}',OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
elif "ORPSS" in ycol:
print('\nprint updating ORPSS')
# try:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, ORDPSS='{np.nan}', ORAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
# except:
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, ORTA='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
else:
print('No update')#MISSED: ',ycol)
#sys.exit()
elif 'D' in str(self.df['BEAMTYPE'].iloc[0]):
print("D in beamtype")
if "TSYS1" in ycol or "TSYS2" in ycol:# or "SLTA" in ycol:
if 'TSYS1' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AOLTA='{np.nan}', ASLTA='{np.nan}', ANLTA='{np.nan}', AOLTAERR='{np.nan}', ASLTAERR='{np.nan}', ANLTAERR='{np.nan}', AOLFLAG=200, AOLPSS='{pss}', AOLDPSS={errpss}, AOLPC={pc}, AOLAPPEFF={appeff}, ACOLTA={ota}, ACOLTAERR={otaerr} \
BOLTA='{np.nan}', BSLTA='{np.nan}', BNLTA='{np.nan}', BOLTAERR='{np.nan}', BSLTAERR='{np.nan}', BNLTAERR='{np.nan}', BOLFLAG=200, BOLPSS='{pss}', BOLDPSS={errpss}, BOLPC={pc}, BOLAPPEFF={appeff}, BCOLTA={ota}, BCOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
if 'TSYS2' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AORTA='{np.nan}', ASRTA='{np.nan}', ANRTA='{np.nan}', AORTAERR='{np.nan}', ASRTAERR='{np.nan}', ANRTAERR='{np.nan}', AORFLAG=200, AORPSS='{pss}', AORDPSS={errpss}, AORPC={pc}, AORAPPEFF={appeff}, ACORTA={ota}, ACORTAERR={otaerr} \
BORTA='{np.nan}', BSRTA='{np.nan}', BNRTA='{np.nan}', BORTAERR='{np.nan}', BSRTAERR='{np.nan}', BNRTAERR='{np.nan}', BORFLAG=200, BORPSS='{pss}', BORDPSS={errpss}, BORPC={pc}, BORAPPEFF={appeff}, BCORTA={ota}, BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print("\n",stmt)
elif "AOLTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AOLTAERR='{np.nan}', OLFLAG=200, AOLPSS='{np.nan}',AOLDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "AORTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AORTAERR='{np.nan}',ORFLAG=200, AORPSS='{np.nan}' ,AORDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "BOLTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', BOLTAERR='{np.nan}', OLFLAG=200, BOLPSS='{np.nan}' ,BOLDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "BORTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', BORTAERR='{np.nan}',ORFLAG=200, BORPSS='{np.nan}' ,BORDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "AOLPSS" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AOLDPSS='{np.nan}', OLFLAG=200 WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "AORPSS" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "BOLPSS" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, BOLDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "BORPSS" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORDPSS='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "ASLTA" in ycol or "BSLTA" in ycol:
if "ASLTA" in ycol:
onta=self.df['AOLTA'].iloc[i]
ontaerr=self.df['AOLTAERR'].iloc[i]
nta=self.df['ANLTA'].iloc[i]
ntaerr=self.df['ANLTAERR'].iloc[i]
print('updating dual aolpss - aSlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0, nta,ntaerr,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, AOLPSS='{pss}', AOLDPSS={errpss}, AOLPC={pc}, AOLAPPEFF={appeff},ACOLTA={ota},ACOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
elif "BSLTA" in ycol:
onta=self.df['BOLTA'].iloc[i]
ontaerr=self.df['BOLTAERR'].iloc[i]
nta=self.df['BNLTA'].iloc[i]
ntaerr=self.df['BNLTAERR'].iloc[i]
print('updating dual aolpss - bSlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0, nta,ntaerr,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, BOLPSS='{pss}', BOLDPSS={errpss}, BOLPC={pc}, BOLAPPEFF={appeff},BCOLTA={ota},BCOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "ANLTA" in ycol or "BNLTA" in ycol:
if "ANLTA" in ycol:
onta=self.df['AOLTA'].iloc[i]
ontaerr=self.df['AOLTAERR'].iloc[i]
sta=self.df['ASLTA'].iloc[i]
staerr=self.df['ASLTAERR'].iloc[i]
print('updating dual aolpss - aSlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, AOLPSS='{pss}', AOLDPSS={errpss}, AOLPC={pc}, AOLAPPEFF={appeff},ACOLTA={ota},ACOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
elif "BNLTA" in ycol:
onta=self.df['BOLTA'].iloc[i]
ontaerr=self.df['BOLTAERR'].iloc[i]
sta=self.df['BSLTA'].iloc[i]
staerr=self.df['BSLTAERR'].iloc[i]
print('updating dual aolpss - bSlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, BOLPSS='{pss}', BOLDPSS={errpss}, BOLPC={pc}, BOLAPPEFF={appeff},BCOLTA={ota},BCOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "AOLTA" in ycol or "BOLTA" in ycol:
if "AOLTA" in ycol:
print('updating dual aolpss - aSlta')
pss,errpss,pc,ota,otaerr,appeff = np.nan,np.nan,np.nan,np.nan,np.nan,np.nan
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, AOLPSS='{pss}', AOLDPSS={errpss}, AOLPC={pc}, AOLAPPEFF={appeff},ACOLTA={ota},ACOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
elif "BOLTA" in ycol:
print('updating dual aolpss - bSlta')
pss,errpss,pc,ota,otaerr,appeff =np.nan,np.nan,np.nan,np.nan,np.nan,np.nan
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, BOLPSS='{pss}', BOLDPSS={errpss}, BOLPC={pc}, BOLAPPEFF={appeff},BCOLTA={ota},BCOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "ASRTA" in ycol or "BSRTA" in ycol:
if "ASRTA" in ycol:
onta=self.df['AORTA'].iloc[i]
ontaerr=self.df['AORTAERR'].iloc[i]
nta=self.df['ANRTA'].iloc[i]
ntaerr=self.df['ANRTAERR'].iloc[i]
print('updating dual aoRpss - aSRta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0, nta,ntaerr,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORPSS={pss}, AORDPSS={errpss}, AORPC={pc}, AORAPPEFF={appeff},ACORTA={ota},ACORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
elif "BSRTA" in ycol:
onta=self.df['BORTA'].iloc[i]
ontaerr=self.df['BORTAERR'].iloc[i]
nta=self.df['BNRTA'].iloc[i]
ntaerr=self.df['BNRTAERR'].iloc[i]
print('updating dual aoRpss - bSRta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,0.0,0.0, nta,ntaerr,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORPSS={pss}, BORDPSS={errpss}, BORPC={pc}, BORAPPEFF={appeff},BCORTA={ota},BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "ANRTA" in ycol or "BNRTA" in ycol:
if "ANRTA" in ycol:
onta=self.df['AORTA'].iloc[i]
ontaerr=self.df['AORTAERR'].iloc[i]
sta=self.df['ASRTA'].iloc[i]
staerr=self.df['ASRTAERR'].iloc[i]
print('updating dual aoRpss - aSRta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORPSS={pss}, AORDPSS={errpss}, AORPC={pc}, AORAPPEFF={appeff},ACORTA={ota},ACORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
elif "BNRTA" in ycol:
onta=self.df['BORTA'].iloc[i]
ontaerr=self.df['BORTAERR'].iloc[i]
sta=self.df['BSRTA'].iloc[i]
staerr=self.df['BSRTAERR'].iloc[i]
print('updating dual aoRpss - bSlta')
pss,errpss,pc,ota,otaerr,appeff = cp.calc_pc_pss(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df['FLUX'].iloc[i],self.df)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORPSS={pss}, BORDPSS={errpss}, BORPC={pc}, BORAPPEFF={appeff},BCORTA={ota},BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
elif "AORTA" in ycol or "BORTA" in ycol:
if "AORTA" in ycol:
print('updating dual aoRpss - aSRta')
pss,errpss,pc,ota,otaerr,appeff = np.nan,np.nan,np.nan,np.nan,np.nan,np.nan
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORPSS={pss}, AORDPSS={errpss}, AORPC={pc}, AORAPPEFF={appeff},ACORTA={ota},ACORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
elif "BORTA" in ycol:
print('updating dual aoRPss - bSRta')
pss,errpss,pc,ota,otaerr,appeff =np.nan,np.nan,np.nan,np.nan,np.nan,np.nan
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORPSS={pss}, BORDPSS={errpss}, BORPC={pc}, BORAPPEFF={appeff},BCORTA={ota},BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print("updated CALDB")
print(tablename,xcol,ycol)
# complete the code here
# for the rest of them.
else:
self.write(f"Updating {self.dbFile} table: {tablename}, type: {srcType}",'info')
print('\n-BEAMTYPE: ',self.df['BEAMTYPE'].iloc[i],'\n-srcType: ',srcType)
# update the database
if 'S' in str(self.df['BEAMTYPE'].iloc[i]):
print(self.df['BEAMTYPE'].iloc[i])
if "13.0S" == self.df['BEAMTYPE'].iloc[i]:
if "TSYS1" in ycol or "TSYS2" in ycol:
if 'TSYS1' in ycol:
stot=self.df['SRCP'].iloc[i]
stoterr=np.sqrt(self.df['SRCPERR'].iloc[i]**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTA='{np.nan}', OLTAERR='{np.nan}', OLFLAG=200, SLCP='{np.nan}', SLCPERR='{np.nan}', STOT='{stot}', STOTERR='{stoterr}' WHERE FILENAME = '{FN}' ;"
if 'TSYS2' in ycol:
stot=self.df['SLCP'].iloc[i]
stoterr=np.sqrt(self.df['SLCPERR'].iloc[i]**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTA='{np.nan}', ORTAERR='{np.nan}', ORFLAG=200, SRCP='{np.nan}', SRCPERR='{np.nan}', STOT='{stot}', STOTERR='{stoterr}' WHERE FILENAME = '{FN}' ;"
elif "OLTA" in ycol or "SLCP" in ycol :
stot=self.df['SRCP'].iloc[i]
stoterr=np.sqrt(self.df['SRCPERR'].iloc[i]**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',OLTAERR='{np.nan}', OLFLAG=201, TSYS1='{np.nan}', SLCP='{np.nan}', SLCPERR='{np.nan}', STOT='{stot}', STOTERR='{stoterr}' WHERE FILENAME = '{FN}' ;"
elif "ORTA" in ycol or "SRCP" in ycol:
stot=self.df['SLCP'].iloc[i]
stoterr=np.sqrt(self.df['SLCPERR'].iloc[i]**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}',ORTAERR='{np.nan}', ORFLAG=201, TSYS2='{np.nan}', SRCP='{np.nan}', SRCPERR='{np.nan}', STOT='{stot}', STOTERR='{stoterr}' WHERE FILENAME = '{FN}' ;"
# elif "SLCP" in ycol:
# print('\nprint updating SLCP')
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SLCPERR='{np.nan}', OLFLAG=202, OLAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
# elif "SRCP" in ycol:
# print('\nprint updating SRCP')
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=202, ORDPSS='{np.nan}', ORAPPEFF='{np.nan}' WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
print('\nDONE')
print("+"*30,"\n")
else:
if "TSYS1" in ycol or "TSYS2" in ycol:
if 'TSYS1' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTA='{np.nan}', OLTAERR='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}', OLPC='{np.nan}', OLS2N=200, \
SLTA='{np.nan}', SLTAERR='{np.nan}', SLS2N='{np.nan}',\
NLTA='{np.nan}', NLTAERR='{np.nan}', NLS2N='{np.nan}',\
SLCP='{np.nan}', SLCPERR='{np.nan}',\
SLCPPSS='{np.nan}', SLCPPSSERR='{np.nan}',\
STOT='{np.nan}', STOTERR='{np.nan}' \
WHERE FILENAME = '{FN}' ;"
if 'TSYS2' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTA='{np.nan}', ORTAERR='{np.nan}', CORTA='{np.nan}', CORTAERR='{np.nan}', ORPC='{np.nan}', ORS2N=200, \
SRTA='{np.nan}', SRTAERR='{np.nan}', SRS2N='{np.nan}',\
NRTA='{np.nan}', NRTAERR='{np.nan}', NRS2N='{np.nan}',\
SRCP='{np.nan}', SRCPERR='{np.nan}',\
SRCPPSS='{np.nan}', SRCPPSSERR='{np.nan}',\
STOT='{np.nan}', STOTERR='{np.nan}' \
WHERE FILENAME = '{FN}' ;"
elif "OLTA" in ycol or "NLTA" in ycol or "SLTA" in ycol:
srcp=self.df['SRCP'].iloc[i]
srcperr=self.df['SRCPERR'].iloc[i]
if 'NLTA' in ycol:
hpsTa=self.df['SLTA'].iloc[i]
errHpsTa=self.df['SLTAERR'].iloc[i]
onTa=self.df['OLTA'].iloc[i]
errOnTa=self.df['OLTAERR'].iloc[i]
pss=self.df['SLCPPSS'].iloc[i]
errpss=self.df['SLCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, hpsTa, errHpsTa, 0, 0, onTa, errOnTa, self.df)
slcp=corrta*pss
slcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
stot = (slcp+srcp)/2.
stoterr=0.5*((slcperr)**2 + (srcperr)**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', NLTAERR='{np.nan}', OLFLAG=200, OLPC={pc}, COLTA={corrta}, COLTAERR={errta}, SLCP={slcp}, SLCPERR={slcperr},STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
elif 'SLTA' in ycol:
hpnTa=self.df['NLTA'].iloc[i]
errHpnTa=self.df['NLTAERR'].iloc[i]
onTa=self.df['OLTA'].iloc[i]
errOnTa=self.df['OLTAERR'].iloc[i]
pss=self.df['SLCPPSS'].iloc[i]
errpss=self.df['SLCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, 0, 0, hpnTa, errHpnTa, onTa, errOnTa, self.df)
slcp=corrta*pss
slcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
stot = (slcp+srcp)/2.
stoterr=0.5*((slcperr)**2 + (srcperr)**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SLTAERR='{np.nan}', OLFLAG=200, OLPC={pc}, COLTA={corrta}, COLTAERR={errta}, SLCP={slcp}, SLCPERR={slcperr},STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
else:
# print('print updating OLTA')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLTAERR='{np.nan}', SLTA='{np.nan}',SLTAERR='{np.nan}',NLTA='{np.nan}',\
NLTAERR'{np.nan}',OLPC='{np.nan}', OLFLAG=200, COLTA='{np.nan}', COLTAERR='{np.nan}',OLAPPEFF='{np.nan}',SLCP='{np.nan}', \
SLCPERR='{np.nan}',STOT={srcp},STOTERR={srcperr} WHERE FILENAME = '{FN}' ;"
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"\nupdated {self.dbFile} ")
print(tablename,xcol,ycol)
print('\nDONE')
print("+"*30,"\n")
elif "ORTA" in ycol or "NRTA" in ycol or "SRTA" in ycol:
slcp=self.df['SLCP'].iloc[i]
slcperr=self.df['SLCPERR'].iloc[i]
if 'NRTA' in ycol:
hpsTa=self.df['SRTA'].iloc[i]
errHpsTa=self.df['SRTAERR'].iloc[i]
onTa=self.df['ORTA'].iloc[i]
errOnTa=self.df['ORTAERR'].iloc[i]
pss=self.df['SRCPPSS'].iloc[i]
errpss=self.df['SRCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, hpsTa, errHpsTa, 0, 0, onTa, errOnTa, self.df)
srcp=corrta*pss
srcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
stot = (slcp+srcp)/2.
stoterr=0.5*((slcperr)**2 + (srcperr)**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', NRTAERR='{np.nan}', ORFLAG=200, ORPC={pc}, CORTA={corrta}, CORTAERR={errta}, SRCP={slcp}, SRCPERR={slcperr},STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
elif 'SRTA' in ycol:
hpnTa=self.df['NLTA'].iloc[i]
errHpnTa=self.df['NLTAERR'].iloc[i]
onTa=self.df['OLTA'].iloc[i]
errOnTa=self.df['OLTAERR'].iloc[i]
pss=self.df['SRCPPSS'].iloc[i]
errpss=self.df['SRCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, 0, 0, hpnTa, errHpnTa, onTa, errOnTa, self.df)
srcp=corrta*pss
srcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
stot = (slcp+srcp)/2.
stoterr=0.5*((slcperr)**2 + (srcperr)**2)
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SRTAERR='{np.nan}', ORFLAG=200, ORPC={pc}, CORTA={corrta}, CORTAERR={errta}, SRCP={slcp}, SRCPERR={slcperr},STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
else:
# print('print updating OLTA')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORTAERR='{np.nan}', SRTA='{np.nan}',SRTAERR='{np.nan}',NRTA='{np.nan}',NRTAERR'{np.nan}',\
ORPC='{np.nan}', ORFLAG=200, CORTA='{np.nan}', CORTAERR='{np.nan}',ORAPPEFF='{np.nan}',SRCP='{np.nan}', SRCPERR='{np.nan}',\
STOT={slcp},STOTERR={slcperr} WHERE FILENAME = '{FN}' ;"
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"\nupdated {self.dbFile} ")
print(tablename,xcol,ycol)
print('\nDONE')
print("+"*30,"\n")
elif "SLCP" in ycol:
# print('\nprint updating OLPSS')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SLCPERR='{np.nan}', OLFLAG=201, \
SLTA='{np.nan}', SLTAERR='{np.nan}', SLS2N='{np.nan}',\
NLTA='{np.nan}', NLTAERR='{np.nan}', NLS2N='{np.nan}',\
OLTA='{np.nan}', OLTAERR='{np.nan}', COLTA='{np.nan}', COLTAERR='{np.nan}', OLPC='{np.nan}', OLS2N=202, \
STOT={srcp}, STOTERR={srcperr}\
WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
elif "SRCP" in ycol:
# print('\nprint updating ORPSS')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', SRCPERR='{np.nan}', ORFLAG=202, \
SRTA='{np.nan}', SRTAERR='{np.nan}', SRS2N='{np.nan}',\
NRTA='{np.nan}', NRTAERR='{np.nan}', NRS2N='{np.nan}',\
ORTA='{np.nan}', ORTAERR='{np.nan}', CORTA='{np.nan}', CORTAERR='{np.nan}', ORPC='{np.nan}', ORS2N=202, \
STOT={srcp}, STOTERR={srcperr}\
WHERE FILENAME = '{FN}' ;"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print(f"updated {self.dbFile}")
print(tablename,xcol,ycol)
else:
print('No update algorithm setup yet\n')
elif 'D' in str(self.df['BEAMTYPE'].iloc[0]):
# print("D in beamtype")1
aslcp=self.df['ASLCP'].iloc[i]
bslcp=self.df['BSLCP'].iloc[i]
asrcp=self.df['ASRCP'].iloc[i]
bsrcp=self.df['BSRCP'].iloc[i]
aslcperr=self.df['ASLCPERR'].iloc[i]
bslcperr=self.df['BSLCPERR'].iloc[i]
asrcperr=self.df['ASRCPERR'].iloc[i]
bsrcperr=self.df['BSRCPERR'].iloc[i]
if "TSYS1" in ycol or "TSYS2" in ycol:
if 'TSYS1' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', \
AOLTA='{np.nan}', AOLTAERR='{np.nan}', OLFLAG=200, AOLPC='{np.nan}', ACOLTA='{np.nan}', ACOLTAERR='{np.nan}', \
ASLTA='{np.nan}', ASLTAERR='{np.nan}', SLFLAG=200, ANLTA='{np.nan}', ANLTAERR='{np.nan}' \
BSLTA='{np.nan}', BSLTAERR='{np.nan}', NLFLAG=200, BNLTA='{np.nan}', BNLTAERR='{np.nan}', \
BOLTA='{np.nan}', BOLTAERR='{np.nan}', BOLPC='{np.nan}', BCOLTA='{np.nan}', BCOLTAERR='{np.nan}', \
ASLCP='{np.nan}', ASLCPERR='{np.nan}',BSLCP='{np.nan}', BSLCPERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
if 'TSYS2' in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', \
AORTA='{np.nan}', AORTAERR='{np.nan}', ORFLAG=200, AORPC='{np.nan}', ACORTA='{np.nan}', ACORTAERR='{np.nan}', \
ASRTA='{np.nan}', ASRTAERR='{np.nan}', SRFLAG=200, ANRTA='{np.nan}', ANRTAERR='{np.nan}', \
BSRTA='{np.nan}', BSRTAERR='{np.nan}', NRFLAG=200, BNRTA='{np.nan}', BNRTAERR='{np.nan}', \
BORTA='{np.nan}', BORTAERR='{np.nan}', BORFLAG=200, BORPC='{np.nan}', BCORTA='{np.nan}', BCORTAERR='{np.nan}', \
ASRCP='{np.nan}', ASRCPERR='{np.nan}', BSRCP='{np.nan}', BSRCPERR='{np.nan}' WHERE FILENAME = '{FN}' ;"
elif "ASLCP" in ycol:
s=[bslcp,asrcp,bsrcp]
er=[bslcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ASLCPERR='{np.nan}', OLFLAG=200, \
STOT={stot}, STOTERR={stoterr} \
WHERE FILENAME = '{FN}' ;"
elif "BSLCP" in ycol:
s=[aslcp,asrcp,bsrcp]
er=[aslcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', BSLCPERR='{np.nan}', OLFLAG=200, \
STOT={stot}, STOTERR={stoterr} \
WHERE FILENAME = '{FN}' ;"
elif "ASRCP" in ycol:
s=[bslcp,asrcp,bsrcp]
er=[bslcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ASRCPERR='{np.nan}', ORFLAG=200, \
STOT={stot}, STOTERR={stoterr} \
WHERE FILENAME = '{FN}' ;"
elif "BSRCP" in ycol:
s=[aslcp,asrcp,bsrcp]
er=[aslcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', BSRCPERR='{np.nan}', ORFLAG=200, \
STOT={stot}, STOTERR={stoterr} \
WHERE FILENAME = '{FN}' ;"
elif "AOLTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AOLTAERR='{np.nan}', OLFLAG=200, AOLPC='{np.nan}', ACOLTA='{np.nan}', ACOLTAERR='{np.nan}', \
ASLTA='{np.nan}', ASLTAERR='{np.nan}', SLFLAG=200, \
ANLTA='{np.nan}', ANLTAERR='{np.nan}', NLFLAG=200 \
WHERE FILENAME = '{FN}' ;"
elif "AORTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', AORTAERR='{np.nan}', ORFLAG=200, AORPC='{np.nan}', ACORTA='{np.nan}', ACORTAERR='{np.nan}', \
ASRTA='{np.nan}', ASRTAERR='{np.nan}', SRFLAG=200, \
ANRTA='{np.nan}', ANRTAERR='{np.nan}', NRFLAG=200 \
WHERE FILENAME = '{FN}' ;"
elif "BOLTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', BOLTAERR='{np.nan}', OLFLAG=200, BOLPC='{np.nan}', BCOLTA='{np.nan}', BCOLTAERR='{np.nan}', \
BSLTA='{np.nan}', BSLTAERR='{np.nan}', SLFLAG=200, \
BNLTA='{np.nan}', BNLTAERR='{np.nan}', NLFLAG=200 \
WHERE FILENAME = '{FN}' ;"
elif "BORTA" in ycol:
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', BORTAERR='{np.nan}', ORFLAG=200, BORPC='{np.nan}', BCORTA='{np.nan}', BCORTAERR='{np.nan}', \
BSRTA='{np.nan}', BSRTAERR='{np.nan}', SRFLAG=200, \
BNRTA='{np.nan}', BNRTAERR='{np.nan}', NRFLAG=200 \
WHERE FILENAME = '{FN}' ;"
elif "ASLTA" in ycol or "BSLTA" in ycol:
if "ASLTA" in ycol:
# recalculate tot flux
hpnTa=self.df['ANLTA'].iloc[i]
errHpnTa=self.df['ANLTAERR'].iloc[i]
onTa=self.df['AOLTA'].iloc[i]
errOnTa=self.df['AOLTAERR'].iloc[i]
pss=self.df['ALCPPSS'].iloc[i]
errpss=self.df['ALCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, 0, 0, hpnTa, errHpnTa, onTa, errOnTa, self.df)
slcp=corrta*pss
slcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
s=[slcp,bslcp,asrcp,bsrcp]
er=[slcperr,bslcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
print('updating dual ASLTA')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, \
ASLTA='{np.nan}', ASLTAERR='{np.nan}', \
AOLPC={pc}, ACOLTA={corrta},ACOLTAERR={errta}, \
ASLCP={slcp}, ASLCPERR={slcperr}, STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
elif "BSLTA" in ycol:
# recalculate tot flux
hpnTa=self.df['BNLTA'].iloc[i]
errHpnTa=self.df['BNLTAERR'].iloc[i]
onTa=self.df['BOLTA'].iloc[i]
errOnTa=self.df['BOLTAERR'].iloc[i]
pss=self.df['BLCPPSS'].iloc[i]
errpss=self.df['BLCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, 0, 0, hpnTa, errHpnTa, onTa, errOnTa, self.df)
slcp=corrta*pss
slcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
s=[aslcp,slcp,asrcp,bsrcp]
er=[aslcperr,slcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
print('updating dual ASLTA')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, \
BSLTA='{np.nan}', BSLTAERR='{np.nan}', \
BOLPC={pc}, BCOLTA={corrta},BCOLTAERR={errta}, \
BSLCP={slcp}, BSLCPERR={slcperr}, STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
elif "ANLTA" in ycol or "BNLTA" in ycol:
if "ANLTA" in ycol:
# recalculate tot flux
hpsTa=self.df['ASLTA'].iloc[i]
errHpsTa=self.df['ASLTAERR'].iloc[i]
onTa=self.df['AOLTA'].iloc[i]
errOnTa=self.df['AOLTAERR'].iloc[i]
pss=self.df['ALCPPSS'].iloc[i]
errpss=self.df['ALCPPSSERR'].iloc[i]
pc,corrta,errta=cp.calc_pc(0, hpsTa, errHpsTa, 0, 0, onTa, errOnTa, self.df)
slcp=corrta*pss
slcperr=np.sqrt((corrta/errta)**2 + (pss/errpss)**2)*slcp
s=[slcp,bslcp,asrcp,bsrcp]
er=[slcperr,bslcperr,asrcperr,bsrcperr]
ss=[]
se=[]
for i in range(len(s)):
if "nan" in str(s[i]):
pass
else:
ss.append(s[i])
se.append(er[i])
stot = (sum(ss))/len(ss)
stoterr=(1/len(se))*np.sqrt(sum([w**2 for w in se]))
print('updating dual ANLTA')
stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, \
ANLTA='{np.nan}', ANLTAERR='{np.nan}', \
AOLPC={pc}, ACOLTA={corrta},ACOLTAERR={errta}, \
ASLCP={slcp}, ASLCPERR={slcperr}, STOT={stot}, STOTERR={stoterr} WHERE FILENAME = '{FN}' ;"
# elif "BNLTA" in ycol:
# onta=self.df['BOLTA'].iloc[i]
# ontaerr=self.df['BOLTAERR'].iloc[i]
# sta=self.df['BSLTA'].iloc[i]
# staerr=self.df['BSLTAERR'].iloc[i]
# print('updating dual BNLTA')
# pc,ota,otaerr= cp.calc_pc(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df)
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, BOLPC={pc}, BCOLTA={ota},BCOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# print(stmt)
# elif "AOLTA" in ycol or "BOLTA" in ycol:
# if "AOLTA" in ycol:
# print('updating dual AOLTA')
# pc,ota,otaerr = np.nan,np.nan,np.nan
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, AOLPC={pc}, ACOLTA={ota},ACOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# elif "BOLTA" in ycol:
# print('updating dual BOLTA')
# pc,ota,otaerr=np.nan,np.nan,np.nan
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', OLFLAG=200, BOLPC={pc}, BCOLTA={ota},BCOLTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# print(stmt)
# elif "ASRTA" in ycol or "BSRTA" in ycol:
# if "ASRTA" in ycol:
# onta=self.df['AORTA'].iloc[i]
# ontaerr=self.df['AORTAERR'].iloc[i]
# nta=self.df['ANRTA'].iloc[i]
# ntaerr=self.df['ANRTAERR'].iloc[i]
# print('updating dual ASRTA')
# pc,ota,otaerr = cp.calc_pc(0,0.0,0.0, nta, ntaerr, onta, ontaerr,self.df)
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORPC={pc},ACORTA={ota},ACORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# elif "BSRTA" in ycol:
# onta=self.df['BORTA'].iloc[i]
# ontaerr=self.df['BORTAERR'].iloc[i]
# nta=self.df['BNRTA'].iloc[i]
# ntaerr=self.df['BNRTAERR'].iloc[i]
# print('updating dual BSRTA')
# pc,ota,otaerr = cp.calc_pc(0,0.0,0.0, nta,ntaerr,onta,ontaerr,self.df)
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORPC={pc}, BCORTA={ota},BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# print(stmt)
# elif "ANRTA" in ycol or "BNRTA" in ycol:
# if "ANRTA" in ycol:
# onta=self.df['AORTA'].iloc[i]
# ontaerr=self.df['AORTAERR'].iloc[i]
# sta=self.df['ASRTA'].iloc[i]
# staerr=self.df['ASRTAERR'].iloc[i]
# print('updating dual ANRTA')
# pc,ota,otaerr = cp.calc_pc(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df)
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORPC={pc},ACORTA={ota},ACORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# elif "BNRTA" in ycol:
# onta=self.df['BORTA'].iloc[i]
# ontaerr=self.df['BORTAERR'].iloc[i]
# sta=self.df['BSRTA'].iloc[i]
# staerr=self.df['BSRTAERR'].iloc[i]
# print('updating dual BNRTA')
# pc,ota,otaerr = cp.calc_pc(0,sta,staerr, 0.0,0.0,onta,ontaerr,self.df)
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORPC={pc}, BCORTA={ota},BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# print(stmt)
# elif "AORTA" in ycol or "BORTA" in ycol:
# if "AORTA" in ycol:
# print('updating dual AORTA')
# pc,ota,otaerr = np.nan, np.nan, np.nan
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, AORPC={pc}, ACORTA={ota},ACORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
# elif "BORTA" in ycol:
# print('updating dual BORTA')
# pc,ota,otaerr =np.nan, np.nan, np.nan
# stmt = f"UPDATE {tablename} SET {ycol}='{np.nan}', ORFLAG=200, BORPC={pc}, BCORTA={ota},BCORTAERR={otaerr} WHERE FILENAME = '{FN}' ;"
#
print(stmt)
db.c.execute(stmt)
db.commit_changes()
db.close_db()
print("updated CALDB")
print(tablename,xcol,ycol)
# complete the code here
# for the rest of them.
print('After deleting: ',len(self.df))
print(self.df.iloc[i][yCol])
self.Canvas.clear_figure()
self.plot_cols_df()
[docs]
def reset_timeseries(self):
self.df=self.orig_df
# plot
self.Canvas.clear_figure()
self.plot_cols()
[docs]
def refresh_db(self):
"""refresh the database entries. Check for updates."""
# open the database and get tables
self.db = SQLiteDB(dbPath=self.dbFile, log=self.log)
self.db.create_db()
# self.db.close_db()
# populate columns
self.dbName=self.dbFile.split('/')[-1]
self.tables=self.db.get_table_names(self.dbName)
self.populate_cols()
# update plot
self.plot_cols()
self.db.close_db()
[docs]
def save_time_db(self,filen=""):
"""Save the analysis results of the time series to a csv file"""
if filen:
self.df.to_csv(f"Analysis_of_table_{filen}.csv")
print(f'saved results to {filen}.csv')
else:
self.df.to_csv("Analysis_results.csv")
print('saved results to Analysis_results.csv')
[docs]
def plot_cols_df(self):
""" Plot database columns. """
# get col names
xCol = self.time_ui.comboBoxColsX.currentText()
yCol = self.time_ui.comboBoxColsY.currentText()
if xCol == yCol:
print("\nYou cannot plot the same column on X and Y\n")
else:
yErr = self.time_ui.comboBoxColsYerr.currentText()
print(f"\nPlotting {xCol} vs {yCol} in table {self.table}\n")
self.df[yCol]=self.df[yCol]#.apply(self.f)
self.df[yCol].fillna(value=np.nan, inplace=True)
self.df[yCol]=self.df[yCol].replace('',np.nan)
self.df[yCol]=self.df[yCol].astype(float)
print(self.df[xCol],self.df[yCol])
try:
self.df[yErr]=self.df[yErr]#.apply(self.f)
self.Canvas.plot_fig(self.df[xCol],self.df[yCol],xCol,yCol,data=self.df,yerr=self.df[yErr]) #,title=f"Plot of {self.df['SOURCEDIR']} obs. {self.df['FILENAME'][:8]}")#,data=self.)
except:
self.Canvas.plot_fig(self.df[xCol],self.df[yCol],xCol,yCol,data=self.df)#,title=f"Plot of {self.df['SOURCEDIR']} obs. {self.df['FILENAME'][:8]}")#,data=self.)
self.Canvas.draw()
# PLOTVIEWER OPERATIONS
[docs]
def open_db_path(self):
# Open a database
self.write("Opening DB",'info')
self.dbFilePath = self.open_file_name_dialog("*.db")
if self.dbFilePath == None:
self.write("You need to select a file to open",'info')
self.write("Please select a file",'info')
pass
else:
# free all else
free=True
self.plot_ui.btnDelete.setEnabled(free)
self.plot_ui.btnRefreshPlotList.setEnabled(free)
self.plot_ui.btnShow.setEnabled(free)
self.plot_ui.comboBox.setEnabled(free)
self.plot_ui.comboBoxFilter.setEnabled(free)
self.plot_ui.comboBoxOptions.setEnabled(free)
self.plot_ui.txtBoxEnd.setEnabled(free)
self.plot_ui.txtBoxStart.setEnabled(free)
# open db and get tables
self.db = SQLiteDB(self.dbFilePath, log=self.log)
self.db.create_db()
self.tables = sorted(list(set(sorted(self.db.get_table_names(self.dbFilePath)))))
print('T2 :',self.tables)
self.plot_ui.comboBox.clear()
self.plot_ui.comboBox.clear()
self.plot_ui.comboBox.addItems(self.tables)
[docs]
def on_combo_changed(self):
print("\nChanged obs. to: ", self.plot_ui.comboBox.currentText())
# get data from db
# check folder name against table name
folderName=self.plot_ui.comboBox.currentText()
# get column names from db and put them in combobox
colInd, colNames, colType = self.db.get_all_table_coloumns(folderName)
plotCols=[]
for name in colNames:
#or 'OBS' in name
# if 'OBSTIME' in name or 'SOURCE' in name or 'CUR' in name or 'NOM' in name or 'TUDE' in name \
# or 'BW' in name or 'FREQ' in name or 'FILE' in name or 'OBJ' in name or 'id' == name or \
# 'RAD' in name or 'TYPE' in name or 'PRO' in name or 'TELE' in name or 'UPGR' in name \
# or 'INST' in name or 'SCAN' in name:
if 'id' in name or 'LOGFREQ' in name or 'CURDATETIME' in name or \
'FILE' in name or 'OBSD' in name \
or 'MJD' in name or 'OBS' in name or 'OBJ' in name or 'id' == name \
or 'RAD' in name or 'TYPE' in name or 'PRO' in name or 'TELE' in\
name or 'UPGR' in name or 'INST' in name or \
'SCANDIR' in name or 'SRC' in name or 'COORDSYS' in name or 'LONGITUD' in name \
or 'LATITUDE' in name or 'POINTING' in name \
or 'DICHROIC' in name \
or 'PHASECAL' in name or 'HPBW' in name or 'FNBW' in name or 'SNBW' in name\
or 'FRONTEND' in name or 'BASE' in name:
pass
else:
plotCols.append(name)
# self.plot_ui.comboBoxOptions.clear()
self.plot_ui.comboBoxOptions.clear()
self.plot_ui.comboBoxOptions.clear()
self.plot_ui.comboBoxOptions.addItems(plotCols)
# self.plot_ui.comboBoxFilter.clear()
self.plot_ui.comboBoxFilter.clear()
self.plot_ui.comboBoxFilter.clear()
self.plot_ui.comboBoxFilter.addItems(['','>','>=','=','<','<=','between'])
[docs]
def on_filter_changed(self):
if self.plot_ui.comboBoxFilter.currentText()=="between":
print(self.plot_ui.comboBoxFilter.currentText())
#Add second filter
self.toggle_range_filter('on')
self.plot_ui.txtBoxFilter.setVisible(False)
else:
print(self.plot_ui.comboBoxFilter.currentText())
# remove second filter
self.toggle_range_filter('off')
self.plot_ui.txtBoxFilter.setVisible(True)
[docs]
def toggle_range_filter(self,filter):
if filter=='off':
toggle=False
elif filter=="on":
toggle=True
self.plot_ui.LblRangeFilter.setVisible(toggle)
self.plot_ui.LblStart.setVisible(toggle)
self.plot_ui.LblStop.setVisible(toggle)
self.plot_ui.LblFormat.setVisible(toggle)
self.plot_ui.txtBoxEnd.setVisible(toggle)
self.plot_ui.txtBoxStart.setVisible(toggle)
[docs]
def add_items_to_combobox(self):
"""Refresh the list of folders containing plots to be displayed. """
# self.db.close_db()
# open db and get tables
self.db = SQLiteDB(self.dbFilePath, log=self.log)
self.db.create_db()
tables = sorted(self.db.get_table_names(self.dbFilePath))
self.plot_ui.comboBox.clear()
self.plot_ui.comboBox.clear()
self.plot_ui.comboBoxOptions.clear()
self.plot_ui.comboBoxOptions.clear()
self.plot_ui.comboBoxFilter.clear()
self.plot_ui.comboBoxFilter.clear()
# self.plot_ui.comboBox.clear()
self.plot_ui.comboBox.addItems(tables)
[docs]
def get_data_from_db(self, folderName):
db = SQLiteDB(self.dbFilePath, log=self.log)
db.create_db()
tableNames = sorted(db.get_table_names(self.dbFilePath))
colInd, colNames, colType = db.get_all_table_coloumns(folderName)
rows = self.db.get_rows_of_cols(folderName,colNames)
# create dataframe
df = pd.DataFrame(list(rows), columns = colNames)
df=df.sort_values('FILENAME')
return df ,db
[docs]
def checkKeysFound(self,pol,option,img,path,images,keys):
for key in keys:
if f'{key}{pol}TA' in option or f'{key}{pol}MID' in option or \
f'{key}{pol}S2N' in option or f'{key}{pol}FLAG' in option or \
f'{key}{pol}RMS' in option or f'{key}{pol}SLOPE' in option or \
f'{key}{pol}BRMS' in option:
if key=='S'and f"HPS_{pol}CP" in img:
images.append(path)
elif key=='N' and f"HPN_{pol}CP" in img:
images.append(path)
elif key=='O' and f"ON_{pol}CP" in img:
images.append(path)
return images
[docs]
def show_plot_browser(self):
"""
Open a webrowser containg the plots to be displayed.
"""
# Run checks first
option=self.plot_ui.comboBoxOptions.currentText()
filter=self.plot_ui.comboBoxFilter.currentText()
if filter !="between":
# Get params
txt=self.plot_ui.txtBoxFilter.text()
print(f"\nShowing plots where {option} {filter} {txt}")
if txt is not None or isinstance(txt, str):
if option=="":
print("Need to choose an option")
else:
if filter=="":
print("Please make a valid selection")
else:
if txt=="":
# show everything
print("no value added, should show all pics, but wont")
else:
# Get data from the database
# check folder name against table name
folderName=self.plot_ui.comboBox.currentText()
print('Folder name: ',folderName)
print('Path to file: ',self.dbFilePath)
# get data from database
df,db = self.get_data_from_db(folderName)
if option == 'OBSDATE':
print('Showing overall stats')
print(f'min: {df[option].iloc[0]}')
print(f'max: {df[option].iloc[-1]}')
try:
print(f'mean: {df[option].mean()}')
print(f'std: {df[option].std()}')
except Exception as e:
print(e)
print("*"*30,"\n")
else:
# print(df[option].apply(fx))
df[option]=df[option].astype(float) #.apply(fx)
# df[option]=df[option].apply(fx)
print("\n","*"*30)
try:
print('Showing basic stats for selection')
print(f'DATE start: {df["OBSDATE"].iloc[0]}')
print(f'DATE end: {df["OBSDATE"].iloc[-1]}')
print(f'MJD start: {df["MJD"].iloc[0]:.1f}')
print(f'MJD end: {df["MJD"].iloc[-1]:.1f}')
print(f'min: {df[option].dropna().min():.3f}')
print(f'max: {df[option].dropna().max():.3f}')
print(f'mean: {df[option].dropna().mean():.3f}')
print(f'std: {df[option].dropna().std():.3f}')
print(f'3sigma upper limit: {df[option].mean() + (df[option].mean()*df[option].std()):.3f}')
print(f'3sigma lower limit: {df[option].mean() - (df[option].mean()*df[option].std()):.3f}')
except:
print('Showing overall stats')
print(f'DATE start: {df["OBSDATE"].iloc[0]}')
print(f'DATE end: {df["OBSDATE"].iloc[-1]}')
print(f'MJD start: {df["MJD"].iloc[0]:.1f}')
print(f'MJD end: {df["MJD"].iloc[-1]:.1f}')
print(f'min: {df[option].dropna().min():.3f}')
print(f'max: {df[option].dropna().max():.3f}')
print(f'mean: {df[option].dropna().mean():.3f}')
print(f'std: {df[option].dropna().std():.3f}')
print("*"*30,"\n")
# sys.exit()
try:
txt=float(txt)
except ValueError:
pass
if type(txt).__name__ != "str":
txt=float(txt)
df[option] = df[option].astype(float)
df=df.sort_values('FILENAME')
if filter == ">":
ndf = df[df[option] > txt]
# print(len(ndf))
elif filter == ">=":
ndf = df[df[option] >= txt]
# print(len(ndf))
elif filter == "<":
ndf = df[df[option] < txt]
# print(len(ndf))
elif filter == "<=":
ndf = df[df[option] <= txt]
# print(len(ndf))
elif filter == "=":
ndf = df[df[option] == txt]
# print(len(ndf))
else:
ndf=df
#
# print(len(df),len(ndf),option,txt)
if len(ndf) > 0:
# Show only what is requested
#print(ndf['FILENAME'])
ndf=ndf.sort_values('FILENAME')
ndf['plot_tag']=ndf['FILENAME'].apply(lambda x:x[:18])
plots=list(ndf['plot_tag'])
# print(plots)
# sys.exit()
# print stats
print("\n--- Basic stats ---\n")
print("Min: ", ndf[option].min())
print("Max: ", ndf[option].max())
print("Mean: ", ndf[option].mean())
print("Median: ", ndf[option].median())
print("-"*20,"\n")
# get plot folder and plot to browser
imgDir = "plots/"
ls=os.listdir(imgDir)
# fln=(folderName.split("_")[0]).upper()
# print(sorted(ls))#,fln)
# sys.exit()
msg_wrapper("info", self.log.debug,"Plotting folder "+folderName)
try:
src=(ndf["OBJECT"].iloc[-1])#.replace(" ","")
except:
src=(ndf["OBJECT"].iloc[0])#.replace(" ","")
# src=s#src.replace("-","_")
if folderName.startswith('_'):
folderName=folderName[1:]
fp=folderName.split('_')
try:
freq=int((ndf["CENTFREQ"].iloc[-1]))
except:
print('Struggling to find frequency - raise an issue on github ')
sys.exit()
# if src !=fp[0]:
# imgPath=f'plots/{src}/{src}_{fp[1]}/'
# else:
imgPath=f'plots/{src}/{freq}/'
# imgPath=imgPath.strip()
imgPath=imgPath.replace(' ','')
imgPath=imgPath.replace('','')
# print(imgPath)
# sys.exit()
# imgPath=imgPath.replace('m','-')
imageNames = sorted(os.listdir(imgPath))
images=[]
if ".DS_Store" in imageNames:
imageNames.remove(".DS_Store")
keys=['S','N','O'] # south, north, on
# # get images for current point
for i in range(len(plots)):
# print('>>',imgPath,imageNames[i])
# found=0
# TODO: FIX this mess, i only want to see relvant data
for j in range(len(imageNames)):
if plots[i] in imageNames[j]: #[:18]:
# print(imageNames[j],plots[i])
path=imgPath+imageNames[j]
images1=self.checkKeysFound('L',option,imageNames[j],path,images,keys)
images2=self.checkKeysFound('R',option,imageNames[j],path,images,keys)
# checkKeysFound(pol,option,img,path,images,keys)
if len(images1)==0 and len(images2)==0:
# no matches found
images.append(imgPath+imageNames[i])
else:
images = images + images1+images2
# images.append(imgPath+imageNames[j])
# if "SL" in option and "HPS_LCP" in imageNames[j]:
# #print(imageNames[i],plots[j])
# images.append(imgPath+imageNames[j])
# elif "SR" in option and "HPS_RCP" in imageNames[j]:
# #print(imageNames[i],plots[j])
# images.append(imgPath+imageNames[j])
# elif "NL" in option and "HPN_LCP" in imageNames[j]:
# #print(imageNames[i],plots[j])
# images.append(imgPath+imageNames[j])
# elif "NR" in option and "HPN_RCP" in imageNames[j]:
# #print(imageNames[i],plots[j])
# images.append(imgPath+imageNames[j])
# elif "OL" in option and "ON_LCP" in imageNames[j]:
# #print(imageNames[i],plots[j])
# images.append(imgPath+imageNames[j])
# elif "OR" in option and "ON_RCP" in imageNames[j]:
# #print(imageNames[i],plots[j])
# images.append(imgPath+imageNames[j])
# else:
# images.append(imgPath+imageNames[i])
print(images)
sys.exit()
htmlstart = '<html> <head>\
<meta charset = "utf-8" >\
<meta name = "viewport" content = "width=device-width, initial-scale=1" > <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous"> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/js/bootstrap.bundle.min.js" integrity="sha384-b5kHyXgcpbZJO/tY9Ul7kGkf1S0CWuKcCD38l8YkeH8z8QjE0GmW1gYU5S9FOnJ0" crossorigin="anonymous"></script> \
<style> \
img {border: 3px solid # ddd; /* Gray border */border-radius: 5px; /* Rounded border */padding: 5px; /* Some padding */width: 400px; /* Set a small width */}/* Add a hover effect (blue shadow) */\
img:hover {box-shadow: 0 0 2px 1px rgba(0, 140, 186, 0.5);}\
</style> \
<title>Driftscan plots</title>\
</head>\
<div class="container-fluid"> \
<div class="row">\
<hr>\
<h1> Plotting folder '+folderName.upper() + '</h1> \
<p>'
htmlmid=''
imageOptions=[]
if "OL" in option or "SL" in option or "NL" in option:
for img in images:
if "LCP" in img:
imageOptions.append(img)
images=imageOptions
elif "OR" in option or "SR" in option or "NR" in option:
for img in images:
if "RCP" in img:
imageOptions.append(img)
images=imageOptions
else:
images=images
images=set(sorted(images))
images=list(images)
print()
for img in images:
print(f'Showing: {img}')
# sys.exit()
# add images to html page
for i in range(len(images)):
#print(images.split("/")[1],images.split("/")[2])
pathtoimg = images[i]
img = '<small class="card-title">'+images[i].split("/")[3]+'</small><br/>\
<a target="_blank" href="'+pathtoimg + \
'"><img src="'+pathtoimg + \
'" class="card-img-top" alt="image goes here"></a>'
imglink ='<div class = "card" style = "width: 13rem;" >\
'+img+'\
</div>'
htmlmid=htmlmid+imglink
htmlmid=htmlmid+'</p></div>'
htmlend = '</div></html>'
html = htmlstart+htmlmid+htmlend
# create the html file
path = os.path.abspath('temp.html')
url = 'file://' + path
with open(path, 'w') as f:
f.write(html)
webbrowser.open(url)
else:
html = '<html> <head>\
<meta charset = "utf-8" >\
<meta name = "viewport" content = "width=device-width, initial-scale=1" > <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous"> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/js/bootstrap.bundle.min.js" integrity="sha384-b5kHyXgcpbZJO/tY9Ul7kGkf1S0CWuKcCD38l8YkeH8z8QjE0GmW1gYU5S9FOnJ0" crossorigin="anonymous"></script> <style> img {border: 2px solid # ddd; /* Gray border */border-radius: 4px; /* Rounded border */padding: 5px; /* Some padding */width: 400px; /* Set a small width */}/* Add a hover effect (blue shadow) */img:hover {box-shadow: 0 0 2px 1px rgba(0, 140, 186, 0.5);}</style> \
<title>Plots</title>\
</head>\
<div class="container-fluid"> \
<div class="row">\
<hr>\
<h1> Plotting folder '+folderName.upper() + '''</h1>
<p> No data to show</p>
</div>
</div>
</html>
'''
# create the html file
path = os.path.abspath('temp.html')
url = 'file://' + path
with open(path, 'w') as f:
f.write(html)
webbrowser.open(url)
print("No data to show")
else:
# if txt=="OBSDATE":
print(f"{txt} filter not implemented yet\n")
else:
print('Invalid entry for filter')
elif filter =="between":
print("\nFilter by: ", option)
print('Key: ', filter)
# get params
text1=self.plot_ui.txtBoxStart.text()
text2=self.plot_ui.txtBoxEnd.text()
print(f'Values: {text1}, {text2}')
if option=="":
print("Need to choose an option")
else:
if filter=="":
print("Plotting everything")
else:
if text1=="" and text2=="":
# show everything
print("no value added, showing all images")
else:
# run filter
#print("All checked")
# get data from db
# check folder name against table name
plotFolderName=self.plot_ui.comboBox.currentText()
folderName=plotFolderName.lower().replace(".","_")
print(folderName,self.tables)
if folderName in self.tables:
print("\n",folderName, " found")
#target=folderName
db=""
# get column names from db and put them in combobox
self.colInd, self.colName, self.colType = self.db.get_all_table_coloumns(folderName)
rows = self.db.get_rows(folderName)
# create dataframe
df = pd.DataFrame(list(rows), columns = self.colName)
# df = df[(df['OBSDATE']>="2008-01-01") & (df['OBSDATE']<="2018-12-31")]
try:
txt1=float(text1)
except:
print("\nOnly integers or floats accepted\n")
txt1=""
try:
txt2=float(text2)
except:
print("\nOnly integers or floats accepted\n")
txt2=""
print(f'start: {txt1}, end: {txt2}\n')
print(f'start: {type(txt1)}, end: {type(txt2)}\n')
print(f'start: {text1}, end: {text2}\n')
if type(txt1).__name__ != "str" and type(txt2).__name__ != "str":
df[option] = df[option].astype(float)
ndf = df[(df[option] >= txt1)& (df[option] <= txt2)]
print(len(ndf))
elif type(txt1).__name__ == "str" and type(txt2).__name__ != "str":
df[option] = df[option].astype(float)
ndf = df[(df[option] <= txt2)]
print(len(ndf))
elif type(txt1).__name__ != "str" and type(txt2).__name__ == "str":
df[option] = df[option].astype(float)
ndf = df[(df[option] >= txt1)]
print(len(ndf))
elif type(txt1).__name__ == "str" and type(txt2).__name__ == "str":
print(option)
if option == 'OBSDATE':
df["OBSDATE"] = pd.to_datetime(df["OBSDATE"], format="%Y-%m-%d")
# T2=datetime.strptime(text2, '%Y-%m-%d')
# print(T2)
# print(txt1)
if text1=="" and text2!="":
try:
T2=datetime.strptime(text2, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
df = df[df[option] <= T2]
elif text1!="" and text2=="":
try:
T1=datetime.strptime(text1, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
df = df[df[option] >= T1]
elif text1!="" and text2!="":
try:
T1=datetime.strptime(text1, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
try:
T2=datetime.strptime(text2, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
df = df[(df[option] >= T1)& (df[option] <= T2)]
else:
print()
print ('Invalid date formats, please use "yyyy-mm-dd" for start and end dates')
sys.exit()
else:
print('showing everything may freeze yyour machine')
try:
df = df[(df[option] >= txt1)& (df[option] <= txt2)]
except:
print('Invalid entry/entries, check values correspond to table entries')
sys.exit()
else:
print("Failed to find working solution")
sys.exit()
if len(ndf) > 0:
# ndf=df
#print(ndf['FILENAME'])
ndf['plot_tag']=ndf['FILENAME'].apply(lambda x:x[:18])
plots=list(ndf['plot_tag'])
#print(ndf[['plot_tag',option]])
# print stats
print("\n--- Basic stats ---\n")
print("Min: ", ndf[option].min())
print("Max: ", ndf[option].max())
print("Mean: ", ndf[option].mean())
print("Median: ", ndf[option].median())
print("-"*20,"\n")
# get plot folder and plot to browser
imgDir = "plots/"
msg_wrapper("debug", self.log.debug,"Plotting folder "+plotFolderName)
ls=os.listdir(imgDir)
for fn in ls:
fn=fn.strip()
if ".DS_Store" in fn:
pass
else:
f=fn.replace(".","_")
# f=f.replace("-","_")
#print(fn,type(f),f,folderName,len(fn),len(folderName))
if f.lower()== folderName.lower():
print(">>",f,folderName)
folderName=fn
break
# fld=(folder.replace(".","_")).lower()
# if fld==plotFolderName:
# plotFolderName=folder
# print(f'{folder} = {fld} = {plotFolderName}')
if "_"==folderName[0]:
foldName=folderName[1:]
else:
foldName=folderName
src=(ndf["OBJECT"].iloc[0]).replace(" ","")
s=f"{src}_{foldName.upper().split('_')[-1]}"
imgPath=f'plots/{src}/{s}/'
dr = os.listdir(imgPath)
imageNames=sorted(dr)
images=[]
if ".DS_Store" in imageNames:
imageNames.remove(".DS_Store")
print(imageNames,'\n')
# print(plots)
# sys.exit()
# # get images for current point
for i in range(len(plots)):
for j in range(len(imageNames)):
if plots[i] in imageNames[j]: #[:18]:
if "SL" in option and "HPS_LCP" in imageNames[j]:
#print(imageNames[i],plots[j])
images.append(imgPath+imageNames[j])
elif "SR" in option and "HPS_RCP" in imageNames[j]:
#print(imageNames[i],plots[j])
images.append(imgPath+imageNames[j])
elif "NL" in option and "HPN_LCP" in imageNames[j]:
#print(imageNames[i],plots[j])
images.append(imgPath+imageNames[j])
elif "NR" in option and "HPN_RCP" in imageNames[j]:
#print(imageNames[i],plots[j])
images.append(imgPath+imageNames[j])
elif "OL" in option and "ON_LCP" in imageNames[j]:
#print(imageNames[i],plots[j])
images.append(imgPath+imageNames[j])
elif "OR" in option and "ON_RCP" in imageNames[j]:
#print(imageNames[i],plots[j])
images.append(imgPath+imageNames[j])
else:
images.append(imgPath+imageNames[i])
#print(images)
# html link
htmlstart = '<html> <head>\
<meta charset = "utf-8" >\
<meta name = "viewport" content = "width=device-width, initial-scale=1" > <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous"> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/js/bootstrap.bundle.min.js" integrity="sha384-b5kHyXgcpbZJO/tY9Ul7kGkf1S0CWuKcCD38l8YkeH8z8QjE0GmW1gYU5S9FOnJ0" crossorigin="anonymous"></script> <style> img {border: 2px solid # ddd; /* Gray border */border-radius: 4px; /* Rounded border */padding: 5px; /* Some padding */width: 400px; /* Set a small width */}/* Add a hover effect (blue shadow) */img:hover {box-shadow: 0 0 2px 1px rgba(0, 140, 186, 0.5);}</style> \
<title>Plots</title>\
</head>\
<div class="container-fluid"> \
<div class="row">\
<hr>\
<h1> Plotting folder '+foldName.upper() + '</h1> <p>'
htmlmid=''
imageOptions=[]
if "OL" in option or "SL" in option or "NL" in option:
for img in images:
if "LCP" in img:
imageOptions.append(img)
images=imageOptions
elif "OR" in option or "SR" in option or "NR" in option:
for img in images:
if "RCP" in img:
imageOptions.append(img)
images=imageOptions
else:
images=images
# add images to html page
for i in range(len(images)):
# print(images.split("/")[1],images.split("/")[2])
# print(images[i])
pathtoimg = images[i]
img = '<small class="card-title">'+images[i].split("/")[3]+'</small><br/>\
<a target="_blank" href="'+pathtoimg + \
'"><img src="'+pathtoimg + \
'" class="card-img-top" alt="image goes here"></a>'
imglink ='<div class = "card" style = "width: 10rem;" >\
'+img+'\
</div>'
htmlmid=htmlmid+imglink
htmlmid=htmlmid+'</p></div>'
htmlend = '</div></html>'
html = htmlstart+htmlmid+htmlend
# create the html file
path = os.path.abspath('temp.html')
url = 'file://' + path
with open(path, 'w') as f:
f.write(html)
webbrowser.open(url)
else:
html = '<html> <head>\
<meta charset = "utf-8" >\
<meta name = "viewport" content = "width=device-width, initial-scale=1" > <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BmbxuPwQa2lc/FVzBcNJ7UAyJxM6wuqIj61tLrc4wSX0szH/Ev+nYRRuWlolflfl" crossorigin="anonymous"> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta2/dist/js/bootstrap.bundle.min.js" integrity="sha384-b5kHyXgcpbZJO/tY9Ul7kGkf1S0CWuKcCD38l8YkeH8z8QjE0GmW1gYU5S9FOnJ0" crossorigin="anonymous"></script> <style> img {border: 2px solid # ddd; /* Gray border */border-radius: 4px; /* Rounded border */padding: 5px; /* Some padding */width: 400px; /* Set a small width */}/* Add a hover effect (blue shadow) */img:hover {box-shadow: 0 0 2px 1px rgba(0, 140, 186, 0.5);}</style> \
<title>Plots</title>\
</head>\
<div class="container-fluid"> \
<div class="row">\
<hr>\
<h1> Plotting folder '+folderName.upper() + '''</h1>
<p> No data to show</p>
</div>
</div>
</html>
'''
# create the html file
path = os.path.abspath('temp.html')
url = 'file://' + path
with open(path, 'w') as f:
f.write(html)
webbrowser.open(url)
print("No data to show")
else:
print(f"\n{folderName} not found in {self.tables}\n")
else:
print("\nFilter by: ", option)
print('Key: ', filter)
print('Min: ', self.plot_ui.txtBoxStart.text())
print('Max: ', self.plot_ui.txtBoxEnd.text())
[docs]
def delete_obs(self):
''' Delete an observation. '''
option=self.plot_ui.comboBoxOptions.currentText()
filter=self.plot_ui.comboBoxFilter.currentText()
txt=self.plot_ui.txtBoxFilter.text()
print(f'\noption: {option}, filter: {filter}, text: {txt}\n')
try:
txt=float(txt)
except:
pass
# print('-'* 5, txt)
if txt!=None and type(txt).__name__ !='str':
print('in txt')
if option == "":
print("Choose an option")
else:
if filter is not None and txt is not None:
# check folder name against table name
folderName=self.plot_ui.comboBox.currentText()
print('opt: ',option,", filt: ",filter,', table: ',folderName)
# sys.exit()
df,db = self.get_data_from_db(folderName)
# df = df[(df['OBSDATE']>="2008-01-01") & (df['OBSDATE']<="2018-12-31")]
txt=float(txt)
if type(txt).__name__ != "str":
#txt=float(txt)
df[option] = df[option].astype(float)
if filter == ">":
ndf = df[df[option] > txt]
print(len(ndf))#,option,txt ,df[option])
elif filter == ">=":
ndf = df[df[option] >= txt]
print(len(ndf))
elif filter == "<":
ndf = df[df[option] < txt]
print(len(ndf))
elif filter == "<=":
ndf = df[df[option] <= txt]
print(len(ndf))
elif filter == "=":
ndf = df[df[option] == txt]
print(len(ndf))
else:
ndf=df
ndf=ndf.sort_values('FILENAME')
cols=list(ndf.columns)
files=list(ndf.FILENAME)
print(f'cols: {len(cols)}, files: {len(files)}')
# print(cols)
# sys.exit()
if "FLUXTOT" in cols:
for j in files:
stmt=f"UPDATE '{folderName}' SET "
if "OL" in option:
ols=[]
for i in cols:
if 'OL' in i or 'NL' in i or 'SL' in i:
ols.append(i)
# print(ols)
for k in ols:
stmt=stmt+f"{k}='{np.nan}', "
print(stmt)
sys.exit()
else:
stmt=stmt+f"{option}='{np.nan}', "
print()
stmt=stmt[:-2]+f" WHERE FILENAME='{j}'; \n"
print(stmt)
db.c.execute(stmt)
db.commit_changes()
print('done')
# sys.exit()
else:
for j in files:
print('\nprocessing_file : ',j)
stmt=f"UPDATE '{folderName}' SET "
# print(option)
# sys.exit()
# if "DTA" in option:
# s = option.replace("D",'')
# print(s,option)
# sys.exit()
if "OL" in option:
ols=[i for i in cols if 'OL' in i]
print(ols)
for k in ols:
if "FLAG" in k:
stmt=stmt+f"{k}=200, TSYS1='{np.nan}', "
else:
stmt=stmt+f"{k}='{np.nan}', "
elif "OR" in option:
#if "OL" in option:
ors=[i for i in cols if 'OR' in i]
print(ors)
for k in ors:
if "FLAG" in k:
stmt=stmt+f"{k}=200, TSYS2='{np.nan}', "
else:
stmt=stmt+f"{k}='{np.nan}', "
elif 'NL' in option:
nl=[i for i in cols if 'NL' in i]
print(nl)
# sys.exit()
for k in nl:
if "FLAG" in k:
stmt=stmt+f"{k}=200, TSYS2='{np.nan}', "
else:
stmt=stmt+f"{k}='{np.nan}', "
elif 'SL' in option:
sl=[i for i in cols if 'SL' in i]
print(sl)
# sys.exit()
for k in sl:
if "FLAG" in k:
stmt=stmt+f"{k}=200, TSYS2='{np.nan}', "
else:
stmt=stmt+f"{k}='{np.nan}', "
elif 'SR' in option:
sr=[i for i in cols if 'SR' in i]
print(sr)
# sys.exit()
for k in sr:
if "FLAG" in k:
stmt=stmt+f"{k}=200, TSYS2='{np.nan}', "
else:
stmt=stmt+f"{k}='{np.nan}', "
elif 'NR' in option:
nr=[i for i in cols if 'NR' in i]
print(nr)
# sys.exit()
for k in nr:
if "FLAG" in k:
stmt=stmt+f"{k}=200, TSYS2='{np.nan}', "
else:
stmt=stmt+f"{k}='{np.nan}', "
stmt=stmt[:-2]+f" WHERE FILENAME='{j}'; \n"
print(f'*** {stmt}\n')
# sys.exit()
db.c.execute(stmt)
db.commit_changes()
print('done')
else:
pass
elif filter=='between':
text1=self.plot_ui.txtBoxStart.text()
text2=self.plot_ui.txtBoxEnd.text()
# check folder name against table name
folderName=self.plot_ui.comboBox.currentText()
print('opt: ',option,", filt: ",filter,', table: ',folderName)
# db = SQLiteDB(databaseName=self.dbFilePath, log=self.log)
# db.create_db()
tableNames = sorted(self.db.get_table_names(self.dbFilePath))
colInd, colNames, colType = self.db.get_all_table_coloumns(folderName)
rows = self.db.get_rows_of_cols(folderName,colNames)
#print(folderName,colNames)
# create dataframe
df = pd.DataFrame(list(rows), columns = colNames)
df = df[(df['OBSDATE']>="2008-01-01") & (df['OBSDATE']<="2018-12-31")]
if folderName in self.tables:
try:
txt1=float(text1)
except:
# print("\nOnly integers or floats accepted\n")
txt1=""
try:
txt2=float(text2)
except:
# print("\nOnly integers or floats accepted\n")
txt2=""
print(f'start: {txt1}, end: {txt2}\n')
if type(txt1).__name__ != "str" and type(txt2).__name__ != "str":
df[option] = df[option].astype(float)
df = df[(df[option] >= txt1)& (df[option] <= txt2)]
# print(len(df))
elif type(txt1).__name__ == "str" and type(txt2).__name__ != "str":
df[option] = df[option].astype(float)
df = df[(df[option] <= txt2)]
# print(len(df))
elif type(txt1).__name__ != "str" and type(txt2).__name__ == "str":
df[option] = df[option].astype(float)
df = df[(df[option] >= txt1)]
# print(len(df))
elif type(txt1).__name__ == "str" and type(txt2).__name__ == "str":
if option == 'OBSDATE':
if text1=="" and text2!="":
try:
T2=datetime.strptime(text2, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
df = df[df[option] <= T2]
elif text1!="" and text2=="":
try:
T1=datetime.strptime(text1, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
df = df[df[option] >= T1]
elif text1!="" and text2!="":
try:
T1=datetime.strptime(text1, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
try:
T2=datetime.strptime(text2, '%Y-%m-%d')
except:
print ('Invalid date format, please use "yyyy-mm-dd"')
sys.exit()
df = df[(df[option] >= T1)& (df[option] <= T2)]
else:
print ('Invalid date formats, please use "yyyy-mm-dd" for start and end dates')
sys.exit()
else:
print('showing everything may freeze yyour machine')
try:
df = df[(df[option] >= text1)& (df[option] <= text2)]
except:
print('Invalid entry/entries, check values correspond to table entries')
sys.exit()
else:
print("Failed to find working solution")
sys.exit()
cols=list(df.columns)
files=list(df.FILENAME)
print(f'cols: {len(cols)}, files: {len(files)}')
for j in files:
stmt=f"UPDATE {folderName} SET "
if "OL" in option:
ols=[i for i in cols if 'OL' in i]
print(ols)
for k in ols:
if "FLAG" in k:
stmt=stmt+f"{k}=200, "
else:
stmt=stmt+f"{k}='{np.nan}', "
elif "OR" in option:
#if "OL" in option:
ors=[i for i in cols if 'OR' in i]
print(ors)
for k in ors:
if "FLAG" in k:
stmt=stmt+f"{k}=200, "
else:
stmt=stmt+f"{k}='{np.nan}', "
else:
for col in cols:
if 'SL' in col or 'SR' in col or 'OL' in col or 'OR' in col or 'NL' in col or 'NR' in col :
if "FLAG" in col:
stmt=stmt+f"{col}=200, "
else:
stmt=stmt+f"{col}='{np.nan}', "
else:
pass
#or 'OBSTIME' in col or 'MJD' in col or 'OBJECT' in col or 'OBJECTTYPE' in col or 'CENTFREQ' in col or 'LOGFREQ' in col or 'FOCUS' in col or 'TILT' in col or 'HA' in col or 'ZA' in col or 'RADIOMETER' in col or 'BANDWDTH' in col or 'TCAL1' in col or 'TCAL2' in col or 'NOMTSYS' in col or 'TSYS1', 'TSYSERR1', 'TSYS2', 'TSYSERR2', 'TAMBIENT', 'PRESSURE', 'HUMIDITY', 'WINDSPD', 'HPBW', 'FNBW', 'ELEVATION', 'LONGITUDE', 'LATITUDE', 'SCANTYPE', 'BEAMTYPE', 'OBSERVER', 'OBSLOCAL', 'PROJNAME', 'PROPOSAL', 'TELESCOPE', 'UPGRADE', 'INSTFLAG', 'SCANDIST', 'SCANTIME', 'PWV', 'SVP', 'AVP', 'DPT', 'WVD', 'MEAN_ATMOS_CORRECTION', 'TAU10', 'TAU15', 'TBATMOS10', 'TBATMOS15', 'SLTA', 'SLDTA', 'SLS2N', 'SLTAPEAKLOC', 'SLFLAG', 'SLRMSB', 'SLRMSA', 'SLBSLOPE', 'SLBSRMS', 'NLTA', 'NLDTA', 'NLS2N', 'NLTAPEAKLOC', 'NLFLAG', 'NLRMSB', 'NLRMSA', 'NLBSLOPE', 'NLBSRMS', 'OLTA', 'OLDTA', 'OLPC', 'COLTA', 'COLDTA', 'OLS2N', 'OLTAPEAKLOC', 'OLFLAG', 'OLRMSB', 'OLRMSA', 'OLBSLOPE', 'OLBSRMS', 'SRTA', 'SRDTA', 'SRS2N', 'SRTAPEAKLOC', 'SRFLAG', 'SRRMSB', 'SRRMSA', 'SRBSLOPE', 'SRBSRMS', 'NRTA', 'NRDTA', 'NRS2N', 'NRTAPEAKLOC', 'NRFLAG', 'NRRMSB', 'NRRMSA', 'NRBSLOPE', 'NRBSRMS', 'ORTA', 'ORDTA', 'ORPC', 'CORTA', 'CORDTA', 'ORS2N', 'ORTAPEAKLOC', 'ORFLAG', 'ORRMSB', 'ORRMSA', 'ORBSLOPE', 'ORBSRMS'
stmt=stmt[:-2]+f" WHERE FILENAME='{j}'; \n"
print(stmt)
# db.c.execute(stmt)
# db.commit_changes()
sys.exit()
else:
print(f"\n{folderName} not found in {self.tables}\n")
else:
print("No valid entries detected for filter")
#pass
# checked end ----------
# -----------------------------
# == Widget connections
[docs]
def connect_buttons(self):
""" Connect button key presses to widgets. """
msg_wrapper("debug", self.log.debug,"Connecting buttons to main canvas")
# Plot parameters buttons
self.drift_ui.BtnFilterData.clicked.connect(self.filter_data)
self.drift_ui.BtnViewFit.clicked.connect(self.view_fit)
# PSS values buttons
self.drift_ui.BtnCalc.setEnabled(False)
self.drift_ui.BtnCalc.setText("Calc PSS/Flux")
# self.drift_ui.BtnCalc.clicked.connect(self.calc)
# self.drift_ui.BtnPopulatePSS.clicked.connect(self.populate_pss_from_file)
# beam fitting buttons
self.drift_ui.BtnFitData.clicked.connect(self.fit_data)
self.drift_ui.BtnClearSelection.clicked.connect(self.clear_selection)
self.drift_ui.BtnSave.clicked.connect(self.save_fit)
# Current plot selection
self.drift_ui.BtnChoosePlot.clicked.connect(self.choose_plot)
self.drift_ui.BtnOpenFile.clicked.connect(self.open_file)
self.drift_ui.BtnResetPlot.clicked.connect(self.reset_plot)
self.popup_msg_dict()
self.drift_ui.BtnSaveToDb.clicked.connect(self.save_to_DB)
# Current fit status buttons
self.drift_ui.BtnResetStatus.clicked.connect(self.reset_status)
self.drift_ui.BtnViewStatus.clicked.connect(self.view_status)
[docs]
def update_plot(self, x, y, title, res, label="", label2=""):
""" Update the current figure. """
msg_wrapper("debug", self.log.debug,
"Update the current figure.")
self.Canvas.plot_figure(
x, y, title, label)
self.Canvas.plot_residual(res, label=label2)
# === connections for fit_data() ===
[docs]
def get_fit_type(self):
""" Get fit type from combobox."""
msg_wrapper("debug", self.log.debug, "Get fit type")
return self.drift_ui.ComboBoxFitType.currentText()
[docs]
def get_fit_order(self):
""" Get order for fitting polynomial."""
msg_wrapper("debug", self.log.debug, "Get polynomial order")
return int(self.drift_ui.ComboBoxFitOrder.currentText())
[docs]
def get_fit_loc(self):
""" Get fit location from combobox. """
msg_wrapper("debug", self.log.debug, "Get fit location")
return self.drift_ui.ComboBoxFitLoc.currentText()
[docs]
def clear_selection(self):
""" Clear selections made for current scan. """
self.write("Clearing selections for current scan.",'info')
# Get the plot index
plotIndex = self.get_plot_index()
# Set plot title
title = self.set_title()
self.Canvas.plot_figure(self.x, self.y, title, "Cleared selection")
self.Canvas.plot_residual()
self.set_fit_parmeters()
self.set_flags()
[docs]
def fit_data(self):
""" Fit the drift scan. """
# Open plot manager
plotIndex = self.get_plot_index()
# Get order to fit
order = self.get_fit_order()
fit_type = self.get_fit_type()
if fit_type == "Polynomial":
# Get location we are fitting
fit_loc = self.get_fit_loc()
# Check the selected fit type
if fit_loc == "Base":
# Get base locations
if len(self.Canvas.click_index) == 0:
self.write("No points selected yet",'info')
self.clear_selection()
else:
self.write("Fitting the baseline",'info')
# ensure we have a minimum of 4 points for the base fit
if len(self.Canvas.click_index) < 4 or len(self.Canvas.click_index) % 2 != 0:
self.write("Need a minimum of 4 points or an even set to fit a baseline.\nClicked points reset, pick new points",'info')
self.clear_selection()
self.write(f"selected points: {self.Canvas.click_index}",'info')
self.Canvas.click_index = []
else:
# prep Canvas and perform fit
self.Canvas.clear_canvas() # Update figures
# Perform baseline fit
self.write("Fitting "+fit_loc+" with order "+str(order)+" "+fit_type,'info')
self.reset_peak(plotIndex)
self.fit_base(order, plotIndex)
self.update_base_fit(plotIndex) # Update plot
self.peak_is_fit = 0
elif fit_loc == "Peak":
self.write("Fitting peak with poly",'info')
# Get base locations
if len(self.Canvas.click_index) == 0:
self.write("No points selected yet. ","info")
self.peak_is_fit = 0
self.clear_selection()
else:
# ensure we have a minimum of 2 points for the peak fit
if len(self.Canvas.click_index) < 2 or len(self.Canvas.click_index) % 2 != 0:
self.write("Need a minimum of 2 points or an even set to fit a peak.\nClicked points reset, pick new points",'info')
self.peak_is_fit = 0
self.Canvas.click_index = []
self.clear_selection()
else:
# Perform peak fit
self.write("Fitting "+fit_loc+" with order "+str(order)+" "+fit_type, 'info')
#CHECK IF DUAL BEAM
print(self.dataKeys)
# sys.exit()
if self.data['INSTRUME'] == "Dicke Switched":
# Update figures
self.Canvas.clear_dual_canvas()
self.fit_dual_peak(order, plotIndex)
self.update_dual_peak_fit(plotIndex)
else:
# Update figures
self.Canvas.clear_canvas()
self.fit_peak(order, plotIndex)
self.update_peak_fit(plotIndex) # Update plot
self.peak_is_fit = 1
#self.view_fit()
self.Canvas.click_index = []
#self.clear_selection()
'''elif fit_type == "Gaussian":
msg_wrapper("info", self.log.info,
"Fitting peak with gaussian\n")
# Get base locations
if len(self.Canvas.click_index) == 0:
msg_wrapper("info", self.log.info,
"No points selected yet. ")
self.peak_is_fit = 0
else:
# ensure we have a minimum of 2 points for the peak fit
if len(self.Canvas.click_index) < 2 or len(self.Canvas.click_index) % 2 != 0:
msg_wrapper("info", self.log.info,
"We need a minimum of 2 points or an even set to fit a baseline")
msg_wrapper("info", self.log.info,
"Clicked points reset, pick new points")
self.peak_is_fit = 0
self.Canvas.click_index = []
else:
# Perform baseline fit
self.fit_peak(order, plotIndex)
# Update figures
self.Canvas.clear_canvas()
self.update_peak_fit(plotIndex) # Update plot
self.peak_is_fit = 1
self.view_fit()
self.Canvas.click_index = []
'''
[docs]
def update_dual_peak_fit(self, plotIndex):
"""
Updates figure on Canvas
"""
self.write("Update peak fit",'info')
plotTitle = self.set_title()
self.drift_ui.ComboBoxFitOrder.currentText()
self.Canvas.plot_dual_residuals(self.peakResA, self.peakResB,label="Peak residual")
if len(self.xpA) == 0 and len(self.xpB) != 0:
self.Canvas.plot_dual_peaks([], [],
0.0, 0.0, self.xpB, self.peakModelB,
self.peakFitB, self.peakRmsB, self.x, self.y, plotTitle)
elif len(self.xpA) != 0 and len(self.xpB) == 0:
self.Canvas.plot_dual_peaks(self.xpA, self.peakModelA,
self.peakFitA, self.peakRmsA, [], [],
0.0, 0.0, self.x, self.y, plotTitle)
elif len(self.xpA) != 0 and len(self.xpB) != 0:
self.Canvas.plot_dual_peaks(self.xpA, self.peakModelA,
self.peakFitA, self.peakRmsA, self.xpB, self.peakModelB,
self.peakFitB, self.peakRmsB, self.x, self.y, plotTitle)
[docs]
def update_peak_fit(self, plotIndex):
"""
Updates figure on Canvas
"""
self.write("Update peak fit ",'info')
plotTitle = self.set_title()
self.drift_ui.ComboBoxFitOrder.currentText()
self.Canvas.plot_residual(self.peakRes, label="Peak residual")
self.Canvas.plot_peak(self.xp, self.peakModel, self.peakFit,
self.peakRms, self.x, self.y, plotTitle)
[docs]
def fit_base(self, order, plotIndex):
""" Fit the baseline. """
self.write("Fitting baseline @ "+str(self.Canvas.click_index))
self.write("Fit points: "+str(self.x[self.Canvas.click_index]))
# Create tags for the plot file
self.tag = self.create_tag(plotIndex)
# Get baseline fitting locations
self.xb, self.yb = fit.get_base_pts(
self.x, self.y, self.Canvas.click_index)
# Fit the baseline
self.dataModel, self.driftModel, self.driftRes, self.base_rms, self.driftCoeffs = fit.correct_drift(
self.xb, self.yb, self.x, self.log,order)
# Set the slope,rms,x and y values
self.baseSlope = self.driftCoeffs[0]
self.write(
"y = {:.3f}x + {:.3f}".format(self.baseSlope, self.driftCoeffs[1]))
self.y = np.array(self.y - self.dataModel)
self.x = np.array(self.x)
# update fit parameter
self.base_is_fit = 1
[docs]
def create_tag(self, ind):
""" Create a tag for the plots according to the current scan being processed. e.g. ONRCP
"""
msg_wrapper("debug", self.log.debug, "Setting up naming tags")
#print(f'index: {ind}')
if int(ind) > 2:
tag = self._set_file_tag_nb()
else:
tag = self._set_file_tag_wb()
return tag
def _set_file_tag_wb(self):
"""
A naming tag for wide beam data.
"""
msg_wrapper("debug", self.log.debug, "Setting tag for wide beam data")
plotIndex = self.get_plot_index()
tagId = int(plotIndex)
if tagId == 0:
self.write("Processing: ONLCP\n")
return "ON_LCP"
else:
self.write("rocessing: ONRCP\n")
return "ON_RCP"
[docs]
def update_base_fit(self, plotIndex):
""" Updates figure on Canvas """
# Update plot
self.write("Figure updated with fit")
plotTitle = self.set_title()
pi = self.Canvas.click_index
self.update_plot(self.x, self.y, plotTitle, self.driftRes,
"Data: baseline corrected", label2="")
pi = []
[docs]
def fit_peak(self, order, plotIndex):
""" Fit peak """
self.write("Fitting the Peak","info")
self.xp, self.yp = fit.get_base_pts(
self.x, self.y, self.Canvas.click_index)
self.write("locs: "+str(self.x[self.Canvas.click_index]),'info')
if self.drift_ui.ComboBoxFitType.currentText() == "Polynomial":
# poly fit
self.write("Fitting a polynomial",'info')
degree = int(self.drift_ui.ComboBoxFitOrder.currentText())
peak_res, peakRms, peakModel = fit.fit_poly_peak(
self.xp, self.yp, order=degree, log=self.log) # Fit the peak
peakLoc = self.xp[((np.where(peakModel == max(peakModel))[0])[0])]
self.peakFit = max(peakModel)
self.peakRms = peakRms
self.peakRes = peak_res
self.peakModel = peakModel
self.peakLoc = peakLoc
# Calculate s2n
try:
# calculate s2n using drift residual
self.s2n = misc.sig_to_noise(self.peakFit, self.driftRes,self.log)
except Exception:
# else calculate s2n using peak fit residual
# if base is not corrected first
self.s2n = misc.sig_to_noise(self.peakFit, self.peakRes,self.log)
'''elif self.drift_ui.ComboBoxFitType.currentText() == "Gaussian":
# gauss fit
msg_wrapper("info", self.log.info,
"Fitting a gaussian")
p0 = [max(self.yp), self.xp[np.argmax(self.yp)],
self.data["HPBW"]/2]
gaus_coeff, var_matrix, gauss_fit, err_gauss_fit, res = fit.fit_gauss(self.xp, self.yp, p0) # Fit the peak
peakLoc = self.x[((np.where(peakModel == max(peakModel))[0])[0])]
self.peakFit = max(gauss_fit)
self.peakRms = err_gauss_fit
self.peakRes = res
self.peakModel = gauss_fit
self.peakLoc = peakLoc
# Calculate s2n
try:
# calculate s2n using drift residual
self.s2n = fit.sig_to_noise(self.peakFit, self.driftRes)
except Exception:
# else calculate s2n using peak fit residual
# if base is not corrected first
self.s2n = fit.sig_to_noise(self.peakFit, self.peakRes)
'''
# Calculate PSS
self.update_fit_parmeters()
self.write("Ta = {:.3f} +- {:.3f} [K]".format(self.peakFit, self.peakRms),'info')
[docs]
def fit_dual_peak(self, order, plotIndex):
""" Fit dual peaks. """
self.write("Fitting the Peaks",'info')
# get values of both peak click indeces
p1ClickIndex, p2ClickIndex = [], []
scan_mid=int(len(self.x)/2)
[p1ClickIndex.append(n) for n in self.Canvas.click_index if n <= scan_mid]
[p2ClickIndex.append(n) for n in self.Canvas.click_index if n > scan_mid]
# fit the peaks
if len(p1ClickIndex) !=0 and len(p2ClickIndex) !=0:
self.fit_p1(p1ClickIndex)
self.fit_p2(p2ClickIndex)
if len(p1ClickIndex) !=0 and len(p2ClickIndex) ==0:
self.fit_p1(p1ClickIndex)
self.fit_p2([])
if len(p1ClickIndex) ==0 and len(p2ClickIndex) !=0:
self.fit_p1([])
self.fit_p2(p2ClickIndex)
# Calculate PSS
self.update_fit_parmeters()
[docs]
def fit_p1(self,locs):
""" Fitting peak 1. """
# Fit peak with selected points
if len(locs) == 0:
# no data, skip fit
self.write("Peak 1 not fit",'info')
self.peakFitA = np.nan
self.peakRmsA = np.nan
self.peakResA = np.nan
self.peakModelA = np.nan
self.peakLocA = np.nan
self.s2nA = np.nan
self.xpA = []
self.ypA = []
else:
if len(locs) % 2 != 0:
self.write("Need a minimum of 2 points or an even set on peakA in order to fit the peak.\nClicked points reset, pick new points",'info')
self.peak_is_fit = 0
self.Canvas.click_index = []
self.clear_selection()
else:
self.xpA, self.ypA = fit.get_base_pts(
self.x, self.y, locs)
self.write("locs: "+str(self.x[locs]),'info')
if self.drift_ui.ComboBoxFitType.currentText() == "Polynomial":
# poly fit
self.write("Fitting a polynomial",'info')
degree = int(self.drift_ui.ComboBoxFitOrder.currentText())
peak_resA, peakRmsA, peakModelA = fit.fit_poly_peak(
self.xpA, self.ypA, order=degree, log=self.log) # Fit the peak
peakLocA = self.x[((np.where(peakModelA == min(peakModelA))[0])[0])]
self.peakFitA = min(peakModelA)
self.peakRmsA = peakRmsA
self.peakResA = peak_resA
self.peakModelA = peakModelA
self.peakLocA = peakLocA
# Calculate s2n
try:
# calculate s2n using drift residual
self.s2nA = fit.sig_to_noise(
self.peakFitA, self.driftResA, self.log)
except Exception:
# else calculate s2n using peak fit residual
# if base is not corrected first
self.s2nA = fit.sig_to_noise(
self.peakFitA, self.peakResA, self.log)
[docs]
def fit_p2(self, locs):
""" Fitting peak 2. """
if len(locs) == 0:
# no data, skip fit
self.write("Peak 2 not fit",'info')
self.peakFitB = np.nan
self.peakRmsB = np.nan
self.peakResB = np.nan
self.peakModelB = np.nan
self.peakLocB = np.nan
self.s2nB = np.nan
self.xpB = []
self.ypB = []
else:
# Fit peak with selected points
if len(locs) % 2 != 0:
self.write(
"Need a minimum of 2 points or an even set on peakA in order to fit the peak.\nClicked points reset, pick new points",'info')
self.peak_is_fit = 0
self.Canvas.click_index = []
self.clear_selection()
else:
self.xpB, self.ypB = fit.get_base_pts(
self.x, self.y, locs)
self.write("locs: "+str(self.x[locs]),'info')
if self.drift_ui.ComboBoxFitType.currentText() == "Polynomial":
# poly fit
self.write("Fitting a polynomial",'info')
degree = int(self.drift_ui.ComboBoxFitOrder.currentText())
peak_resB, peakRmsB, peakModelB = fit.fit_poly_peak(
self.xpB, self.ypB, order=degree, log=self.log) # Fit the peak
peakLocB = self.x[(
(np.where(peakModelB == max(peakModelB))[0])[0])]
self.peakFitB = max(peakModelB)
self.peakRmsB = peakRmsB
self.peakResB = peak_resB
self.peakModelB = peakModelB
self.peakLocB = peakLocB
# Calculate s2n
try:
# calculate s2n using drift residual
self.s2nB = fit.sig_to_noise(
self.peakFitB, self.driftResB, self.log)
except Exception:
# else calculate s2n using peak fit residual
# if base is not corrected first
self.s2nB = fit.sig_to_noise(
self.peakFitB, self.peakResB, self.log)
[docs]
def update_fit_parmeters(self):
""" Update fit parameters. """
msg_wrapper("debug", self.log.debug, "Update fit parameters.")
plotIndex = self.get_plot_index()
if len(self.scans) == 2:
if "13.0S" in self.data["FRONTEND"]:
if self.scans[plotIndex] == "ONLCP":
data_update = ["OLTA", "OLTAERR", "OLS2N",
"OLRMSB", "OLRMSA", "OLBSLOPE", "OLBSRMS", "OLTAPEAKLOC", "OLAPPEFF", "OLPSS", "OLDPSS"]
self.set_data(data_update)
else:
data_update = ["ORTA", "ORTAERR", "ORS2N",
"ORRMSB", "ORRMSA", "ORBSLOPE", "ORBSRMS", "ORTAPEAKLOC", "ORAPPEFF", "ORPSS", "ORDPSS"]
self.set_data(data_update)
else:
pass
else:
if "S" in self.data["FRONTEND"]:
print(self.scans)
print(self.scanKeys)
# sys.exit()
if self.scanKeys[plotIndex] == "HPSLCP":
data_update = ["SLTA", "SLTAERR", "SLS2N",
"SLRMSB", "SLRMSA", "SLBSLOPE", "SLBSRMS", "SLTAPEAKLOC"]
self.set_nb_data(data_update)
elif self.scanKeys[plotIndex] == "HPNLCP":
data_update = ["NLTA", "NLTAERR", "NLS2N",
"NLRMSB", "NLRMSA", "NLBSLOPE", "NLBSRMS", "NLTAPEAKLOC"]
self.set_nb_data(data_update)
elif self.scanKeys[plotIndex] == "ONLCP":
data_update = ["OLTA", "OLTAERR", "OLS2N",
"OLRMSB", "OLRMSA", "OLBSLOPE", "OLBSRMS", "OLTAPEAKLOC"]
self.set_nb_data(data_update)
# if self.data["OBJECTTYPE"] == "CAL":
# self.calc_pss_()
# else:
# pass
elif self.scanKeys[plotIndex] == "HPSRCP":
data_update = ["SRTA", "SRTAERR", "SRS2N",
"SRRMSB", "SRRMSA", "SRBSLOPE", "SRBSRMS", "SRTAPEAKLOC"]
self.set_nb_data(data_update)
elif self.scanKeys[plotIndex] == "HPNRCP":
data_update = ["NRTA", "NRTAERR", "NRS2N",
"NRRMSB", "NRRMSA", "NRBSLOPE", "NRBSRMS", "NRTAPEAKLOC"]
self.set_nb_data(data_update)
else:
data_update = ["ORTA", "ORTAERR", "ORS2N",
"ORRMSB", "ORRMSA", "ORBSLOPE", "ORBSRMS", "ORTAPEAKLOC"]
self.set_nb_data(data_update)
# if self.data["OBJECTTYPE"] == "CAL":
# self.calc_pss_()
# else:
# pass
if "D" in self.data["FRONTEND"]:
if self.scanKeys[plotIndex] == "HPSLCP":
data_update = ["ASLTA", "ASLTAERR", "ASLS2N","ASLTAPEAKLOC", "BSLTA", "BSLTAERR", "BSLS2N","BSLTAPEAKLOC",
"SLRMSB", "SLRMSA", "SLBSLOPE", "SLBSRMS"]
self.set_dual_data(data_update)
elif self.scanKeys[plotIndex] == "HPNLCP":
data_update = ["ANLTA", "ANLTAERR", "ANLS2N", "ANLTAPEAKLOC", "BNLTA", "BNLTAERR", "BNLS2N", "BNLTAPEAKLOC",
"NLRMSB", "NLRMSA", "NLBSLOPE", "NLBSRMS"]
self.set_dual_data(data_update)
elif self.scanKeys[plotIndex] == "ONLCP":
data_update = ["AOLTA", "AOLTAERR", "AOLS2N","AOLTAPEAKLOC", "BOLTA", "BOLTAERR", "BOLS2N","BOLTAPEAKLOC",
"OLRMSB", "OLRMSA", "OLBSLOPE", "OLBSRMS"]
self.set_dual_data(data_update)
# if self.data["OBJECTTYPE"] == "CAL":
# self.set_db_pss()
# else:
# pass
elif self.scanKeys[plotIndex] == "HPSRCP":
data_update = ["ASRTA", "ASRTAERR", "ASRS2N", "ASRTAPEAKLOC", "BSRTA", "BSRTAERR", "BSRS2N", "BSRTAPEAKLOC",
"SRRMSB", "SRRMSA", "SRBSLOPE", "SRBSRMS"]
self.set_dual_data(data_update)
elif self.scanKeys[plotIndex] == "HPNRCP":
data_update = ["ANRTA", "ANRTAERR", "ANRS2N", "ANRTAPEAKLOC", "BNRTA", "BNRTAERR", "BNRS2N", "BNRTAPEAKLOC",
"NRRMSB", "NRRMSA", "NRBSLOPE", "NRBSRMS"]
self.set_dual_data(data_update)
else:
data_update = ["AORTA", "AORTAERR", "AORS2N", "AORTAPEAKLOC", "BORTA", "BORTAERR", "BORS2N", "BORTAPEAKLOC",
"ORRMSB", "ORRMSA", "ORBSLOPE", "ORBSRMS"]
self.set_dual_data(data_update)
# if self.data["OBJECTTYPE"] == "CAL":
# self.set_db_pss()
# else:
# pass
[docs]
def set_dual_data(self, data_update):
""" Set the estimated values in the data dictionary.
"""
try:
self.data[data_update[0]] = self.peakFitA
except:
pass
try:
self.data[data_update[1]] = self.peakRmsA
except:
pass
try:
self.data[data_update[2]] = self.s2nA
except:
pass
try:
self.data[data_update[3]] = self.peakLocA
except:
pass
try:
self.data[data_update[4]] = self.peakFitB
except:
pass
try:
self.data[data_update[5]] = self.peakRmsB
except:
pass
try:
self.data[data_update[6]] = self.s2nB
except:
pass
try:
self.data[data_update[7]] = self.peakLocB
except:
pass
try:
self.data[data_update[8]] = self.rmsb
except:
pass
try:
self.data[data_update[9]] = self.rmsa
except:
pass
try:
self.data[data_update[10]] = self.baseSlope
except:
pass
try:
self.data[data_update[11]] = self.base_rms
except:
pass
[docs]
def set_nb_data(self, data_update):
""" Set the estimated values in the data dictionary. """
try:
self.data[data_update[0]] = self.peakFit
except:
pass
try:
self.data[data_update[1]] = self.peakRms
except:
pass
try:
self.data[data_update[2]] = self.s2n
except:
pass
try:
self.data[data_update[3]] = self.rmsb
except:
pass
try:
self.data[data_update[4]] = self.rmsa
except:
pass
try:
self.data[data_update[5]] = self.baseSlope
except:
pass
try:
self.data[data_update[6]] = self.base_rms
except:
pass
try:
self.data[data_update[7]] = self.peakLoc
except:
pass
[docs]
def plot_new_fig(self):
"""
Plot a new figure from the recently fit data.
"""
self.write("plotting New Fig",'info')
fig = self.Canvas.ax.get_figure()
plot_tag = self.create_tag(len(self.plotBeamName))
print(f'plot tag: {plot_tag}')
fig_name = "plots/" + self.data["SOURCEDIR"] + \
"/" + self.data["FILENAME"][:18] + "_" + plot_tag + ".png"
self.file.create_folder("plots/" + self.data["SOURCEDIR"])
fig.savefig(fig_name)
[docs]
def choose_plot(self):
""" Select the plot to work with. """
msg_wrapper("debug", self.log.debug, "Choose plot")
self.fit_done = 0
plotIndex = self.get_plot_index()
print("Plot Index: ", plotIndex)
self.reset_previous_fits(plotIndex)
# Set default to base fit type
if self.drift_ui.ComboBoxFitLoc.currentText() != "Base":
self.drift_ui.ComboBoxFitLoc.setCurrentIndex(0)
if len(self.scanKeys) == 2:
# Clear the Canvas
self.Canvas.clear_canvas()
# Plot figure
self.x = self.scanKeys["OFFSET"]
scanName = self.scanKeys[plotIndex]
self.y = self.scans[scanName]
self.Canvas.plot_figure(self.x, self.y, label="data")
self.Canvas.plot_residual()
self.set_fit_parmeters()
elif len(self.scanKeys) == 6:
# Clear the Canvas
self.Canvas.clear_canvas()
# Plot figure
self.write("Plotting: " +
self.drift_ui.ComboBoxPlotType.currentText(),'info')
self.x = self.scans["OFFSET"]
scanName = self.scanKeys[plotIndex]
print(scanName.strip("_"), plotIndex, self.scans.keys())
self.y = self.scans[scanName]
self.Canvas.plot_figure(self.x, self.y,label="data")
self.Canvas.plot_residual()
self.set_fit_parmeters()
if plotIndex == 0:
# plot 1,2
self.plotOnCanvas(self,1,2)
elif plotIndex == 1:
# plot 0,2
self.plotOnCanvas(self,0,2)
elif plotIndex == 2:
# plot 0,1
self.plotOnCanvas(self,0,1)
elif plotIndex == 3:
# plot 4,5
self.plotOnCanvas(self,4,5)
elif plotIndex == 4:
# plot 3,5
self.plotOnCanvas(self,3,5)
elif plotIndex == 5:
# plot 3,4
self.plotOnCanvas(self,3,4)
else:
self.write("There are no scans provided. ",'info')
[docs]
def popup_msg_dict(self):
""" Message dictionary for popups."""
self.popup_msg = {
"noBase": "Are you sure you want to save data without fitting the baseline ?",
"noPeak": "Are you sure you want to save data without fitting the peak ?",
"noFit": "Are you sure you want to save data without fitting the plot ?"
}
[docs]
def save_fit(self):
"""
Save the fit. If the data has been previously saved
overwrite the previous fit.
"""
# Update plot settings
obj = self.data['OBJECTTYPE']
beam = self.data['FRONTEND']
plotIndex = self.get_plot_index()
if "S" in beam:
if self.base_is_fit == 0 and self.peak_is_fit == 0:
# Set flag
self.set_flags()
# No fit done on figure
reply = self.popUp(self.popup_msg["noFit"])
if reply == "yes":
self.update_fit_parmeters()
self.plot_new_fig()
self.status[plotIndex] = 2
else:
self.write("No fit done",'info')
self.write("Saving fit> Fit status: %d%d%d" % (
self.base_is_fit, self.peak_is_fit, self.plot_is_smoothed),"info")
elif self.base_is_fit == 0 and self.peak_is_fit == 1:
# Set flag
self.set_flags(101)
# Only peak fit done on figure
reply = self.popUp(self.popup_msg["noBase"])
if reply == "yes":
self.update_fit_parmeters()
self.plot_new_fig()
self.status[plotIndex] = 1
else:
pass
self.write("Saving fit> Fit status: %d%d%d" % (
self.base_is_fit, self.peak_is_fit, self.plot_is_smoothed),"info")
elif self.base_is_fit == 1 and self.peak_is_fit == 0:
# Only base fit done on figure
# Create a popup window to ask if user if sure about saving this
# Set flag
self.set_flags(110)
print(self.data['OLFLAG'])
reply = self.popUp(self.popup_msg["noPeak"])
if reply == "yes":
self.update_fit_parmeters()
self.plot_new_fig()
self.status[plotIndex] = 0
#print(self.status, plotIndex)
else:
pass
self.write("Saving fit> Fit status: %d%d%d" % (
self.base_is_fit, self.peak_is_fit, self.plot_is_smoothed),"info")
elif self.base_is_fit == 1 and self.peak_is_fit == 1:
# Both base and peak were fitted
# Set flag
self.set_flags(111)
self.update_fit_parmeters()
self.view_fit()
self.plot_new_fig()
self.status[plotIndex] = 1
self.write("Saving fit> Fit status: %d%d%d" % (
self.base_is_fit, self.peak_is_fit, self.plot_is_smoothed),"info")
elif "D" in beam:
# Save dual beam fits
if self.base_is_fit == 0 and self.peak_is_fit == 0:
# Set flag
self.set_flags()
# No fit done on figure
reply = self.popUp(self.popup_msg["noFit"])
if reply == "yes":
self.update_fit_parmeters()
self.plot_new_fig()
self.status[plotIndex] = 2
else:
self.write("No fit done",'info')
elif self.base_is_fit == 0 and self.peak_is_fit == 1:
# Set flag
self.set_flags(101)
# Only peak fit done on figure
reply = self.popUp(self.popup_msg["noBase"])
if reply == "yes":
self.update_fit_parmeters()
self.plot_new_fig()
self.status[plotIndex] = 1
else:
pass
elif self.base_is_fit == 1 and self.peak_is_fit == 0:
# Only base fit done on figure
# Create a popup window to ask if user if sure about saving this
# Set flag
self.set_flags(110)
reply = self.popUp(self.popup_msg["noPeak"])
if reply == "yes":
self.update_fit_parmeters()
self.plot_new_fig()
self.status[plotIndex] = 0
else:
pass
elif self.base_is_fit == 1 and self.peak_is_fit == 1:
# Both base and peak were fitted
# Set flag
self.set_flags(111)
self.update_fit_parmeters()
self.view_fit()
self.plot_new_fig()
self.status[plotIndex] = 1
else:
self.write("Cant' save data, basefit not done",'info')
print(self.status)
#sys.exit()
[docs]
def save_to_DB(self):
""" Save fit to database. """
self.write("Saving data to database.")
plotIndex = self.get_plot_index()
if self.data['OBJECTTYPE'] == 'CAL':
# make a copy of the current working db
analDb = "CALDB.db"
# GET TABLE FROM DATABASE
db = SQLiteDB(databaseName=analDb, log=self.log)
db.create_db()
else:
# make a copy of the current working db
analDb = "TARDB.db"
db = SQLiteDB(databaseName=analDb, log=self.log)
db.create_db()
table = db.get_table_name(self.data['SOURCEDIR'])
db.set_table_name(table)
self.write("Updating table: {}\n".format(table),'info')
# Check if file has been processed
try:
files = db.read_data_from_database("FILENAME")
except Exception:
db.create_table(self.data, table)
files = []
# File in DB
# print("File in db: ", self.data['FILENAME'] in files, self.data['FILENAME'])
if self.data['FILENAME'] in files:
self.write("already processed: ".format(self.data['FILENAME']),'info')
question = "Are you sure you want to overwrite the database entry for this source ?"
result = self.popUp(question)
if result == "yes":
self.write("Status: "+ str(self.status),'info')
#print(self.beamName)
#sys.exit()
# Update db entry
db.update_row_in_db(
table, self.data['FILENAME'], self.data, self.status, self.beamName)
self.write('Updated row in table','info')
# Save the figure
self.plot_new_fig()
if 1 in self.status or 2 in self.status:
# update corresponding plots
for i in range(len(self.beamName)):
x = self.x # Data
y = self.y#scans[i]
if str(self.status[i]) == '2' :
pl.plot(x, y, "r", label="raw data")
pl.suptitle(
"Plot "+self.data["FILENAME"][:18]+" of "+self.data["OBJECT"])
pl.title(
self.beamName[i]+" scan at "+str(self.data["CENTFREQ"])+" MHz", fontsize=10)
pl.xlabel("Scandist [deg]")
pl.ylabel("Ta [K]")
pl.legend(loc=2)
pl.savefig(
"plots/"+self.data["SOURCEDIR"]+"/"+self.data["FILENAME"][:18]+"_"+self.plotBeamName[i]+".png")
pl.close()
elif str(self.status[i]) == '1' :
pl.plot(x, y, "C0", label="raw data")
pl.suptitle(
"Plot "+self.data["FILENAME"][:18]+" of "+self.data["OBJECT"])
pl.title(
self.beamName[i]+" scan at "+str(self.data["CENTFREQ"])+" MHz", fontsize=10)
pl.xlabel("Scandist [deg]")
pl.ylabel("Ta [K]")
pl.legend(loc=2)
pl.savefig(
"plots/"+self.data["SOURCEDIR"]+"/"+self.data["FILENAME"][:18]+"_"+self.plotBeamName[i]+".png")
pl.close()
else:
pass
else:
# In the case where drift scans were fit but the
# fit for all the scans are unsatisfactory.
# If there is no data to fit and you want to save
# the raw data to the database, plot raw fits for all
# data and save data as nan in the database
print(self.status)
# sys.exit()
# update corresponding plots
#print(self.scans)
for i in range(len(self.beamName)):
x = self.x#Data
y = self.scans[self.beamName[i]]
pl.plot(x, y, "r", label="raw data")
pl.suptitle(
"Plot "+self.data["FILENAME"][:18]+" of "+self.data["OBJECT"])
pl.title(
self.beamName[i]+" scan at "+str(self.data["CENTFREQ"])+" MHz", fontsize=10)
pl.xlabel("Scandist [deg]")
pl.ylabel("Ta [K]")
pl.legend(loc=2)
pl.savefig(
"plots/"+self.data["SOURCEDIR"]+"/"+self.data["FILENAME"][:18]+"_"+self.plotBeamName[i]+".png")
pl.close()
else:
self.write("Entry not updated in database",'info')
self.write("SELECT POINTS TO FIT",'info')
else:
self.write("Processing new source: " +
self.data['FILENAME'],'info')
question = "Are you sure you want create a new entry for this source in the database ?"
result = self.popUp(question)
if result == "yes":
self.write(
"Saving new entry to database",'info')
# Update db
# If source is a calibrator and is 2280-MHz
# and int(self.data["CENTFREQ"]) == 2280:
if self.data["OBJECTTYPE"] == "CAL":
# Save the figure
self.plot_new_fig()
db.create_table(self.data, table)
tbname = db.validate_table_name(table)
db.populate_table(self.data, tbname)
# and int(self.data["CENTFREQ"]) == 2280:
elif self.data["OBJECTTYPE"] == "TAR":
# Save the figure
self.plot_new_fig()
db.create_table(self.data, table)
tbname = db.validate_table_name(table)
db.populate_table(self.data, tbname)
self.write(
"Added new row to table",'info')
else:
self.write(
"Entry not added to database",'info')
self.write(
"SELECT POINTS TO FIT",'info')
[docs]
def reset_status(self):
self.status=[0,0,0,0,0,0]
[docs]
def view_status(self):
print("# Status: ",self.status)
# # plottinf funcs
# def plot_delete_filter(self,key,option,cols):
# # e.g('OL',option,cols)
# if key in option:
# ls=[]
# for i in cols:
# if key