Read in-situ metadata from *.csv

# -*- coding: latin-1 -*-
'''
Script to import data from a CSV file, creating an in-situ plot 

Run the script, then select a csv file via the provided UI
If data from the same time period is already open in DigitalMicrograph, 
it will be automatically synced with plot produced by this script.
The plot is interactive, so clicking within the plot will update the playback 
position of all concurrent in-situ datasets open in the same workspace.
The interactive plot is not automatically saved to disk, but can be 
saved as a DM4 file for later use.

Here is an example of what a CSV might look like. For this code to work as-is 
the CSV must have this format, with metadata in column 1 and data in subsequent columns.
Any one of End Date/Time, Time Interval, or Duration are sufficient

Metadata Info                       Temperature (°C)
Time Column? (0:False 1:True)       50.004257
0                                   50.004406
Start Date (YYYY-MM-DD)             50.004654
2023-10-18                          50.00338
Start Time (Hrs:min:sec.msec)       50.00135
10:05:07.002                        49.999134
End Date (YYYY-MM-DD)               49.997536
""                                  49.997089
End Time (Hrs:min:sec:msec)         49.998394
""                                  49.999859
Time Interval (s)                   50.001385
""                                  50.002232
Duration (s)                        50.003418
2480.5                              50.004368
Device Type (string)                50.005623
Heating2                            50.006329
Device Name (string)                50.005596
Wildfire Holder Test                50.003571

Code written by Ben Miller
Last Updated May 2025
'''

import pandas as pd
import numpy as np
import os, sys
import re
import tkinter as tk
from pathlib import PureWindowsPath
import tkinter.filedialog as tkfd
from datetime import datetime, timedelta

# Function to get file location using file dialog
def GetSelectedFile():
    """
    Function to Get File Location with a File Dialog

    Returns:
        listOfFiles (list of strings): List of files selected by user
    """
    sys.argv.extend(['-a', ' '])
    root = tk.Tk()
    root.withdraw()  # Use to hide tkinter window
    currdir = os.getcwd()
    listOfFiles = list(tkfd.askopenfilenames(parent=root, title='Choose CSV File'))
    if len(listOfFiles) > 0:
        os.chdir(os.path.dirname(listOfFiles[0]))
    root.destroy()
    return listOfFiles

# Function to compute variables based on provided inputs
def compute_variables(start_time_str, N, end_time_str=None, time_interval=None, duration=None):
    """
    Compute the necessary variables for plotting based on provided inputs.

    Args:
        start_time_str (str): Start time in string format.
        N (int): Number of data points.
        end_time_str (str, optional): End time in string format. Defaults to None.
        time_interval (float, optional): Time interval between data points. Defaults to None.
        duration (float, optional): Duration of the experiment. Defaults to None.

    Returns:
        tuple: Computed start_time, end_time, time_interval, and duration.
    """
    # Check which variables are provided
    provided_vars = {k: v for k, v in locals().items() if v is not None and k not in ['start_time_str', 'N']}
    if len(provided_vars) == 3:
        print("You have provided all three variables. Please specify which one to ignore.")
        ignore_var = input(f"Which variable to ignore ({', '.join(provided_vars.keys())})? ")
        provided_vars.pop(ignore_var)
    
    # Convert start_time and end_time from strings to datetime objects
    start_time = datetime.strptime(start_time_str, '%Y-%m-%d %H:%M:%S.%f')
    end_time = datetime.strptime(end_time_str, '%Y-%m-%d %H:%M:%S.%f') if end_time_str else np.nan
    
    # Compute missing variables based on provided ones
    if len(provided_vars) == 2:
        
        if 'end_time_str' in provided_vars and 'duration' in provided_vars:
            time_interval = provided_vars['duration'] / N
        elif 'end_time_str' in provided_vars and 'time_interval' in provided_vars:
            duration = provided_vars['time_interval'] * N
        elif 'duration' in provided_vars and 'time_interval' in provided_vars:
            end_time = start_time + timedelta(seconds=provided_vars['duration'])
    
    if len(provided_vars) == 1:
        print("provided 1")
        if 'end_time_str' in provided_vars:
            print("Provided end")
            duration = (end_time - start_time).total_seconds()
            time_interval = duration / N
        elif 'duration' in provided_vars:
            end_time = start_time + timedelta(seconds=provided_vars['duration'])
            time_interval = provided_vars['duration'] / N
        elif 'time_interval' in provided_vars:
            duration = provided_vars['time_interval'] * N
            end_time = start_time + timedelta(seconds=duration)
    print((start_time, end_time, time_interval, duration))
    return start_time, end_time, time_interval, duration
# Function to get system ticks from time
def system_ticks_from_time(start_time):
    """
    Get system ticks from the provided start time.

    Args:
        start_time (datetime): Start time.

    Returns:
        str: System ticks corresponding to the start time.
    """
    date_str, time_str = str(start_time).split()
    dm_script = 'string date = "' + date_str + '"\n string time = "' + time_str + '"\n'
    dm_script += '''
    string ParseDateFormat = "%Y-%m-%d"
    string parseTimeFormat = "%H:%M:%S"
    string ExpStartTICKS = CHRONO_GetSystemClockTicksStringFromDateAndTime(date,time,ParseDateFormat,parseTimeFormat)
    GetPersistentTagGroup().TagGroupSetTagAsString("Python_temp:Start_Ticks",ExpStartTICKS)'''
    DM.ExecuteScriptString(dm_script)
    bool, ExpStartTICKS = DM.GetPersistentTagGroup().GetTagAsString("Python_temp:Start_Ticks")
    return ExpStartTICKS

# Function to set in-situ tagging for the plot
def set_IS_tagging(plot, info):
    """
    Set in-situ tagging for the plot based on provided info.

    Args:
        plot (object): Plot object.
        info (list): List of metadata information.
    """
    info = info.copy()
    for i, item in enumerate(info[:20]): 
        try:
            if np.isnan(item): info[i] = None
        except: pass
        try:  
            info[i] = float(info[i])
        except: pass
    
    has_time_col = int(info[1])
    if has_time_col: return  # Not yet implemented
    
    start_time = info[3] + " " + str(info[5])
    try:
        end_time = info[7] + " " + str(info[9])
    except: end_time = None
    
    time_interval = info[11]
    duration = info[13]
    device_type = info[15]
    device_name = info[17]
    
    N = len(plot.GetNumArray())
    print((start_time, N, end_time, time_interval, duration))
    start_time, end_time, time_interval, duration = compute_variables(start_time, N, end_time, time_interval, duration)
    ExpStartTICKS = system_ticks_from_time(start_time)
    print(type(time_interval))
    plot.SetDimensionScale(0,time_interval)
    plot.SetDimensionUnitString(0,"s")
    plot.GetTagGroup().SetTagAsString("In-situ:Recorded:Sync:Experiment Start (System-clock tick)", ExpStartTICKS)
    plot.GetTagGroup().SetTagAsString("In-situ:Recorded:Experiment Start Time", str(start_time))
    plot.GetTagGroup().SetTagAsFloat("In-situ:Recorded:Duration (s)", duration)
    plot.GetTagGroup().SetTagAsLong("In-situ:Recorded:# Frames", N)
    plot.GetTagGroup().SetTagAsFloat("In-situ:Recorded:Sync:Linear:Frame Rate (fps)", 1 / time_interval)
    plot.GetTagGroup().SetTagAsString("In-situ:Databar Info:Device Name", device_name)
    plot.GetTagGroup().SetTagAsString("In-situ:Databar Info:Device Type", device_type)
    plot.GetTagGroup().SetTagAsString("Meta Data:Function", "InSitu profile")
    
    
# Function to extract content within parentheses
def extract_parentheses_content(text):
    """
    Extract content within parentheses from the provided text.

    Args:
        text (str): Text containing parentheses.

    Returns:
        str: Content within parentheses.
    """
    match = re.search(r'$(.*?)$', text)
    if match: return match.group(1)
    else: return None

# Main script execution
# Read the CSV file
filepath = GetSelectedFile()[0]
df = pd.read_csv(filepath)
csv_info = df.iloc[:, 0]

# Check the number of columns
num_columns = df.shape[1]

# Extract all columns except the first one
arrays = [df.iloc[:, i].values for i in range(1, num_columns)]

# Convert to numpy arrays
numpy_arrays = [np.array(arr) for arr in arrays]

# Plot the numpy arrays as in-situ plots in DigitalMicrograph compatible with the IS Player
for idx, array in enumerate(numpy_arrays, start=1):
    plot = DM.CreateImage(array)
    set_IS_tagging(plot,csv_info)
    #Set Data name and units from header row in CSV
    name = df.columns[idx].split("(")[0].strip()
    plotpath = os.path.join(os.path.dirname(filepath),"Imported_IS_Plots",name+".dm4")
    if not os.path.exists(os.path.dirname(plotpath)): os.makedirs(os.path.dirname(plotpath))
    unit = extract_parentheses_content(df.columns[idx])
    if unit is not None: plot.SetIntensityUnitString(unit) #note that DM has a problem displaying the degree symbol
    #show the plot temporarily so we can more easily add the Slice Label with a bit of DM script
    plot.ShowImage()
    #set lineplot slice name (used for databars) and axis labels and save the plot
    dm_legend_names = ('ImageDisplay disp = GetFrontImage().ImageGetImageDisplay( 0 )\n'
        'ImageSetDimensionLabel( GetFrontImage(), 0, "Time" )\n'
        'ImageSetIntensityLabel( GetFrontImage(), "'+name+'" )\n'
        'disp.ImageDisplaySetSliceLabelByID(disp.ImageDisplayGetSliceIDByIndex( 0 ),"'+name+'")\n')
    DM.ExecuteScriptString(dm_legend_names) 
    plot_doc = plot.GetOrCreateImageDocument()
    plot_doc.SaveToFile('Gatan Format',plotpath)
    DM.Sleep(0.1)
    #close the plot and then re-display it so it syncs correctly with concurrent IS data
    plot_doc.Close(False) 
    doc = DM.NewImageDocumentFromFile(plotpath)
    doc.Show()
    
print("Done")