#! /usr/bin/env python
#
# GOAL
#
#
# EXAMPLES
#    python pp_RIFT --use-ini sample_pp_config.ini --test
#
# REFERENCES
#   util_RIFT_pseudo_pipe  (for parsing/restructuring arguments)

import numpy as np
import argparse
import os
import sys
import shutil

import configparser

from RIFT.misc.dag_utils import mkdir
import RIFT.lalsimutils as lalsimutils
import lal
import lalsimulation as lalsim

# Backward compatibility
from RIFT.misc.dag_utils import which
lalapps_path2cache = which('lal_path2cache')
if lalapps_path2cache == None:
    lalapps_path2cache =  which('lalapps_path2cache')


def unsafe_config_get(config,args,verbose=False):
    """
    unsafe_config_get(config, args):
    """
    if verbose:
        print(" Retrieving ", args, end=' ') 
        print(" Found ",eval(config.get(*args)))
    return eval( config.get(*args))


#activate_env ="source /cvmfs/oasis.opensciencegrid.org/ligo/sw/pycbc/x86_64_rhel_7/virtualenv/pycbc-v1.16.13/bin/activate" # else NRSur frames were not loading
def add_frames(channel,input_frame, noise_frame,combined_frame):
    cmd = " add_frames.py " + channel + " " + input_frame + " " + noise_frame + " " + combined_frame #{} {} {} {} ".format(channel,input_frame,noise_frame,combined_frame)
    print(cmd)
    if not opts.test:
        os.system(cmd)
        
def m1m2(mc, eta):
    # Step 1: Calculate total mass (M = m1 + m2) using mc and eta
    M = mc / (eta**(3/5))  # Based on the chirp mass equation: mc = (m1*m2)^(3/5) / (m1+m2)^(1/5)
    
    # Step 2: Solve for m1 and m2 using the quadratic formula
    # We know: eta = (m1 * m2) / (m1 + m2)^2 => m1*m2 = eta * M^2
    m1m2_product = eta * M**2
    
    # Quadratic equation: x^2 - M*x + m1m2_product = 0, where x is m1
    # Discriminant for the quadratic equation:
    discriminant = M**2 - 4 * m1m2_product
    
    if discriminant < 0:
        raise ValueError("Discriminant is negative. No real solution for m1 and m2.")
    
    # Step 3: Solve for m1 and m2 (roots of the quadratic equation)
    m1 = (M + np.sqrt(discriminant)) / 2
    m2 = (M - np.sqrt(discriminant)) / 2
    
    return m1, m2

# def dmax_seglen_ladder(mc):
#     """
#     dmax, seglen ladder motivated by G2201558 but with nonoverlapping mass bins.  Not sure how we are suppose do seglen < 4, so I am making that the shortest length
#     De facto enables a conditional distance prior, based on the chirp mass.
#     Not clear that a single PP run with this weird conditional distance prior is what is intended.
#     """
#     if mc>50 and mc<100:
#         return [3500,4]
#     elif mc>20 and mc<=50:
#         return [2500,4]
#     elif mc>12.3 and mc<=20:
#         return [1500,4]
#     elif mc>7.9 and mc <=12.3:
#         return [1000,8]
#     elif mc > 5.2 and mc<=7.9:
#         return [800,16]
#     elif mc > 3.4 and mc<=5.2:
#         return [400,32]
#     elif mc > 2.2 and mc<=3.4:
#         return [400,64]
#     elif mc>1.4 and mc<=2.2:
#         return [300,128]
#     elif mc < 1.4:
#         return [150,256]

parser = argparse.ArgumentParser()
parser.add_argument("--use-ini",default=None,type=str,help="Pass ini file for parsing. Intended to reproduce lalinference_pipe functionality. Overrides most other arguments. Full path recommended")
parser.add_argument("--use-gwsignal",action='store_true')
#parser.add_argument("--internal-distance-ladder",action='store_true',help="Modify injection distance ladder as in G2201558 ")
parser.add_argument("--use-osg",action='store_true',help="Attempt OSG operation. Command-line level option, not at ini level to make portable results")
parser.add_argument("--add-extrinsic",action='store_true',help="Add extrinsic posterior.  Corresponds to --add-extrinsic --add-extrinsic-time-resampling --batch-extrinsic for pipeline")
parser.add_argument("--test",default=False,action='store_true',help="Used to test the pipeline : prints out commands, generates workflows as much as possible without high-cost steps")
opts =  parser.parse_args()


config = configparser.ConfigParser(allow_no_value=True) #SafeConfigParser deprecated from py3.2
config.read(opts.use_ini)

base_dir = os.getcwd() # relative directory, for use if absolute paths not provided

test_convergence=False
if config.has_option('pp','test_convergence'):
    test_convergence =config.getboolean('pp','test_convergence', fallback=False)

# Create, go to working directory
working_dir = config.get('pp','working_directory')
print(" Working dir ", working_dir)
mkdir(working_dir)
os.chdir(working_dir)
working_dir_full = os.getcwd()

# Define how many events are needed
n_events = unsafe_config_get(config,['pp','n_events'])


# Create injection set

mc_min = float(config.get('priors','mc_min'))
mc_max = float(config.get('priors','mc_max'))
m_min=float(config.get('priors','m_min'))
mc_range = [mc_min,mc_max]
eta_min = float(config.get('priors','eta_min'))
eta_max = float(config.get('priors','eta_max'))
eta_range = [eta_min,eta_max]
d_min = float(config.get('priors','d_min'))
d_max = float(config.get('priors','d_max'))
chi_max = float(config.get('priors','chi_max'))

m1a,m2a = m1m2(mc_range[0], eta_range[0])
m1b,m2b = m1m2(mc_range[0], eta_range[1])
m1c,m2c = m1m2(mc_range[1], eta_range[0])
m1d,m2d = m1m2(mc_range[1], eta_range[1])

# use matter?
lambda_max=0
use_matter=False
if config.has_option('priors','use_matter'):
    use_matter=True
    lambda_max = float(config.get('priors','lambda_max'))

approx_str = config.get('waveform','approx')
approx_recover_str = approx_str
if config.has_option('waveform','approx_template'):
    approx_recover_str = config.get('waveform','approx_template')
# danger: lmax only used in analysis, not in generation usually
l_max = int(config.get('waveform','lmax')  )
fmin_template = float(config.get('waveform','fmin_template'))
fmax = float(config.get('data','fmax'))
flow_dict = unsafe_config_get(config,['data','flow'])
srate_data = float(config.get('data', 'srate_data'))
seglen_data = float(config.get('data','seglen_data'))
seglen_analysis = float(config.get('data','seglen_analysis'))

if 'NRHybSur' in approx_str:       #for NRSur models
	group_str = config.get('waveform','group')
	param_str = config.get('waveform','param')

no_spin =config.getboolean('priors','no_spin', fallback=True)
aligned_spin =config.getboolean('priors','aligned_spin', fallback=False)
if aligned_spin:
    print(" === aligned-spin PP ==== ")
elif no_spin:
    print(" === zero-spin PP ==== ")
else:
    print(" === precessing PP ==== ")

volumetric_spin =unsafe_config_get(config,['priors','volumetric_spin'])
if volumetric_spin:
    print("  (volumetric spin prior)  ")

fix_sky_location=config.getboolean('priors','fix_sky_location', fallback=True)
try:
    fiducial_ra=float(config.get('priors','fiducial_ra'))
    fiducial_dec=float(config.get('priors','fiducial_dec'))
except:
    fiducial_ra = 0
    fiducial_dec = 0
fiducial_event_time=float(config.get('priors','fiducial_event_time'))


ifos = unsafe_config_get(config,['data','ifos'])

if os.path.exists(working_dir_full+"/mdc.xml.gz"):
    P_list = lalsimutils.xml_to_ChooseWaveformParams_array(working_dir_full+"/mdc.xml.gz")

else:
  P_list =[]; indx=0
  while len(P_list) < n_events:
    P = lalsimutils.ChooseWaveformParams()
    # Randomize (sky location, etc)
    P.randomize(dMax=d_max,dMin=d_min,aligned_spin_Q=aligned_spin,volumetric_spin_prior_Q=volumetric_spin,sMax=chi_max)
    P.tref = fiducial_event_time
    P.fmin=fmin_template
    P.deltaF = 1./seglen_data
    P.deltaT = 1./srate_data
    # sky location
    if fix_sky_location:
        P.theta = fiducial_dec
        P.phi = fiducial_ra
    # some redundancy
    if no_spin:
        P.s1x=P.s1y=P.s1z=0
        P.s2x=P.s2y=P.s2z=0
    elif aligned_spin:
        P.s1x = P.s1y=0
        P.s2x = P.s2y=0
    if use_matter:
        P.lambda1 = np.random.uniform(0,lambda_max)
        P.lambda2 = np.random.uniform(0,lambda_max)
    if not(opts.use_gwsignal):  # with gwsignal, we just pass the approximant on the command line, and must track it differently : XML files have no string approx fields.
        P.approx=lalsim.GetApproximantFromString(approx_str)

    # Uniform in m1 and m2: 
   # m1 = np.random.uniform(mc_range[0],mc_range[1]*2)
   # m2 = np.random.uniform(m_min,mc_range[1]*1.5)
   # m1,m2 = [np.maximum(m1,m2), np.minimum(m1,m2)]
   # P.m1 = m1*lal.MSUN_SI
   # P.m2 = m2*lal.MSUN_SI
    # ...but downselect in mchirp, eta
   # mc_val = P.extract_param('mc')/lal.MSUN_SI
   # eta_val = P.extract_param('eta')
    m1 = np.random.uniform (np.min([m1a,m1b,m1c,m1d]),np.max([m1a,m1b,m1c,m1d]))
    m2 = np.random.uniform (np.min([m2a,m2b,m2c,m2d]),np.max([m2a,m2b,m2c,m2d]))
    m1,m2 = [np.maximum(m1,m2), np.minimum(m1,m2)]
    P.m1 = m1*lal.MSUN_SI                                                                                                                                                                                  
    P.m2 = m2*lal.MSUN_SI
    mc_val , eta_val = lalsimutils.Mceta(m1,m2)
    if mc_val < mc_range[0] or mc_val > mc_range[1]:
        continue
    if eta_val < eta_range[0] or eta_val > eta_range[1]:
        continue
    # rescale distance of injection based on mc
    # if opts.internal_distance_ladder:
    #     dmax_scale, seglen_scale = dmax_seglen_ladder(mc_val)
    #     P.dist *=  dmax_scale/d_max   # direct scale
    #     P.deltaF = 1./seglen_scale  


    P_list.append(P)
    indx+=1

  lalsimutils.ChooseWaveformParams_array_to_xml(P_list,"mdc")

# Create data files and cache file
#   - probably want to create workflow to build these, rather than do it all in one go... very slow!

print(" === Writing signal files === ")

# made up durations for now
# must be compatible with duration of the noise frames
t_start = int(fiducial_event_time)-150
t_stop = int(fiducial_event_time)+150

mkdir('signal_frames')
for indx in np.arange(n_events):
    os.chdir(working_dir_full)
    target_subdir = 'signal_frames/event_{}'.format(indx)
    # Test if directory already exists
    if os.path.exists(target_subdir):
        print(" Signal frames exist for event {}, skipping ".format(indx))
        continue
    print(" Writing ", indx)
    mkdir(target_subdir)
    os.chdir(working_dir_full+"/"+target_subdir)
    # Loop over instruments, write files
    for ifo in ifos:
        if opts.use_gwsignal:
            cmd = "util_GWSignalWriteFrame.py  --inj " + working_dir_full+"/mdc.xml.gz --event {} --start {} --stop {} --instrument {} --seglen {} --approx {}".format(indx, t_start, t_stop, ifo, seglen_analysis, approx_str)
        elif 'NRHybSur' in approx_str:
            cmd = "util_ROMWriteFrame.py --inj " + working_dir_full+"/mdc.xml.gz --event {} --start {} --stop {} --instrument {} --seglen {} --group {} --param {} --lmax {}".format(indx, t_start, t_stop, ifo, seglen_analysis, group_str, param_str, l_max)
        else:
            cmd = "util_LALWriteFrame.py --inj " + working_dir_full+"/mdc.xml.gz --event {} --start {}  --stop {}  --instrument {} --seglen {} --approx {}".format(indx, t_start,t_stop,ifo, seglen_analysis, approx_str)
        print(cmd)
        if not opts.test:
            os.system(cmd)

    if not opts.test:
        cmd = "/bin/find .  -name '*.gwf' | {} > signals.cache".format(lalapps_path2cache)
        os.system(cmd)

    # Evaluate zero-noise SNR
    print(" --> FIXME: evaluate zero-noise SNR, to use as hint in place of search <-- ")

print(" === Using fiducial noise frames === ")

noise_frame_dir = config.get("make_data","fiducial_noise_frames")

print(" === Joining synthetic signals to reference noise === ")


seglen_actual = t_stop - t_start
os.chdir(working_dir_full)
mkdir('combined_frames')
for indx in np.arange(n_events):
    os.chdir(working_dir_full)
#    mkdir('analysis_event_{}'.format(indx))
    target_subdir='combined_frames/event_{}'.format(indx)
    if os.path.exists(target_subdir):
        print(" Combined frames exist for event {}, skipping ".format(indx))
        continue
    print(" Writing ", indx)
    mkdir(target_subdir)
    os.chdir(working_dir_full+"/"+target_subdir)
    for ifo in ifos:
        fname_input = working_dir_full+"/signal_frames/event_{}/{}-fake_strain-{}-{}.gwf".format(indx,ifo[0],int(t_start),seglen_actual)
        fname_noise= noise_frame_dir + "/" + ifo + ("/%08d.gwf" % indx)
#        fname_output = working_dir_full+"/analysis_event_{}/{}-combined-{}-{}.gwf".format(indx,ifo[0],int(t_start),seglen_actual)
        fname_output = working_dir_full+"/{}/{}-combined-{}-{}.gwf".format(target_subdir,ifo[0],int(t_start),seglen_actual)
        add_frames(ifo+":FAKE-STRAIN",fname_input, fname_noise,fname_output)


print(" === Building single-event workflows === ")
use_gpu = config.getboolean('make_workflow','use_gpu', fallback=False)
if use_gpu:
    print(" ====> USING GPUS <=== ")
else:
    print(" ====> NO GPUS <=== ")
try:
    use_distance_marginalization = unsafe_config_get(config,['make_workflow','use_distance_marginalization'])
except:
    use_distance_marginalization = False
if use_distance_marginalization:
    print(" ====> USING DMARG <=== ")
    print("  ...building common distance marginalization table ")
    cmd_dmarg_init = " util_InitMargTable --d-min {} --d-max {} ".format(d_min,d_max)
    os.system(cmd_dmarg_init)
try:
    ile_n_eff = int(unsafe_config_get(config, ['make_workflow', 'ile_n_eff']))
except:
    ile_n_eff = 50  # default will be 50, as in original testsa
use_mu_coord = config.getboolean('make_workflow','use_mu_coord', fallback=False)
ile_use_lnL = config.getboolean('make_workflow','ile_use_lnL', fallback=False)
if use_mu_coord:
    print(" ====> Using SM's phase-based coordinates <====")

n_events_per_worker = int(config.get('make_workflow','n_events_per_worker'))
n_copies = int(config.get('make_workflow','n_copies'))
n_iter = int(config.get('make_workflow','n_iterations'))
puff_cadence = int(config.get('make_workflow','puff_cadence'))

psd_dict = unsafe_config_get(config,["make_psd",'fiducial_psds'])

cip_fit_method = config.get('make_workflow','cip_fit_method')
cip_sampler_method = config.get('make_workflow','cip_sampler_method')
add_extrinsic=opts.add_extrinsic
use_osg=opts.use_osg #False
if not(use_osg) and 'helper_args' in config:
  if ('use_osg' in config['helper_args']):
    use_osg = unsafe_config_get(config,['helper_args','use_osg'])  # this should almost always evaluate to true
cip_explode_jobs=False
if 'cip_explode_jobs' in config['make_workflow']:
    cip_explode_jobs = config.get('make_workflow','cip_explode_jobs')

# Use guess_mc_range to get a tailored mc prior, 
#compare against the default mc-prior set by ini file and then choose each limit accordingly
P = [P_list[i] for i in range(len(P_list))]

for indx in np.arange(n_events):
    print(" ::: Event {} ::: ".format(indx))
    dir_target= working_dir_full+'/analysis_event_{}'.format(indx)
    mkdir(dir_target)
    os.chdir(dir_target)
    os.system("ln -sf ../combined_frames/event_{} .".format(indx))
    if not opts.test:
        cmd = "/bin/find -L . -name '*.gwf' | {} > local.cache".format(lalapps_path2cache)
        print(cmd)
        os.system(cmd)
    here = os.getcwd()
    mc_min_utils, mc_max_utils = lalsimutils.guess_mc_range_mdc(P_list[indx],force_mc_range=None) # this chooses the right mc-prior
    cmd = "helper_LDG_Events.py --sim-xml {}/mdc.xml.gz --event {} --event-time {}  --cache {}/local.cache --working-directory {}  ".format(working_dir_full,indx,P_list[indx].tref,here,dir_target)

    cmd += " --ile-n-eff {} ".format(ile_n_eff)
    cmd += " --fmin {} ".format(fmin_template)  # used for placement, we will be undoing some of these effects later with a 'replace'
    cmd += " --fmax {} ".format(fmax)
    # Pull out arbitrary arguments to be determined later, lieke --internal-marginalize-distance, --internal-use-aligned-phase-coordinates, etc
    if 'helper_args' in config:
        for arg in config["helper_args"]:
            arg_renamed =  arg.replace('_','-')
            val = config.get("helper_args",arg)
            if val == False or val=="False":
                continue;
            elif val == None or val=='True':
                cmd += " --{} ".format(arg_renamed)
            else:
                cmd += " --{} {} ".format(arg_renamed,val)
            
    cmd += " --fmin-template {} ".format(fmin_template)
    cmd += " --force-mc-range [{},{}] --force-eta-range [{},{}] ".format(np.maximum(mc_min,mc_min_utils),np.minimum(mc_max,mc_max_utils),eta_min,eta_max) 
    if use_mu_coord:
        cmd += " --internal-use-aligned-phase-coordinates " 
    if use_osg:
        cmd += " --use-osg "
    if test_convergence:
        cmd += " --test-convergence "  # removes 'always succeed' as an option (by the helper)
    # add in PSDs.  Allow relative names for PSDs relative to ini file
    for psd in psd_dict:
        psd_name = psd_dict[psd]
        if not( '/' in psd_name):
            if use_osg:
                psd_name = psd_name
                # copy/link PSD files so they are correctly transferred by the downstream tools; yes it is a little silly
                os.system(" cp {}/{} {} ".format(base_dir, psd_name, working_dir_full) )
            else:
                psd_name = base_dir +"/"+psd_name
        cmd += " --psd-file {}={} ".format(psd,psd_name)
    # add in run settings
    cmd += "  --fake-data --propose-initial-grid --propose-fit-strategy --propose-ile-convergence-options --no-propose-limits "
    if no_spin:
        cmd += " --assume-nospin "
    elif not(aligned_spin):
        cmd += " --assume-precessing "
    if volumetric_spin:
        cmd += " --assume-volumetric-spin "
    if use_matter:
        cmd += " --assume-matter "
    if ile_use_lnL:
        cmd += " --internal-ile-use-lnL "

    print(cmd)
    if not opts.test:
        os.system(cmd)

    if not opts.test:
    ## Override /augment features of helper (e.g, enforce our prior on mc, eta)
        # ILE
        instructions_ile = np.loadtxt("helper_ile_args.txt", dtype=str)  # should be one line
        line = ' '.join(instructions_ile)
        if opts.use_gwsignal:
            line += " --use-gwsignal "
        if not (use_gpu):
            line+= " --vectorized --force-xpy "  # make sure to use vectorized code
            # line = line.replace("--gpu","")  # make SURE this option is not set
        if (use_gpu):
            line+= " --vectorized --force-xpy  --force-gpu-only "
            line = line.replace("--gpu","")  # make SURE this option is not set
        if fix_sky_location:
            line = line.replace("--declination-cosine-sampler", "")   # fix, so known sky location is interpreted correctly
            line += " --right-ascension {} --declination {} ".format(fiducial_ra, fiducial_dec)
        for ifo in ifos:
            line =line.replace("--fmin-ifo {}={} ".format(ifo, fmin_template),"--fmin-ifo {}={} ".format(ifo, flow_dict[ifo]))
        line += " --l-max " + str(l_max) 
#        if not opts.internal_distance_ladder:
        line += " --d-max " + str(d_max)
        t_start_analysis = fiducial_event_time - (seglen_analysis-2)
#        else:
#            dmax_scale, seglen_scale = dmax_seglen_ladder(P_list[indx].extract_param('mc')/lal.MSUN_SI)
#            line += " --d-max " + str(dmax_scale)    
#            t_start_analysis = fiducial_event_time - (seglen_scale-2)
        t_end_analysis = fiducial_event_time +2
        line += " --d-min " + str(d_min)
        line += " --inv-spec-trunc-time 0 --data-start-time {} --data-end-time {} --window-shape {}".format(t_start_analysis, t_end_analysis, 0.4/seglen_analysis)
    
        if 'ile_sampler_method' in config['make_workflow']:
            ile_sampler = config.get('make_workflow','ile_sampler_method')
            line += " --sampler-method {} ".format(ile_sampler)

        if opts.use_gwsignal:
            line += " --approx " + approx_recover_str
        elif not 'NR' in approx_recover_str:
            line += " --approx " + approx_recover_str
        elif "NRHybSur3dq8Tidal" in approx_recover_str:       # added NRHybSur3dq8Tidal to the list
            line += " --rom-group surrogate_downloads/  --rom-param NRHybSur3dq8Tidal  "
        elif "NRSur7dq2" in approx_recover_str:
            line += " --rom-group my_surrogates/nr_surrogates/ --rom-param NRSur7dq2.h5  "
        elif "NRSur7dq4" in approx_recover_str:
            line += " --approx " + approx_recover_str
        elif 'NRHybSur' in approx_recover_str:
            line += " --rom-group my_surrogates/nr_surrogates/ --rom-param NRHybSur3dq8.h5 "
        elif ("SEOBNR" in approx_recover_str) or 'NRTidal' in approx_recover_str: 
            line += " --approx " + approx_recover_str
        else:
            print(" Unknown approx ", approx_recover_str)
            sys.exit(1)
        if use_distance_marginalization:
            # only implemented for a very specific code path
            line += " --force-xpy --gpu --distance-marginalization --distance-marginalization-lookup-table "
            if not use_osg:
                line += " {}/{} ".format(working_dir_full,"distance_marginalization_lookup.npz")
            else: 
                line += " ./distance_marginalization_lookup.npz "
                # add to transfer file list
                os.system('echo  {}/distance_marginalization_lookup.npz >> helper_transfer_files.txt '.format(working_dir_full))
        with open('args_ile.txt','w') as f:
            f.write(line)

        # test
        os.system("cp helper_test_args.txt args_test.txt")

        # CIP
        with open("helper_cip_arg_list.txt",'r') as f:
            raw_lines = f.readlines()
        instructions_cip = [x.rstrip().split(' ') for x in raw_lines]#np.loadtxt("helper_cip_arg_list.txt", dtype=str)
        n_iterations =0
        lines  = []
        for indx_instr in np.arange(len(instructions_cip)):
            if instructions_cip[indx_instr][0] == 'Z':
                n_iterations += 1
            else:
                n_iterations += int(instructions_cip[indx_instr][0])
            line = ' ' .join(instructions_cip[indx_instr])
            # Add CIP range
            line += " --mc-range [{},{}] --eta-range [{},{}] --chi-max {} ".format(np.maximum(mc_min,mc_min_utils),np.minimum(mc_max,mc_max_utils),eta_min,eta_max,chi_max)
            if use_matter:
                line += " --lambda-max {} ".format(lambda_max)
            if not(cip_explode_jobs is False):
                n_workers = int(cip_explode_jobs)
            else:
                n_workers = 1
            n_sample_target = 10000 # fixed right now
            n_eff_cip_here = int(n_sample_target/n_workers)
            internal_cip_cap_neff = 500 # fixed
            if indx_instr < len(instructions_cip)-1: # on all but 
                n_eff_cip_here = np.min([internal_cip_cap_neff, n_eff_cip_here]) # n_eff: make sure to do *less* than the limit. Lowering this saves immensely on internal/exploration runtime
            #n_sample_min_per_worker = int(n_eff_cip_here/100)+2   # for fail-unless-n-eff option
            line +=" --n-output-samples {} --n-eff {} --n-max 10000000   --downselect-parameter m2 --downselect-parameter-range [1,1000] ".format(int(n_sample_target/n_workers), n_eff_cip_here) 
            if not(cip_fit_method is None):
                line = line.replace('--fit-method gp', '--fit-method ' + cip_fit_method)
            if (cip_sampler_method=="GMM"):
                line += " --sampler-method GMM --internal-correlate-parameters 'mc,delta_mc,s1z,s2z' "
            elif (cip_sampler_method!="GMM"):
                line += " --sampler-method "+cip_sampler_method
            #line +=" --n-output-samples {}  --n-eff {} ".format(int(n_sample_target/n_workers), n_eff_cip_here)
#            if aligned_spin:
#                line += " --aligned-prior alignedspin-zprior "
            line += "\n"
            lines.append(line)

        with open("args_cip_list.txt",'w') as f: 
            for line in lines:
                f.write(line)

        # puff
        with open("helper_puff_args.txt",'r') as f:
            raw_lines = f.readlines()
        line = ' '.join(raw_lines)
        puff_max_it =4
        try:
            with open("helper_puff_max_it.txt",'r') as f:
                puff_max_it = int(f.readline())
        except:
            print(" No puff file ")
        with open("args_puff.txt",'w') as f:
            puff_args = line + " --downselect-parameter chi1 --downselect-parameter-range [0,{}] --downselect-parameter chi2 --downselect-parameter-range [0,{}] --mc-range [{},{}] ".format(chi_max,chi_max,mc_min,mc_max)
            f.write("X " + puff_args)


    # Build workflow
    cmd = "create_event_parameter_pipeline_BasicIteration --input-grid proposed-grid.xml.gz --ile-n-events-to-analyze {} --ile-exe  `which integrate_likelihood_extrinsic_batchmode`  --ile-args {}/args_ile.txt --request-memory-ILE 4096 ".format(n_events_per_worker,dir_target)
    cmd+= "--cip-args-list {}/args_cip_list.txt  --test-args {}/args_test.txt --request-memory-CIP 30000  --n-samples-per-job 1000 --working-directory `pwd` --n-iterations {} --n-copies {} ".format(dir_target,dir_target,n_iter, n_copies)
    cmd+= " --puff-exe `which util_ParameterPuffball.py` --puff-max-it 10 --puff-cadence {} --puff-args {}/args_puff.txt".format(puff_cadence,dir_target)

    # Use GPUs if we are not
    if use_gpu:
      cmd += " --request-gpu-ILE "

    if add_extrinsic:
          cmd += " --last-iteration-extrinsic --last-iteration-extrinsic-nsamples 8000 --last-iteration-extrinsic-batched-convert --last-iteration-extrinsic-time-resampling "
    if cip_explode_jobs:
        cmd+= " --cip-explode-jobs  " + str(cip_explode_jobs) + " --cip-explode-jobs-dag "
        if cip_fit_method and not(cip_fit_method == 'gp'):
            cmd += " --cip-explode-jobs-flat " 
    if use_osg:
        print(" NOT VALIDATED/IMPLEMENTED for PP use yet")
        cmd += " --use-osg --use-singularity --cache-file local.cache --condor-nogrid-nonworker --frames-dir {}/frames_dir ".format(here)   # run on the OSG, make sure to get frames (rather than try to transfer them). --frames_dir bring in ile_pre.sh for fake data 
        cmd+= " --transfer-file-list  "+here +"/helper_transfer_files.txt"
        #os.system("echo {}/frames_dir >> helper_transfer_files.txt".format(here))  # can't use symbolic links alas
        os.system("echo {}/local.cache >> helper_transfer_files.txt".format(here)) # adds local.cache to transfer files
        try:
            os.system("mkdir frames_dir")
        except:
            continue
        os.system("cp -r {}/event_{}/* frames_dir/".format(here,indx))
    else:
        cmd += " --cache-file {}/local.cache   ".format(here)

    print(cmd)
    if not opts.test:
        os.system(cmd)


print(" === Building master workflow === ")

os.chdir(working_dir_full)
cmd = 'util_ConsolidateDAGsUnderMaster.sh analysis_event*'
print(cmd)
if not opts.test:
    os.system(cmd)
