#!/usr/bin/env python

import os
import saga
import sys
import pprint
import radical.utils       as ru
import radical.pilot       as rp
import radical.pilot.utils as rpu


_DEFAULT_DBURL = 'mongodb://user:password@ec2-184-72-89-141.compute-1.amazonaws.com:27017/radicalpilot/'

if  'RADICAL_PILOT_DBURL' in os.environ :
    _DEFAULT_DBURL = os.environ['RADICAL_PILOT_DBURL']

_DEFAULT_DBURL = ru.Url (_DEFAULT_DBURL)
if  not _DEFAULT_DBURL.path or '/' == _DEFAULT_DBURL.path :
    _DEFAULT_DBURL.path = 'radicalpilot'

_DEFAULT_DBURL = str(_DEFAULT_DBURL)

# for graphing events, we assign numerical pseudo values to each event.  
# Note: make sure that those are translated back into event tags via 
# 'set [xy]tics'
# 
#   set xtics ("lbl1" 1, "lbl2" 2, "lbl3" 3, "lbl4" 4)
#
_EVENT_ENCODING = {
        'session': {
            'created'                  :  1
        }, 
        'pmgr': {
        }, 
        'pilot' : {
            rp.PENDING_LAUNCH          :  1,
            rp.LAUNCHING               :  2,
            rp.PENDING_ACTIVE          :  3,
            rp.ACTIVE                  :  4,
            rp.DONE                    :  5,
            rp.CANCELED                :  6,
            rp.FAILED                  :  7
        },
        'umgr': {
        }, 
        'unit': {
            rp.NEW                          :  1,
            rp.SCHEDULING                   :  2,
            rp.PENDING_INPUT_STAGING        :  3,
            rp.STAGING_INPUT                :  4,
            rp.PENDING_AGENT_INPUT_STAGING  :  5,
            rp.AGENT_STAGING_INPUT          :  6,
            rp.PENDING_EXECUTION            :  7,
            rp.ALLOCATING                   :  8,
            rp.EXECUTING                    :  9,
            rp.PENDING_OUTPUT_STAGING       : 10,
            rp.STAGING_OUTPUT               : 11,
            rp.PENDING_AGENT_OUTPUT_STAGING : 12,
            rp.AGENT_STAGING_OUTPUT         : 13,
            rp.DONE                         : 14,
            rp.CANCELED                     : 15,
            rp.FAILED                       : 16
        }
    }

# # common states
# DONE                        
# CANCELED                    
# FAILED                      
# 
# # pilot states
# PENDING_LAUNCH              
# LAUNCHING                   
# PENDING_ACTIVE              
# ACTIVE                      
# 
# # ComputeUnit States
# NEW                         
# PENDING_INPUT_TRANSFER      
# TRANSFERRING_INPUT          
# 
# PENDING_EXECUTION           
# SCHEDULING                  
# ALLOCATING
# EXECUTING                   
# 
# PENDING_OUTPUT_TRANSFER     
# TRANSFERRING_OUTPUT         

# ------------------------------------------------------------------------------
#
def usage (msg=None, noexit=False) :

    if  msg :
        print "\n      Error: %s" % msg

    print """
      usage     : %s -m <mode>[,<mode>] [-d <dburl>] [-s <session>[,<session>]]

      examples  : %s -m stat -d mongodb://user:password@localhost/radicalpilot -s 536afe101d41c83696ea0135
                  %s -m list -d mongodb://user:password@localhost/radicalpilot
                  %s -m prof -s 536afe101d41c83696ea0135,536afe101d41c83696ea0136

      mode(s)   :

        help    : show this message
        list    : show  a  list   of sessions in the database
        tree    : show  a  tree   of session objects
        dump    : show  a  tree   of session objects, with full details
        sort    : show  a  list   of session objects, sorted by type
        hist    : show timeline   of session history
        stat    : show statistics of session history
        prof    : collect agent profiling file of session
        plot    : save gnuplot representing session history


      arguments :
        
        -d      : database URL
        -s      : session id(s)
        -c      : cachedir where <sid>.json caches are kept
        -p      : profile directory where <sid-pid>.prof files are kept
        -f      : filter for listings
        -t      : terminal type for plotting (pdf and/or png, default is both)


      filters   :

        The following filters are supported at the moment:

        'pilots = n' : only list sessions with exactly    'n' pilots
        'pilots > n' : only list sessions with more  than 'n' pilots
        'pilots < n' : only list sessions with fewer than 'n' pilots

      Notes     :

        The default mode is 'list'.
        The default MongoDB is '%s'
        Multiple modes can be specified as <mode>,<mode>,...
        For mode 'prof', multiple session ids can be specified as <session>,<session>,...

""" % (sys.argv[0], sys.argv[0], sys.argv[0], sys.argv[0], _DEFAULT_DBURL)

    if  msg :
        sys.exit (1)

    if  not noexit :
        sys.exit (0)


# ------------------------------------------------------------------------------
#
def dump_session (db, dbname, session, cachedir) :

    print "session : %s" % session
    handle_session (db, 'dump', dbname, session, cachedir, None)


# ------------------------------------------------------------------------------
#
def tree_session (db, dbname, session, cachedir) :

    handle_session (db, 'tree', dbname, session, cachedir, None)


# ------------------------------------------------------------------------------
#
def list_sessions (db, cachedir, filters) :

    sids = rpu.get_session_ids (db)

    if  filters :

        if  filters.startswith ('pilots') :

            filters = filters.replace (' ', '')
            op      =     filters[6 ]
            n       = int(filters[7:])
            f_sids  = list()

            for sid in sids :

                docs     = rpu.get_session_docs (db, sid, cachedir=cachedir)
                n_pilots = 0

                for doc in docs['pilot'] :
                    n_pilots   += 1


                if  op == '>' and n_pilots >  n : f_sids.append (sid)
                if  op == '=' and n_pilots == n : f_sids.append (sid)
                if  op == '<' and n_pilots <  n : f_sids.append (sid)

                print "%s: %3s (%s)" % (sid, n_pilots, len(f_sids))

            sids = f_sids


    if  not sids :
        print 'no matching sessions recorded in database at %s' % url

    else :
        print "Session IDs:"
        for sid in sids :
            print "  %s" % sid


# ------------------------------------------------------------------------------
def sort_session (db, session, cachedir) :

    docs = rpu.get_session_docs (db, session, cachedir=cachedir)

    print "pilot managers :" 
    for doc in docs['pmgr'] :
        print "  %s" %  doc['_id']

    print "pilots :" 
    for doc in docs['pilot'] :
        print "  %s" %  doc['_id']

    print "unit manager"
    for doc in docs['umgr'] :
        print "  %s" %  doc['_id']

    print "units"
    for doc in docs['unit'] :
        print "  %s" %  doc['_id']


# ------------------------------------------------------------------------------
def hist_session (db, session, cachedir) :

    events    = rpu.get_session_events   (db, session, cachedir=cachedir)
    slothists = rpu.get_session_slothist (db, session, cachedir=cachedir)

    if  not events :
        print "no events found for session %s" % session
        sys.exit (-1)

    if  not slothists :
        print "no slot configuration for session %s" % session
        print "Please enable benchmarking: export RADICAL_PILOT_PROFILE="



    start = events[0][4]

    # ascii output of time sorted events and slot history

    print "session : %s" % session
    print "start   : %s" % str(ru.time_stamp (start))

    for e in events :
        seconds = ru.time_diff (start, e[4])
        print "          %08.2fs : %10s : %15s : %20s (%s)" % (seconds, e[1], e[2], e[5], e[0])


    if  slothists :
        for pilot_info in slothists :
            print "pilot   : %s" % pilot_info['pilot_id']
            for slothist in pilot_info['slothist'] :
                seconds = ru.time_diff (start, slothist['timestamp'])
                print "          %08.2fs : %s" % (seconds, str(slothist['slotstate']))


# ------------------------------------------------------------------------------
def get_stats (docs, events, slothist) :

    n_units               = 0
    n_pilots              = 0
    unit_state_durations  = dict()
    pilot_stats           = dict()
    units                 = dict()

    pilot_stats['pilots'] = dict()

    for cu in docs['unit'] :
        units[str(cu['_id'])] = cu

    for doc in docs['pilot'] :

        n_pilots   += 1
        pid         = str(doc['_id'])
        pilot_info  = dict()

        pilot_info['resource']     = doc['description']['resource']
        pilot_info['cores']        = doc['description']['cores']
        pilot_info['n_units']      = 0
        pilot_info['unit_states']  = dict()
        pilot_info['pilot_states'] = list()

        # we assume that states are time-ordered
        state = doc['statehistory'][0]['state']
        start = doc['statehistory'][0]['timestamp']
        for t in doc['statehistory'][1:] :
            ts = t['timestamp']
            s  = t['state']
            pilot_info['pilot_states'].append ({'state'    : state, 
                                                'duration' : ru.time_diff(start, ts)})
            state = s
            start = ts

        for cu in doc['unit_ids'] :
            uid                    = str(cu)
            n_units               += 1
            pilot_info['n_units'] += 1

            if  not uid in units :
                print 'unknonwn unit %s' % uid
                sys.exit ()

            unit_doc = units[uid]
            state    = unit_doc['statehistory'][0]['state']
            start    = unit_doc['statehistory'][0]['timestamp']

            for t in unit_doc['statehistory'][1:] :
                ts = t['timestamp']
                s  = t['state']

                if not state in pilot_info['unit_states'] :
                    pilot_info['unit_states'][state]        = dict()
                    pilot_info['unit_states'][state]['dur'] = list()

                pilot_info['unit_states'][state]['dur'].append (ru.time_diff(start, ts))
                state = s
                start = ts

        started  = doc.get('started',  None)
        finished = doc.get('finished', None)
        cores    = doc.get('cores',    None)

        if not started or not finished or not cores:
            pilot_runtime = 1
        else:
            pilot_runtime = (finished - started) * cores

        pilot_busy = 0.0
        if pid in slothist:
            for slot in slothist[pid]['slot_infos'] :
                for slot_used in slothist[pid]['slot_infos'][slot] :
                    pilot_busy += slot_used[1] - slot_used[0]

        pilot_info['started']     = doc['started']
        pilot_info['finished']    = doc['finished']
        pilot_info['cpu_burned']  = pilot_runtime
        pilot_info['cpu_used']    = pilot_busy
        pilot_info['utilization'] = pilot_busy * 100 / pilot_runtime

        for s in pilot_info['unit_states'] :
            import numpy
            array = numpy.array (pilot_info['unit_states'][s]['dur'])
            pilot_info['unit_states'][s]['num' ] = len        (array)
            pilot_info['unit_states'][s]['mean'] = numpy.mean (array)
            pilot_info['unit_states'][s]['std' ] = numpy.std  (array)
            pilot_info['unit_states'][s]['dur' ] = list()


        pilot_stats['pilots'][pid] = pilot_info

    pilot_stats['n_pilots'] = n_pilots

    return pilot_stats

    
# ------------------------------------------------------------------------------
def stat_session (db, session, cachedir) :

    docs       = rpu.get_session_docs     (db, session, cachedir=cachedir)
    events     = rpu.get_session_events   (db, session, cachedir=cachedir)
    slothists  = rpu.get_session_slothist (db, session, cachedir=cachedir)

    stats      = get_stats (docs, events, slothists)

  # session_df, pilot_df, unit_df = rpu.get_session_frames (db, session, cachedir)
  #
  # print unit_df
  # sys.exit ()

    t_x_start  = None   # first unit started executing
    t_y_start  = None   # last  unit started executing
    t_x_stop   = None   # last unit finished executing
    unit_cores = dict() # sum of core counts from all units per pilot

    for unit in docs['unit'] :
        if not t_x_start : t_x_start =     unit['started']
        else             : t_x_start = min(unit['started'],  t_x_start)
        if not t_y_start : t_y_start =     unit['started']
        else             : t_y_start = max(unit['started'],  t_y_start)
        if not t_x_stop  : t_x_stop  =     unit['finished']
        else             : t_x_stop  = max(unit['finished'], t_x_stop )

        pid = unit.get ('pilot', 'None')
        if not pid in unit_cores : unit_cores[pid]  = unit['description'].get ('cores', 1)
        else                     : unit_cores[pid] += unit['description'].get ('cores', 1)

    t_0  = docs['session']['created']  # session started (absolute time)
    t_h  = t_x_start - t_0             # session start 'til first execution
    t_x  = t_x_stop  - t_x_start       # first start 'til last stop  of unit execution
    t_xy = t_y_start - t_x_start       # first start 'til last start of unit execution

    session_name = docs['session']['name']

    print "Session Statistics"
    print "------------------"
    print ""
    print "  session: %s"    % session_name
    print "  pilots : %s"    % stats['n_pilots']
    print "  t_0    : %12.1fs" % t_0
    print "  t_h    : %12.1fs" % t_h
    print "  t_x    : %12.1fs" % t_x
    print "  t_xy   : %12.1fs" % t_xy

    for pid in stats['pilots'] :

        pilot = stats['pilots'][pid]

        print "  pilot [%s] [%s]"       % (pid, pilot['resource'])
        print "      cores       : %6d" % pilot['cores']
        print "      units       : %6d" % pilot['n_units']
        print "      units*cores : %6s" % unit_cores.get (pid, '-')

        if unit_cores.get (pid, 0) <= pilot['cores'] :
            urate = pilot['n_units'] / t_xy
            print "      unit-rate   : %8.1f/sec" % urate

        print "      utilization : %8.1f%%" % pilot['utilization']
        print "      pilot states:"
        for ps in pilot['pilot_states'] :
            print "        state %-18s : %10.1fs" % (ps['state'], ps['duration'])
        print "      unit states :"
        for us  in pilot['unit_states'] :
            data = pilot['unit_states'][us]
            print "        state %-18s : %10.1fs  +/- %8.1fs" % (us, data['mean'], data['std'])


# ------------------------------------------------------------------------------
def prof_session(db, sessions, cachedir, profdir):

    print ""
    print "Collecting agent profiling data"
    print "-------------------------------"
    print ""

    for session in sessions.split(','):

        json_docs = rpu.get_session_docs(db, session, cachedir)

        pilots = json_docs['pilot']
        num_pilots = len(pilots)
        print "Session: %s" % session
        print "Number of pilots in session: %d" % num_pilots

        for pilot in pilots:

            print "Processing pilot '%s'" % pilot['_id']

            sandbox = pilot['sandbox']
            pilot_id = os.path.basename(os.path.dirname(sandbox))

            src = sandbox + 'agent.prof'

            if profdir.startswith('/'):
                dst = 'file://localhost/%s/%s.prof' % (profdir, pilot_id)
            else:
                dst = 'file://localhost/%s/%s/%s.prof' % (os.getcwd(), profdir, pilot_id)

            print "Copying '%s' to '%s'." % (src, dst)
            prof_file = saga.filesystem.File(src)
            prof_file.copy(dst, flags=saga.filesystem.CREATE_PARENTS)
            prof_file.close()

        print ""


# ------------------------------------------------------------------------------
def plot_session (db, session, cachedir, term) :
    """
    plot results :P
    """

    docs      = rpu.get_session_docs     (db, session, cachedir=cachedir)
    events    = rpu.get_session_events   (db, session, cachedir=cachedir)
    slothists = rpu.get_session_slothist (db, session, cachedir=cachedir)

    stats     = get_stats (docs, events, slothists)

    session_name = docs['session']['name']


    if  not events :
        print "no events found for session %s" % session
        sys.exit (-1)

    if  not slothists :
        print "no slot configuration for session %s" % session
        print "Please enable benchmarking: export RADICAL_PILOT_PROFILE="

    start      = events[0][4]
    pids       = list()
    maxtime    = 0.0
    maxslots   = 0
    nodesize   = 0
    slots      = list()
    nunits     = list()
    hosts      = list()
    delete_me  = list()
    maxqueue   = 0

    # some data cleanup
    for doc in docs['pilot'] :
        if  not doc['nodes'] :
            doc['nodes'] = list()
        if  not doc['cores_per_node'] :
            doc['cores_per_node'] = 1

    # the plots look nicer if the largest pilots are plotted first, and smaller
    # ones are overlayed.  Thus we sort the pilot docs by by size (reversed).
    # Pilot's of same sizes are ordered as-is.
    pilot_sizes = list()
    for doc in docs['pilot'] :
        cores = len(doc['nodes']) * int(doc['cores_per_node'])
        pilot_sizes.append (cores)


    pilot_docs = list()
    for pilot_size in sorted (pilot_sizes, reverse=True) :
        for doc in docs['pilot'] :
            cores = len(doc['nodes']) * int(doc['cores_per_node'])
            if  cores == pilot_size :
                if  doc not in pilot_docs :
                    pilot_docs.append (doc)

    for pilot in pilot_docs :

        pid = str(pilot['_id'])
        pids.append  (pid)
        hosts.append (ru.Url (pilot['sandbox']).host.split('.')[0])

        slotnum  = len(pilot['nodes'] * pilot['cores_per_node'])
        maxslots = max(slotnum,maxslots)
        slots.append (slotnum)
        nunits.append (stats['pilots'][pid]['n_units'])

        with open ("/tmp/rp.%s.pilot.states.%s.dat" % (session, pid), "w") as dat :
            for event in pilot['statehistory'] :
                etag    = _EVENT_ENCODING['pilot'].get (event['state'], 0)
                seconds = ru.time_diff (start, event['timestamp'])
                maxtime = max (maxtime, seconds)
                dat.write (" %10.2f  %-25s\n" % (seconds, etag))
            dat.write ("\n")
            delete_me.append (dat.name)
            
        with open ("/tmp/rp.%s.pilot.callbacks.%s.dat" % (session, pid), "w") as dat :
            if  'callbackhistory' in pilot :
                for event in pilot['callbackhistory'] :
                    etag    = _EVENT_ENCODING['pilot'].get (event['state'], 0)
                    seconds = ru.time_diff (start, event['timestamp'])
                    maxtime = max (maxtime, seconds)
                    dat.write ("%10.2f  %-25s\n" % (seconds, etag))
                dat.write ("\n")
            else :
                print 'no pilot callbacks'
            delete_me.append (dat.name)

            
        with open ("/tmp/rp.%s.unit.states.%s.dat" % (session, pid), "w") as dat :

            for unit_id in pilot['unit_ids'] :

                for unit in docs['unit'] :
                    if  unit_id == str(unit['_id']) :
                        for event in unit['statehistory'] :
                            etag    = _EVENT_ENCODING['unit'].get (event['state'], 0)
                            seconds = ru.time_diff (start, event['timestamp'])
                            maxtime = max (maxtime, seconds)
                            dat.write (" %10.2f  %-25s\n" % (seconds, etag))
                        dat.write ("\n")
            delete_me.append (dat.name)
            
        with open ("/tmp/rp.%s.unit.callbacks.%s.dat" % (session, pid), "w") as dat :
            for unit_id in pilot['unit_ids'] :
                for unit in docs['unit'] :
                    if  unit_id == str(unit['_id']) :
                        if  'callbackhistory' in unit :
                            for event in unit['callbackhistory'] :
                                etag    = _EVENT_ENCODING['unit'].get (event['state'], 0)
                                seconds = ru.time_diff (start, event['timestamp'])
                                maxtime = max (maxtime, seconds)
                                dat.write (" %10.2f  %-25s\n" % (seconds, etag))
                            dat.write ("\n")
            delete_me.append (dat.name)

            
        with open ("/tmp/rp.%s.pilot.queue.%s.dat" % (session, pid), "w") as dat :

            queue_size = 0
            queued     = list()
            dequeued   = list()

            dat.write ("%10.2f  %6d\n" % (0, queue_size))


            for event in events :
                if  event[0] == 'state' and \
                    event[1] == 'unit'  and \
                    event[3] ==  pid    :
                    uid = event[2]

                    if  _EVENT_ENCODING['unit'][event[5]] > _EVENT_ENCODING['unit'][rp.NEW] :
                        if  not uid in queued :
                            queued.append (uid)
                            seconds     = ru.time_diff (start, event[4])
                            queue_size += 1
                            maxqueue    = max (maxqueue, queue_size)
                            dat.write ("%10.2f  %6d\n" % (seconds, queue_size))

                    if  _EVENT_ENCODING['unit'][event[5]] > _EVENT_ENCODING['unit'][rp.EXECUTING] :
                        if  not uid in dequeued :
                            dequeued.append (uid)
                            seconds     = ru.time_diff (start, event[4])
                            queue_size -= 1
                            dat.write ("%10.2f  %6d\n" % (seconds, queue_size))


        with open ("/tmp/rp.%s.pilot.slots.%s.dat" % (session, pid), "w") as dat :

            if pid in slothists:
                slothist = slothists[pid]

                slot_idx = 0
                for slot in slothist['slots'] :

                    used = False

                    for entry in slothist['slot_infos'][slot] :

                        busy_start = ru.time_diff (start, entry[0])
                        busy_stop  = ru.time_diff (start, entry[1])

                        dat.write ("%10.2f  %6d\n" % (busy_start, slot_idx))
                        dat.write ("%10.2f  %6d\n" % (busy_stop,  slot_idx))
                        dat.write ("\n")

                    dat.write ("\n")
                    slot_idx += 1

                # sometimes, pilots get more cores than requested (if nodes are
                # larger etc) -- then we plot them if they are used.
                maxslots = max(maxslots, slot_idx)

            delete_me.append (dat.name)

    pilotnum = len(pids)

    timetics = 10
    for i in range(1,10) :
        if  maxtime  > 1*(10**i) :
            timetics = 1*(10**(i-1))
        if  maxtime  > 2*(10**i) :
            timetics = 2*(10**(i-1))
        if  maxtime  > 5*(10**i) :
            timetics = 5*(10**(i-1))

  # mtimetics = 10

    plotfile = "%s/radicalpilot-stats.plot" % os.path.dirname (__file__)
    plotname = os.environ.get ('RP_PLOTNAME', None)

    # if maxslots and maxqueue differ by max 25% then we use the same scale.  If
    # maxslots is larger we also use the same scale.
    max_scale  = max(maxslots, maxqueue)
    min_scale  = min(maxslots, maxqueue)
    mean_scale = (maxslots+maxqueue)/2
    scale_25   = 0.25 * mean_scale

    if  maxslots > maxqueue :
        slotsscale = maxslots+(nodesize/2)
        queuescale = maxslots+(nodesize/2)

    elif (mean_scale + scale_25) > max_scale and \
        (mean_scale - scale_25) < min_scale :
        slotsscale = maxslots+(nodesize/2)
        queuescale = maxslots+(nodesize/2)

    else :
        slotsscale = maxslots+(nodesize/2)
        queuescale = maxqueue+(nodesize/2)

    slotsscale = int(max(slotsscale, maxslots*1.1))
    queuescale = int(max(queuescale, maxqueue*1.1))

    terms = " ".join (term.split (','))

    cmd  = "gnuplot -e  maxtime=%d "        % int(maxtime+10)
    cmd +=        " -e  timetics=%d "       % timetics
  # cmd +=        " -e  mtimetics=%d "      % mtimetics
    cmd +=        " -e  maxslots=%d "       % maxslots
    cmd +=        " -e  maxqueue=%d "       % maxqueue
    cmd +=        " -e  slotsscale=%d "     % slotsscale
    cmd +=        " -e  queuescale=%d "     % queuescale
    cmd +=        " -e  'slotnum_list=\""
    for idx,pid in enumerate(pids) :
        cmd +=    "%d "     % (slots[idx])
    cmd +=        "\"'"
    cmd +=        " -e  'unitnum_list=\""
    for idx,pid in enumerate(pids) :
        cmd +=    "%d "     % (nunits[idx])
    cmd +=        "\"'"
    cmd +=        " -e  nodesize=%d "       % nodesize
    cmd +=        " -e 'session=\"%s\"' "   % session
    cmd +=        " -e 'sname=\"%s\"' "     % session_name
    cmd +=        " -e 'terms=\"%s\"' "     % terms
    cmd +=        " -e 'plottitle=\"RADICAL-Pilot\\n============="
    if plotname :
        cmd +=    "\\n[%s]" % plotname
    cmd +=        "\\nPilot and Unit Event Traces\\nSession %s\"' " % session_name
    cmd +=        " -e  pilot_num=%d "       % len(pids)
    cmd +=        " -e  'pilot_name_list=\""
    for idx,pid in enumerate(pids) :
        cmd +=    "%s " % hosts[idx]
    cmd +=        "\"'"
    cmd +=        " -e  'pilot_id_list=\""
    for idx,pid in enumerate(pids) :
        cmd +=    "%s "       % (pids[idx])
    cmd +=        "\"'"
    cmd +=        "     %s "                        % plotfile

  # print cmd
    print "\nplotting..."
    os.system (cmd) 

    if  plotname :
        os.system ("mv %s.png %s.png" % (session, plotname))
        os.system ("mv %s.pdf %s.pdf" % (session, plotname))

    for filename in delete_me :
      # print "removing %s" % filename
        try :
          # os.remove (filename)
            pass
        except Exception as e :
            print "Error removing %s: %s" % (filename, str(e))

    DO_SPLOTS = False
    if  not DO_SPLOTS :
        return


    # --------------------------------------------------------------------------
    #
    # also do splots
    #
    entity_states = dict ()

    BEGIN  = ">"
    END    = "<"
    ONCE   = "!"
    COLORS = {rp.NEW                     : ' 1',
              rp.SCHEDULING              : ' 2',
              rp.PENDING_INPUT_STAGING   : ' 3',
              rp.STAGING_INPUT           : ' 4',
              rp.PENDING_EXECUTION       : ' 5',
              rp.ALLOCATING              : ' 6',
              rp.EXECUTING               : ' 7',
              rp.PENDING_OUTPUT_STAGING  : ' 8',
              rp.STAGING_OUTPUT          : ' 9',
              rp.DONE                    : '10',
              rp.CANCELED                : '11',
              rp.FAILED                  : '12'}

    for pilot in docs['pilot'] :

        this_pid = str(pilot['_id'])

        with open ("/tmp/rp.%s.pilot.slots.%s.sdat" % (session, pid), "w") as dat :

            for e in events :

                etype   = e[0]
                otype   = e[1]
                uid     = e[2]
                pid     = e[3]
                ts      = e[4]
                state   = e[5]
                doc     = e[6]

                if  pid != this_pid :
                    continue

                if  otype == 'unit' :

                    if 'slots' in doc :
                        slots = doc['slots']
                    else :
                        pprint.pprint (doc)
                        slots = '?'

                    color = COLORS[state]

                    for slot in slots :

                        if  not uid in entity_states :
                            
                            if  state not in [rp.EXECUTING] :
                                continue

                            entity_states[uid] = state
                          # print      "%s %s%s %s"   % (ts, BEGIN, slot, COLORS[state])
                            dat.write ("%s %s%s %s\n" % (ts, BEGIN, slot, COLORS[state]))

                        else :
                            old_state = entity_states[uid]
                          # print      "%s %s%s %s"   % (ts, END,   slot, COLORS[old_state])
                            dat.write ("%s %s%s %s\n" % (ts, END,   slot, COLORS[old_state]))

                            entity_states[uid] = state
                          # print      "%s %s%s %s"   % (ts, BEGIN, slot, COLORS[state])
                            dat.write ("%s %s%s %s\n" % (ts, BEGIN, slot, COLORS[state]))


    for pilot in docs['pilot'] :

        this_pid = str(pilot['_id'])

        with open ("/tmp/rp.%s.pilot.units.%s.sdat" % (session, pid), "w") as dat :

            entity_states = dict()
            idxs          = list()

            # index the units by the start of their EXECUTING state
            for e in events :

                etype = e[0]
                otype = e[1]
                uid   = e[2]
                pid   = e[3]
                ts    = e[4]
                state = e[5]
                doc   = e[6]

                if  pid != this_pid :
                    continue

                if  otype == 'unit' :
                    if  state in [rp.EXECUTING] :
                        idxs.append (uid)

            for idx_id in idxs :

                for e in events :

                    etype = e[0]
                    otype = e[1]
                    uid   = e[2]
                    pid   = e[3]
                    ts    = e[4]
                    state = e[5]
                    doc   = e[6]

                    if  idx_id != uid :
                        continue

                    if  pid != this_pid :
                        continue

                    if  otype == 'unit' :

                        if 'slots' in doc :
                            slots = doc['slots']
                        else :
                            slots = '?'

                        for slot in slots :

                            if  not uid in entity_states :

                                entity_states[uid] = state

                              # print      "%s %s%s %s"   % (ts, BEGIN, uid, COLORS[state])
                                dat.write ("%s %s%s %s\n" % (ts, BEGIN, uid, COLORS[state]))

                            else :

                                old_state = entity_states[uid]
                              # print      "%s %s%s %s"   % (ts, END,   uid, COLORS[old_state])
                                dat.write ("%s %s%s %s\n" % (ts, END,   uid, COLORS[old_state]))

                                entity_states[uid] = state
                              # print      "%s %s%s %s"   % (ts, BEGIN, uid, COLORS[state])
                                dat.write ("%s %s%s %s\n" % (ts, BEGIN, uid, COLORS[state]))


# ------------------------------------------------------------------------------
def handle_session (db, mode, dbname, session, cachedir, pname) :
    """
    For the given db, traverse collections
    """

    # FIXME: cache(dir) is not used

    print " +-- db   %s" % dbname

    cnames = list()
    cnames.append ("%s"    % session)
    cnames.append ("%s.pm" % session)
    cnames.append ("%s.p"  % session)
    cnames.append ("%s.um" % session)
    cnames.append ("%s.cu" % session)

    for name in cnames :

        if  mode == 'list' and not cname :
            print " | +-- coll %s" % name

        elif  mode == 'remove' and not pname :
            try :
                db.drop_collection (name)
                print "  removed collection %s" % name
            except :
                pass # ignore errors

        else :
            handle_coll (db, mode, name, pname)



# ------------------------------------------------------------------------------
def handle_coll (db, mode, cname, pname) :
    """
    For a given collection, traverse all documents
    """

    if 'indexes' in cname :
        return

    collection = db[cname]
    print " | +-- coll %s" % cname

    docs = collection.find ()

    for doc in docs :

        name = doc['_id']

        if  mode == 'list' and not pname :
            print " | | +-- doc  %s" % name

        elif  mode == 'remove' :
            if (not pname) or (str(name)==str(pname)) :
                try :
                    collection.remove (name)
                    print "  removed document %s" % name
                except Exception as e:
                    pass # ignore errors

        else :
            if (not pname) or (str(name)==str(pname)) :
                handle_doc (collection, mode, doc)


# ------------------------------------------------------------------------------
def handle_doc (collection, mode, doc) :
    """
    And, surprise, for a given document, show it according to 'mode'
    """

    name = doc['_id']

    if  mode == 'list' :

        for key in doc :
            print " | | | +-- %s" % (key)

    elif  mode == 'tree' :
        print " | | +-- doc  %s" % (name)
        for key in doc :
            print " | | | +-- %s" % (key)

    elif  mode == 'dump' :
        print " | | +-- doc  %s" % (name)
        for key in doc :
            txt_in  = pprint.pformat (doc[key])
            txt_out = ""
            lnum    = 1
            for line in txt_in.split ('\n') :
                if  lnum != 1 :
                    txt_out += ' | | | |                '
                txt_out += line
                txt_out += '\n'
                lnum    += 1

            print " | | | +-- %-10s : %s" % (key, txt_out[:-1]) # remove last \n
# ------------------------------------------------------------------------------
# 
def parse_commandline():

    return options


# ------------------------------------------------------------------------------
#
if __name__ == '__main__' :

    import optparse
    parser = optparse.OptionParser (add_help_option=False)

    parser.add_option('-s', '--session',   dest='session')
    parser.add_option('-d', '--dburl',     dest='url')
    parser.add_option('-m', '--mode',      dest='mode')
    parser.add_option('-c', '--cachedir',  dest='cachedir')
    parser.add_option('-p', '--profdir',   dest='profdir')
    parser.add_option('-f', '--filter',    dest='filters')
    parser.add_option('-t', '--terminal',  dest='term')
    parser.add_option('-h', '--help',      dest='help', action="store_true")

    options, args = parser.parse_args ()

    if  args :
        usage ("Too many arguments (%s)" % args)

    if  options.help :
        usage ()

    if  options.mode in ['help'] : 
        usage ()

    if  not options.mode :
        usage ("No mode specified")

    if  options.url : 
        default_dburl = options.url
    else :
        if  'RADICAL_PILOT_DBURL' in os.environ : 
            default_dburl = os.environ['RADICAL_PILOT_DBURL']
        else :
            default_dburl = _DEFAULT_DBURL 

    if  not options.url :
        options.url = default_dburl


    mode     = options.mode 
    url      = options.url
    session  = options.session
    filters  = options.filters
    term     = options.term
    cachedir = options.cachedir
    profdir  = options.profdir

    if  not term :
        term = "pdf,png"

    url = ru.Url (options.url)
    if  not url.path or '/' == url.path :
        url.path = 'radicalpilot'

    if  not cachedir :
        cachedir = os.getcwd ()

    if  not os.path.isdir (cachedir) :
        usage ("%s is no valid cachedir" % cachedir)

    print "modes   : %s" % mode
    print "db url  : %s" % url
    print "cachedir: %s" % cachedir
    mongo, db, dbname, cname, pname = ru.mongodb_connect (str(url), default_dburl)

    if not session and mode != 'list' :
        usage("mode %s needs a session id specified" % mode)
    elif session and mode == 'list':
        usage("invalid session parameter '%s' on mode 'list'" % session)
    else:
        print "session : %s" % session

    for m in mode.split (',') :

        if  m not in ['list', 'dump', 'tree', 'hist', 'sort', 'stat', 'prof', 'plot', 'help'] :
            usage ("Unsupported mode '%s'" % m)

        if   m == 'list' : list_sessions (db, cachedir, filters)
        elif m == 'tree' : tree_session  (db, dbname, session, cachedir) 
        elif m == 'dump' : dump_session  (db, dbname, session, cachedir)
        elif m == 'sort' : sort_session  (db, session, cachedir)
        elif m == 'hist' : hist_session  (db, session, cachedir)
        elif m == 'stat' : stat_session  (db, session, cachedir)
        elif m == 'prof' : prof_session  (db, session, cachedir, profdir)
        elif m == 'plot' : plot_session  (db, session, cachedir, term)
        elif m == 'help' : usage (noexit=True)
        else             : usage ("unknown mode '%s'" % mode)

    # ------------------------------------------------------------------------------------
    mongo.disconnect ()

# ------------------------------------------------------------------------------

