Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

""" 

Module for storing and managing statistics for each calculation. 

 

This includes creating the statistics (or calc summary) file. 

 

""" 

import time 

import datetime 

import itertools 

import os 

import shutil 

 

from libensemble.message_numbers import EVAL_SIM_TAG, EVAL_GEN_TAG 

 

#Todo: Move calc_status tags here - and make manager signals diff. This will then 

#need to be accessed by sim_func...Currently get from message_numbers 

from libensemble.message_numbers import WORKER_KILL 

from libensemble.message_numbers import WORKER_KILL_ON_ERR 

from libensemble.message_numbers import WORKER_KILL_ON_TIMEOUT 

from libensemble.message_numbers import JOB_FAILED 

from libensemble.message_numbers import WORKER_DONE 

from libensemble.message_numbers import MAN_SIGNAL_FINISH 

from libensemble.message_numbers import MAN_SIGNAL_KILL 

from libensemble.message_numbers import CALC_EXCEPTION 

 

class CalcInfo(): 

"""A class to store and manage statistics for each calculation. 

 

An object of this class represents the statistics for a given calculation. 

 

**Class Attributes:** 

 

:cvar string stat_file: 

A class attribute holding the name of the global summary file (default: 'libe_summary.txt') 

 

:cvar string statfile_dir: 

A class attribute holding the directory name for the worker summary files (default: 'libe_stat_files') 

 

:cvar string worker_statfile: 

A class attribute holding the pathname of the current workers summary file 

(default: Initially None, but is set to <stat_file>.w<workerID> when the file is created) 

 

:cvar boolean keep_worker_stat_files: 

A class attribute determining whether worker stat files are kept after merging 

to global summary file (default: False). 

 

 

**Object Attributes:** 

 

:ivar float time: Calculation run-time 

:ivar string date_start: Calculation start date 

:ivar string date_end: Calculation end date 

:ivar int calc_type: Type flag:EVAL_SIM_TAG/EVAL_GEN_TAG 

:ivar int id: Auto-generated ID for this calc (unique within Worker) 

:ivar string status: "Description of the status of this calc" 

 

""" 

newid = itertools.count() 

stat_file = 'libe_summary.txt' 

statfile_dir = 'libe_stat_files' 

worker_statfile = None 

keep_worker_stat_files = False 

 

calc_type_strings = { 

EVAL_SIM_TAG: 'sim', 

EVAL_GEN_TAG: 'gen', 

None: 'No type set' 

} 

 

calc_status_strings = { 

MAN_SIGNAL_FINISH: "Manager killed on finish", 

MAN_SIGNAL_KILL: "Manager killed job", 

WORKER_KILL_ON_ERR: " Worker killed job on Error", 

WORKER_KILL_ON_TIMEOUT: "Worker killed job on Timeout", 

WORKER_KILL: "Worker killed", 

JOB_FAILED: "Job Failed", 

WORKER_DONE: "Completed", 

CALC_EXCEPTION: "Exception occurred", 

None: "Unknown Status" 

} 

 

@staticmethod 

def set_statfile_name(name): 

"""Change the name ofr the statistics file""" 

CalcInfo.stat_file = name 

 

@staticmethod 

def smart_sort(l): 

""" Sort the given iterable in the way that humans expect. 

 

For example: Worker10 comes after Worker9. No padding required 

""" 

import re 

convert = lambda text: int(text) if text.isdigit() else text 

alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] 

return sorted(l, key=alphanum_key) 

 

@staticmethod 

def make_statdir(): 

statdir = CalcInfo.statfile_dir 

if os.path.exists(statdir): 

shutil.rmtree(statdir) 

os.mkdir(statdir) 

 

@staticmethod 

def create_worker_statfile(workerID): 

"""Create the statistics file""" 

statfile_name = CalcInfo.stat_file + '.w' + str(workerID) 

CalcInfo.worker_statfile = os.path.join(CalcInfo.statfile_dir, statfile_name) 

with open(CalcInfo.worker_statfile, 'w') as f: 

f.write("Worker %d:\n" % (workerID)) 

 

@staticmethod 

def add_calc_worker_statfile(calc): 

"""Add a new calculation to the statistics file""" 

with open(CalcInfo.worker_statfile, 'a') as f: 

calc.print_calc(f) 

 

@staticmethod 

def merge_statfiles(): 

"""Merge the stat files of each worker into one master file""" 

import glob 

worker_stat_files = os.path.join(CalcInfo.statfile_dir, CalcInfo.stat_file + '.w') 

stat_files = CalcInfo.smart_sort(glob.glob(worker_stat_files + '*')) 

with open(CalcInfo.stat_file, 'w+') as outfile: 

for fname in stat_files: 

with open(fname, 'r') as infile: 

outfile.write(infile.read()) 

if not CalcInfo.keep_worker_stat_files: 

shutil.rmtree(CalcInfo.statfile_dir) 

#for file in stat_files: 

#os.remove(file) 

 

def __init__(self): 

"""Create a new CalcInfo object 

 

A new CalcInfo object is created for each calculation. 

""" 

self.time = 0.0 

self.start = 0.0 

self.end = 0.0 

self.date_start = None 

self.date_end = None 

self.calc_type = None 

self.id = next(CalcInfo.newid) 

self.status = "Not complete" 

 

def start_timer(self): 

"""Start the timer and record datestamp (normally for a calculation)""" 

self.start = time.time() 

self.date_start = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") 

 

def stop_timer(self): 

"""Stop the timer and record datestamp (normally for a calculation) and set total run time""" 

self.end = time.time() 

self.date_end = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") 

#increment so can start and stop repeatedly 

self.time += self.end - self.start 

 

def print_calc(self, fileH): 

"""Print a calculation summary. 

 

This is called by add_calc_worker_statfile to add to statistics file. 

 

Parameters 

---------- 

 

fileH: file handle: 

File to print calc statistics to. 

 

""" 

fileH.write(" Calc %d: %s Time: %.2f Start: %s End: %s Status: %s\n" % (self.id, self.get_type(), self.time, self.date_start, self.date_end, self.status)) 

 

 

def get_type(self): 

"""Returns the calculation type as a string. 

 

Converts self.calc_type to string. self.calc_type should have been set by the worker""" 

return CalcInfo.calc_type_strings.get(self.calc_type, "Unknown type") 

 

 

def set_calc_status(self, calc_status_flag): 

"""Set status description for this calc 

 

Parameters 

---------- 

calc_status_flag: int 

Integer representing status of calc 

 

""" 

#For now assuming if not got an error - it was ok 

self.status = CalcInfo.calc_status_strings.get(calc_status_flag, "Completed")