Coverage for addmo_examples/executables/exe_data_insights.py: 33%

1import os 

2import pandas as pd, csv 

3import datetime 

4from addmo.s5_insights.model_plots.time_series import plot_timeseries_combined 

5from addmo.s5_insights.model_plots.parallel_plots import parallel_plots, parallel_plots_interactive 

6from addmo.util.plotting_utils import save_pdf 

7from addmo.s5_insights.model_plots.carpet_plots import plot_carpets, plot_carpets_with_buckets, prediction_func_4_regressor 

8from addmo.util.definitions import return_results_dir_model_tuning, return_best_model, load_model_config 

9from addmo.s3_model_tuning.models.model_factory import ModelFactory 

10from addmo.util.load_save import load_data 

11 

12def exe_time_series_plot(dir, plot_name, plot_dir, save=True): 

13 """ 

14 Executes plotting of input data. 

15 """ 

16 # Load config 

17 model_config = load_model_config(dir) 

18 

19 # Load data 

20 data_path = model_config['abs_path_to_data'] 

21 data = load_data(data_path) 

22 # Execute plotting 

23 figures = plot_timeseries_combined(model_config, data) 

24 

25 

26 if not isinstance(figures, list): 

27 figures = [figures] 

28 if save: 

29 os.makedirs(plot_dir, exist_ok=True) 

30 for idx, fig in enumerate(figures): 

31 suffix = "_2weeks" if idx == 1 else "" 

32 plot_path = os.path.join(plot_dir, f"{plot_name}{suffix}") 

33 save_pdf(fig, plot_path) 

34 else: 

35 for fig in figures: 

36 fig.show() 

37 

38def exe_carpet_plots(dir, plot_name, plot_dir, save = True, bounds= None, defaults_dict= None, combinations= None, path_to_regressor=None): 

39 """ 

40 Executes carpet model_plots of input data features along with predictions using saved model. 

41 """ 

42 # Load config 

43 model_config = load_model_config(dir) 

44 

45 if path_to_regressor is None: 

46 path_to_regressor = return_best_model(dir) # return default path where model is saved 

47 # Load regressor 

48 regressor = ModelFactory.load_model(path_to_regressor) 

49 pred_func_1 = prediction_func_4_regressor(regressor) 

50 # Load target variable 

51 target = model_config["name_of_target"] 

52 ask_data_path= True 

53 

54 # No need to use input data if user provides bounds and default dictionary 

55 if bounds is not None and defaults_dict is not None: 

56 variables = regressor.metadata["features_ordered"] 

57 measurements_data = None 

58 ask_data_path = False 

59 

60 # Load the input data and fetch variables from it 

61 if ask_data_path: 

62 data_path = model_config['abs_path_to_data'] 

63 data = load_data(data_path) 

64 

65 measurements_data = data.drop(target, axis=1) 

66 variables = list(measurements_data.columns) 

67 

68 # Execute plotting 

69 plt= plot_carpets(variables=variables, measurements_data= measurements_data, regressor_func= pred_func_1, bounds = bounds, combinations = combinations, defaults_dict = defaults_dict) 

70 

71 if save: 

72 os.makedirs(plot_dir, exist_ok=True) 

73 plot_path = os.path.join(plot_dir, plot_name) 

74 plt.show() 

75 save_pdf(plt, plot_path) 

76 else: 

77 plt.show() 

78 

79def exe_scatter_carpet_plots(dir, plot_name, plot_dir, save = True, bounds= None, defaults_dict= None, combinations= None, path_to_regressor= None): 

80 """ 

81 Executes carpet model_plots of input data features along with predictions using saved model. 

82 """ 

83 

84 # If model config is saved in same directory: 

85 # Load config 

86 model_config = load_model_config(dir) 

87 

88 if path_to_regressor is None: 

89 path_to_regressor = return_best_model(dir) # return default path where model is saved 

90 # Load regressor 

91 regressor = ModelFactory.load_model(path_to_regressor) 

92 pred_func_1 = prediction_func_4_regressor(regressor) 

93 

94 # Load target variable 

95 target = model_config["name_of_target"] 

96 

97 # Load the input data and fetch variables from it 

98 

99 data_path = model_config['abs_path_to_data'] 

100 data = load_data(data_path) 

101 

102 measurements_data = data.drop(target, axis=1) 

103 variables = list(measurements_data.columns) 

104 target_values = pd.DataFrame(data[target]) 

105 

106 # Execute plotting 

107 plt = plot_carpets_with_buckets(variables=variables, measurements_data= measurements_data, target_values= target_values, regressor_func= pred_func_1, bounds = bounds, combinations = combinations, defaults_dict = defaults_dict) 

108 

109 if save: 

110 os.makedirs(plot_dir, exist_ok=True) 

111 plot_path = os.path.join(plot_dir, plot_name) 

112 plt.show() 

113 save_pdf(plt, plot_path) 

114 else: 

115 plt.show() 

116 

117def exe_parallel_plot(dir, plot_name, plot_dir, save = True, path_to_regressor=None): 

118 """ 

119 Executes parallel model_plots of input data features along with predictions using saved model. 

120 """ 

121 

122 # Load config 

123 model_config = load_model_config(dir) 

124 # Load regressor 

125 if path_to_regressor is None: 

126 path_to_regressor = return_best_model(dir) 

127 regressor = ModelFactory.load_model(path_to_regressor) 

128 

129 # Load target and data 

130 target = model_config["name_of_target"] 

131 data_path = model_config['abs_path_to_data'] 

132 data = load_data(data_path) 

133 

134 # Execute plotting 

135 plt = parallel_plots(target, data, regressor) 

136 

137 if save: 

138 os.makedirs(plot_dir, exist_ok=True) 

139 plot_path = os.path.join(plot_dir, plot_name) 

140 save_pdf(plt, plot_path) 

141 

142def exe_interactive_parallel_plot(dir, plot_name, plot_dir, save = True, path_to_regressor=None): 

143 """ 

144 Executes parallel model_plots of input data features along with predictions using saved model. 

145 """ 

146 # Load config 

147 model_config = load_model_config(dir) 

148 

149 # Load regressor 

150 if path_to_regressor is None: 

151 path_to_regressor = return_best_model(dir) 

152 regressor = ModelFactory.load_model(path_to_regressor) 

153 

154 # Load target and data 

155 target = model_config["name_of_target"] 

156 data_path = model_config['abs_path_to_data'] 

157 data = load_data(data_path) 

158 # Execute plotting 

159 plt = parallel_plots_interactive(target, data, regressor) 

160 

161 if save: 

162 os.makedirs(plot_dir, exist_ok=True) 

163 plot_path = os.path.join(plot_dir, plot_name) 

164 plt.write_html(plot_path) 

165 

166 return plt 

167 

168 

169if __name__ == '__main__': 

170 

171 

172 # Define directory where the model config and regressor is saved: 

173 _path_to_input_dir = return_results_dir_model_tuning('raw_test', 'test_data_tuning', 'test_model_tuning') 

174 

175 # Path for saving the model_plots 

176 plot_dir = os.path.join(_path_to_input_dir, 'plots') 

177 # Define regressor path if it is not saved as 'best_model.ext' 

178 # path_to_regressor = os.path.join(_path_to_input_dir, 'regressor.keras') 

179 

180 

181 # Execute plotting functions 

182 exe_scatter_carpet_plots(_path_to_input_dir, plot_name = "carpet_scatter_plot", plot_dir= plot_dir,save=True, path_to_regressor=None) #Todo: Note for user: pls check input data path inside the exe func 

183 exe_time_series_plot(_path_to_input_dir, plot_name = "training_data_time_series", plot_dir= plot_dir,save=False) 

184 exe_carpet_plots(_path_to_input_dir, plot_name = "predictions_carpet_test", plot_dir= plot_dir, save=True) 

185 exe_parallel_plot(_path_to_input_dir, plot_name = "parallel_plot", plot_dir= plot_dir, save=False) 

186 exe_interactive_parallel_plot(_path_to_input_dir, plot_name = "interactive_parallel_plot", plot_dir= plot_dir, save=False) 

187 

188 

189