Coverage for aixweather/core_data_format_2_output_file/unconverted_to_x.py: 98%

1""" 

2converts core data to different simpler formats (currently without any transformation) 

3""" 

4 

5import logging 

6import json 

7import pickle 

8import pandas as pd 

9 

10from aixweather import definitions 

11from aixweather.imports.utils_import import MetaData 

12 

13 

14logger = logging.getLogger(__name__) 

15 

16 

17def _get_file_name_and_meta_data_file_name( 

18 meta: MetaData, suffix: str, filename: str = None 

19) -> (str, str): 

20 """ 

21 Return the filename and meta-data filename for the given suffix 

22 """ 

23 if filename is None: 

24 filename = f"Station_{meta.station_name}.{suffix}" 

25 meta_filename = f"Station_{meta.station_name}_meta_data.{suffix}" 

26 else: 

27 meta_filename = filename.replace(f".{suffix}", f"_meta_data.{suffix}") 

28 return filename, meta_filename 

29 

30 

31def to_pickle( 

32 core_df: pd.DataFrame, 

33 meta: MetaData, 

34 result_folder: str = None, 

35 filename: str = None 

36) -> (pd.DataFrame, str): 

37 """Create and save a pickle file from the core data. 

38 

39 Args: 

40 core_df (pd.DataFrame): DataFrame containing core data. 

41 meta (MetaData): Metadata associated with the data. 

42 result_folder (str): 

43 Path to the folder where to save the file. Default will use 

44 the `results_file_path` method. 

45 filename (str): Name of the file to be saved. The default is constructed 

46 based on the station name. 

47 

48 Returns: 

49 pd.DataFrame: DataFrame containing the weather data formatted as core data. 

50 str: Path to the exported file. 

51 """ 

52 

53 core_df = core_df.copy() 

54 filename, meta_filename = _get_file_name_and_meta_data_file_name( 

55 meta=meta, filename=filename, suffix="pkl" 

56 ) 

57 file_path = definitions.results_file_path(filename, result_folder) 

58 meta_file_path = definitions.results_file_path(meta_filename, result_folder) 

59 

60 core_df.to_pickle(file_path) 

61 with open(meta_file_path, "wb") as file: 

62 pickle.dump(meta, file) 

63 

64 logger.info("Pickle saved to %s, meta information saved to %s.", file_path, meta_file_path) 

65 

66 return core_df, file_path 

67 

68 

69def to_json( 

70 core_df: pd.DataFrame, 

71 meta: MetaData, 

72 result_folder: str = None, 

73 filename: str = None 

74) -> (pd.DataFrame, str): 

75 """Create and save a json file from the core data. 

76 

77 Args: 

78 core_df (pd.DataFrame): DataFrame containing core data. 

79 meta (MetaData): Metadata associated with the data. 

80 result_folder (str): 

81 Path to the folder where to save the file. Default will use 

82 the `results_file_path` method. 

83 filename (str): Name of the file to be saved. The default is constructed 

84 based on the station name. 

85 

86 Returns: 

87 pd.DataFrame: DataFrame containing the weather data formatted as core data. 

88 str: Path to the exported file. 

89 """ 

90 

91 core_df = core_df.copy() 

92 

93 filename, meta_filename = _get_file_name_and_meta_data_file_name( 

94 meta=meta, filename=filename, suffix="json" 

95 ) 

96 file_path = definitions.results_file_path(filename, result_folder) 

97 meta_file_path = definitions.results_file_path(meta_filename, result_folder) 

98 

99 # Convert DataFrame to JSON and save to file 

100 core_df.to_json(file_path, orient="records") 

101 

102 # Convert meta_data to a dictionary and save to JSON file 

103 meta_dict = meta.__dict__ # Convert the meta_data object to a dictionary 

104 with open(meta_file_path, "w") as file: 

105 json.dump(meta_dict, file, indent=4) 

106 

107 logger.info("JSON saved to %s, meta information saved to %s.", file_path, meta_file_path) 

108 

109 return core_df, file_path 

110 

111 

112def to_csv( 

113 core_df: pd.DataFrame, 

114 meta: MetaData, 

115 result_folder: str = None, 

116 filename: str = None 

117) -> (pd.DataFrame, str): 

118 """Create and save a csv file from the core data. 

119 

120 Args: 

121 core_df (pd.DataFrame): DataFrame containing core data. 

122 meta (MetaData): Metadata associated with the data. 

123 result_folder (str): 

124 Path to the folder where to save the file. Default will use 

125 the `results_file_path` method. 

126 filename (str): Name of the file to be saved. The default is constructed 

127 based on the station name. 

128 

129 Returns: 

130 pd.DataFrame: DataFrame containing the weather data formatted as core data. 

131 str: Path to the exported file. 

132 """ 

133 

134 core_df = core_df.copy() 

135 

136 filename, meta_filename = _get_file_name_and_meta_data_file_name( 

137 meta=meta, filename=filename, suffix="csv" 

138 ) 

139 file_path = definitions.results_file_path(filename, result_folder) 

140 meta_file_path = definitions.results_file_path(meta_filename, result_folder) 

141 

142 core_df.to_csv(file_path, index=True) 

143 

144 # Convert meta_data to a dictionary and save to JSON file 

145 meta_dict = meta.__dict__ # Convert the meta_data object to a dictionary 

146 with open(meta_file_path, "w") as file: 

147 json.dump(meta_dict, file, indent=4) 

148 

149 logger.info("CSV saved to %s, meta information saved to %s.", file_path, meta_file_path) 

150 

151 return core_df, file_path