25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207 | def load(api_key:str, provinces:list, demand_year:int, vre_year:int) -> pd.DataFrame:
"""
This function is used to get necessary input data from the CODERS API
Arguments:
api_key: str - The API key to access the CODERS data
provinces: list - List of provinces to filter the data
demand_year: int - The year for which the demand data is to be fetched
vre_year: int - The year for which the VRE capacity factors are to be fetched
Returns:
coders_data: pd.DataFrame - A DataFrame containing the CODERS data for the specified provinces and years
"""
with tqdm(total=12, desc="Loading CODERS data") as pbar:
pbar.set_postfix_str('Generators')
generators = coders_generators.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Storage')
storage = coders_storage.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Transmission')
transmission = coders_transmission.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Costs')
cost = coders_cost.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Technology parameters')
technology_params = coders_technology_parameters.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Interprovincial transmission')
provincial_transfers = coders_provincial_transfers.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Pumped hydro storage')
hydro_pump = coders_hydro_pump_storage.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Hydro greefield')
hydro_greenfield = coders_hydro_greenfield.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Hydro renewals')
hydro_renewal = coders_hydro_renewal.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('System parameters')
ca_system_params = coders_ca_system_parameters.get_data(api_key)
# pbar.update(1)
# pbar.set_postfix_str('Cost evolution')
# generation_cost_evolution = coders_generation_cost_evolution.get_data(api_key)
# pbar.update(1)
# pbar.set_postfix_str('Planning reserve margins')
# generation_planning_reserve = coders_generation_planning_reserve.get_data(api_key)
# pbar.update(1)
pbar.set_postfix_str('Transmission parameters')
transmission_generic = coders_transmission_generic.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Demand forecast')
forecasted_annual_demand = coders_forecasted_annual_demand.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Grid cells')
gridcell_data = coders_grid_cells.get_data(api_key)
pbar.update(1)
pbar.set_postfix_str('Provincial data')
coders_data = pd.concat([generators, transmission, cost, technology_params, storage,
hydro_pump, hydro_greenfield, hydro_renewal,
ca_system_params,
# generation_cost_evolution, generation_planning_reserve,
transmission_generic,
forecasted_annual_demand, gridcell_data
])
coders_data['region'] = coders_data['region'].fillna('CAN')
# replace e with accent for Quebec to match with the province names
coders_data['region'] = coders_data['region'].replace('Québec', 'Quebec')
coders_data['region'] = coders_data['region'].replace('Saskatachewan', 'Saskatchewan')
province_filter = provinces + ['CAN']
for prov in provinces:
if prov in province_short_to_long.keys():
province_filter.append(province_short_to_long[prov] + '.a')
province_filter.append(province_short_to_long[prov] + '.b')
province_filter.append(province_short_to_long[prov])
coders_data = coders_data[coders_data.region.isin(province_filter)]
regional_generators = coders_data[coders_data.variable.str.contains('Generators|Capacity|', regex=False)]
hydro_generators = regional_generators[regional_generators.variable.str.contains('hydro_')]
vre_generators = regional_generators[regional_generators.variable.str.contains('solar|wind')]
data = []
no_us_data = []
no_hydro_data = []
for province in (pbar2 := tqdm(provinces, desc="Processing provincial demand, and hydro capacity factors")):
pbar2.set_postfix_str(province)
demand = coders_demand.get_canadian_data(api_key, province, demand_year)
try:
demand_prior = coders_demand.get_canadian_data(api_key, province, demand_year - 1)
except:
# If there is no data for the previous year, just use the data for the current year
demand_prior = demand.copy()
demand_prior['time'] = demand_prior['time'] - pd.DateOffset(years=1)
demand = pd.concat([demand, demand_prior])
demand['time'] = pd.to_datetime(demand['time'])
demand['time'] = demand['time'].apply(lambda x: timezone[province].localize(x).astimezone(pytz.utc))
demand = demand[demand['time'].dt.year == demand_year]
us_demand = coders_demand.get_us_data(api_key, province, demand_year)
if not us_demand.empty:
try:
us_demand['time'] = pd.to_datetime(us_demand['time'])
except Exception as e:
print('Error getting US demand', e)
us_demand['time'] = pd.to_datetime(us_demand['time'], utc=True)
try:
us_demand_prior = coders_demand.get_us_data(api_key, province, demand_year - 1)
try:
us_demand_prior['time'] = pd.to_datetime(us_demand_prior['time'])
except Exception as e:
print('Error getting US demand', e)
us_demand_prior['time'] = pd.to_datetime(us_demand_prior['time'], utc=True)
except:
# If there is no data for the previous year, just use the data for the current year
us_demand_prior = us_demand.copy()
us_demand_prior['time'] = us_demand_prior['time'] - pd.DateOffset(years=1)
us_demand = pd.concat([us_demand, us_demand_prior])
try:
us_demand['time'] = us_demand['time'].apply(
lambda x: timezone[province].localize(x).astimezone(pytz.utc))
except:
us_demand['time'] = us_demand['time'].apply(
lambda x: timezone[province].localize(x.replace(tzinfo=None)).astimezone(pytz.utc))
us_demand = us_demand[us_demand['time'].dt.year == demand_year]
us_demand['region'] = us_demand['region'].apply(lambda x: province_short_to_long[x.split('.')[0]] + '.' + x.split('.')[1])
else:
no_us_data.append(province)
data.append(us_demand)
data.append(demand)
# if not hydro_generators[hydro_generators.region.isin(province_filter)].empty:
# try:
# hydro_cf = capacity_factors.get_hydro_data(api_key, province, vre_year)
# data.append(hydro_cf)
# except:
# no_hydro_data.append(province)
pbar.update(1)
# pbar.set_postfix_str('VRE capacity factors')
# if not vre_generators.empty:
# vre_cf = capacity_factors.get_wind_solar_data(api_key, vre_year)
# regional_vre_cf = vre_cf[vre_cf.region.isin(province_filter + ['CAN'])]
# data.append(regional_vre_cf)
# pbar.update(1)
pbar.set_postfix_str('Final processing')
coders_data = pd.concat([coders_data] + data)
coders_data['region'] = coders_data['region'].fillna('CAN')
coders_data['region'] = coders_data['region'].replace('Québec', 'Quebec')
coders_data['region'] = coders_data['region'].replace('Saskatachewan', 'Saskatchewan')
coders_data = coders_data[coders_data.region.isin(province_filter + ['CAN'])]
coders_data['region'] = coders_data['region']
provincial_transfers['to_region'] = provincial_transfers['variable'].str.split('|').str[-1]
provincial_transfers['variable'] = provincial_transfers['variable'].str.split('|').str[:-1].str.join('|')
provincial_transfers = provincial_transfers[
provincial_transfers.to_region.isin(province_filter) | provincial_transfers.region.isin(province_filter)]
provincial_transfers['region'] = provincial_transfers['region']
provincial_transfers['to_region'] = provincial_transfers['to_region']
provincial_transfers['variable'] += '|' + provincial_transfers['to_region']
provincial_transfers = provincial_transfers.drop(columns='to_region')
coders_data = pd.concat([coders_data, provincial_transfers])
pbar.update(1)
print('No US demand for provinces: ', no_us_data)
print('No hydro data for: ', no_hydro_data)
return coders_data
|