Use the NERC extra generation and fuel consumption (things not yet reported in EIA-923 final data) from the Calculate national and NERC gen and emissions notebook, along with facility EIA and EPA data to calculate generation and emissions in each NERC region.
Make sure the file_date
parameter below is set to whatever value you would like appended to file names.
The entire notebook can be run at once using Run All Cells
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import os
from os.path import join
import sys
import json
idx = pd.IndexSlice
This will be inserted into all filenames (reading and writing)
file_date = '2018-03-06'
%load_ext watermark
%watermark -v -iv
seaborn 0.8.1 pandas 0.22.0 numpy 1.14.5 json 2.0.9 CPython 3.6.5 IPython 6.4.0
# Load the "autoreload" extension
%load_ext autoreload
# always reload modules marked with "%aimport"
%autoreload 1
# add the 'src' directory as one where we can import modules
src_dir = join(os.getcwd(), os.pardir, 'src')
sys.path.append(src_dir)
%aimport Data.make_data
from Data.make_data import states_in_nerc
%aimport Analysis.index
from Analysis.index import facility_emission_gen, group_facility_data
%aimport Analysis.index
from Analysis.index import facility_co2, adjust_epa_emissions, group_fuel_cats
from Analysis.index import reduce_emission_factors, add_datetime
%aimport util.utils
from util.utils import rename_cols, add_facility_location
cwd = os.getcwd()
path = join(cwd, '..', 'Data storage', 'Derived data',
'NERC extra gen fuels {}.csv'.format(file_date))
extra_nerc = pd.read_csv(path, index_col=[0, 1, 2, 3])
--------------------------------------------------------------------------- FileNotFoundError Traceback (most recent call last) <ipython-input-8-5cc7fabc52e5> in <module>() 2 path = join(cwd, '..', 'Data storage', 'Derived data', 3 'NERC extra gen fuels {}.csv'.format(file_date)) ----> 4 extra_nerc = pd.read_csv(path, index_col=[0, 1, 2, 3]) ~/miniconda3/envs/psci/lib/python3.6/site-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision) 707 skip_blank_lines=skip_blank_lines) 708 --> 709 return _read(filepath_or_buffer, kwds) 710 711 parser_f.__name__ = name ~/miniconda3/envs/psci/lib/python3.6/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds) 447 448 # Create the parser. --> 449 parser = TextFileReader(filepath_or_buffer, **kwds) 450 451 if chunksize or iterator: ~/miniconda3/envs/psci/lib/python3.6/site-packages/pandas/io/parsers.py in __init__(self, f, engine, **kwds) 816 self.options['has_index_names'] = kwds['has_index_names'] 817 --> 818 self._make_engine(self.engine) 819 820 def close(self): ~/miniconda3/envs/psci/lib/python3.6/site-packages/pandas/io/parsers.py in _make_engine(self, engine) 1047 def _make_engine(self, engine='c'): 1048 if engine == 'c': -> 1049 self._engine = CParserWrapper(self.f, **self.options) 1050 else: 1051 if engine == 'python': ~/miniconda3/envs/psci/lib/python3.6/site-packages/pandas/io/parsers.py in __init__(self, src, **kwds) 1693 kwds['allow_leading_cols'] = self.index_col is not False 1694 -> 1695 self._reader = parsers.TextReader(src, **kwds) 1696 1697 # XXX pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.__cinit__() pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._setup_parser_source() FileNotFoundError: File b'/Users/greg/Documents/GitHub/carbon-index/Notebooks/../Data storage/Derived data/NERC extra gen fuels 2018-03-06.csv' does not exist
path = join(cwd, '..', 'Data storage', 'Final emission factors.csv')
ef = pd.read_csv(path, index_col=0)
ef_type = reduce_emission_factors(ef)
ef_type = pd.Series(ef_type, name='type')
extra_nerc.loc[:, 'total co2 (kg)'] = (extra_nerc
.loc[:, 'total fuel (mmbtu)']
.multiply(ef_type, 'type'))
extra_nerc.loc[:, 'elec co2 (kg)'] = (extra_nerc
.loc[:, 'elec fuel (mmbtu)']
.multiply(ef_type, 'type'))
extra_nerc.sort_index(inplace=True)
%aimport Analysis.state2nerc
from Analysis.state2nerc import fraction_state2nerc, add_region
fuel_cat_folder = join(cwd, '..', 'Data storage', 'Fuel categories')
state_cats_path = join(fuel_cat_folder, 'State_facility.json')
with open(state_cats_path, 'r') as f:
state_fuel_cat = json.load(f)
custom_cats_path = join(fuel_cat_folder, 'Custom_results.json')
with open(custom_cats_path, 'r') as f:
custom_fuel_cat = json.load(f)
path = join(cwd, '..', 'Data storage', 'Derived data',
'Monthly EPA emissions {}.csv'.format(file_date))
epa = pd.read_csv(path)
path = join(cwd, '..', 'Data storage', 'Facility labels',
'Facility locations_RF.csv')
facility_labels = pd.read_csv(path)
path = join(cwd, '..', 'Data storage', 'Derived data',
'Facility gen fuels and CO2 {}.csv'.format(file_date))
eia_fac = pd.read_csv(path)
co2, gen_fuels_state = facility_emission_gen(eia_facility=eia_fac, epa=epa,
state_fuel_cat=state_fuel_cat,
custom_fuel_cat=custom_fuel_cat,
export_state_cats=True)
Renaming columns Grouping facilities Adjusting EPA emissions Caculating CO2 Gen/fuels to state categories
co2 = add_facility_location(co2, facility_labels,
labels=['lat', 'lon', 'state', 'nerc', 'year'])
co2_nerc = co2.groupby(['year', 'nerc', 'month'])['final co2 (kg)'].sum()
gen_fuels_nerc = add_facility_location(gen_fuels_state,
facility_labels, labels=['nerc', 'year'])
gen_fuels_nerc = (gen_fuels_nerc
.groupby(['year', 'nerc', 'month', 'type'])
['generation (mwh)'].sum())
total_gen = gen_fuels_nerc.copy()
total_gen.loc[idx[2016:, :, :, :]] = (total_gen.loc[2016:]
.add(extra_nerc.loc[:, 'generation (mwh)']
, fill_value=0))
total_gen = total_gen.reset_index()
add_datetime(total_gen)
final = group_fuel_cats(total_gen,
custom_fuel_cat, 'type',
'fuel category', extra_group_cols=['nerc', 'datetime'])
final.set_index(['nerc', 'fuel category', 'datetime'], inplace=True)
total = final.groupby(['nerc', 'datetime']).sum()
nercs = total.index.get_level_values('nerc').unique()
df_list = []
for nerc in nercs:
percent_gen = final.loc[nerc].divide(total.loc[nerc], level='datetime')
percent_gen['nerc'] = nerc
percent_gen.set_index('nerc', append=True, inplace=True)
df_list.append(percent_gen)
percent_gen = pd.concat(df_list)
percent_gen.drop(['year', 'month'], axis=1, inplace=True)
percent_gen.columns = ['% generation']
path = join(cwd, '..', 'Data storage', 'Final NERC data',
'NERC percent gen {}.csv'.format(file_date))
percent_gen.to_csv(path)
total_monthly_gen = final.groupby(['fuel category', 'year', 'nerc', 'month']).sum()
total_monthly_gen.sort_index(inplace=True)
path = join(cwd, '..', 'Data storage', 'Final NERC data',
'NERC generation {}.csv'.format(file_date))
total_monthly_gen.to_csv(path)
total_monthly_gen = total_monthly_gen.groupby(['year', 'nerc', 'month']).sum()
nerc_index = pd.concat([co2_nerc.sort_index(), total_monthly_gen.sort_index()], axis=1)
nerc_index['index'] = nerc_index['final co2 (kg)'] / nerc_index['generation (mwh)']
nerc_index = nerc_index.reset_index()
add_datetime(nerc_index)
path = join(cwd, '..', 'Data storage', 'Final NERC data',
'NERC gen emissions and index {}.csv'.format(file_date))
nerc_index.to_csv(path, index=False)