#!/usr/bin/env python
# coding: utf-8

# # Calculate NERC level results
# Use the NERC extra generation and fuel consumption (things not yet reported in EIA-923 final data) from the *Calculate national and NERC gen and emissions* notebook, along with facility EIA and EPA data to calculate generation and emissions in each NERC region.
# 
# ## Instructions
# Make sure the `file_date` parameter below is set to whatever value you would like appended to file names.
# 
# The entire notebook can be run at once using *Run All Cells*

# In[76]:


get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import os
from os.path import join
import sys
import json
idx = pd.IndexSlice


# ### Date string for filenames
# This will be inserted into all filenames (reading and writing)

# In[77]:


file_date = '2018-03-06'


# In[78]:


get_ipython().run_line_magic('load_ext', 'watermark')


# In[79]:


get_ipython().run_line_magic('watermark', '-v -iv')


# In[80]:


# Load the "autoreload" extension
get_ipython().run_line_magic('load_ext', 'autoreload')

# always reload modules marked with "%aimport"
get_ipython().run_line_magic('autoreload', '1')


# In[81]:


# add the 'src' directory as one where we can import modules
src_dir = join(os.getcwd(), os.pardir, 'src')
sys.path.append(src_dir)


# In[82]:


get_ipython().run_line_magic('aimport', 'Data.make_data')
from Data.make_data import states_in_nerc
get_ipython().run_line_magic('aimport', 'Analysis.index')
from Analysis.index import facility_emission_gen, group_facility_data
get_ipython().run_line_magic('aimport', 'Analysis.index')
from Analysis.index import facility_co2, adjust_epa_emissions, group_fuel_cats
from Analysis.index import reduce_emission_factors, add_datetime
get_ipython().run_line_magic('aimport', 'util.utils')
from util.utils import rename_cols, add_facility_location


# ## Extra NERC emissions and generation

# ### Load extra NERC generation and fuel consumption

# In[83]:


cwd = os.getcwd()
path = join(cwd, '..', 'Data storage', 'Derived data',
            'NERC extra gen fuels {}.csv'.format(file_date))
extra_nerc = pd.read_csv(path, index_col=[0, 1, 2, 3])


# ### Load emission factors

# In[84]:


path = join(cwd, '..', 'Data storage', 'Final emission factors.csv')
ef = pd.read_csv(path, index_col=0)


# In[85]:


ef_type = reduce_emission_factors(ef)


# In[86]:


ef_type = pd.Series(ef_type, name='type')


# ### Calculate CO₂ emissions

# In[87]:


extra_nerc.loc[:, 'total co2 (kg)'] = (extra_nerc
                                       .loc[:, 'total fuel (mmbtu)']
                                       .multiply(ef_type, 'type'))
extra_nerc.loc[:, 'elec co2 (kg)'] = (extra_nerc
                                       .loc[:, 'elec fuel (mmbtu)']
                                       .multiply(ef_type, 'type'))


# In[88]:


extra_nerc.sort_index(inplace=True)


# ## Facilities by NERC

# In[89]:


get_ipython().run_line_magic('aimport', 'Analysis.state2nerc')
from Analysis.state2nerc import fraction_state2nerc, add_region


# In[90]:


fuel_cat_folder = join(cwd, '..', 'Data storage', 'Fuel categories')
state_cats_path = join(fuel_cat_folder, 'State_facility.json')

with open(state_cats_path, 'r') as f:
    state_fuel_cat = json.load(f)
    
custom_cats_path = join(fuel_cat_folder, 'Custom_results.json')
with open(custom_cats_path, 'r') as f:
    custom_fuel_cat = json.load(f)


# In[91]:


path = join(cwd, '..', 'Data storage', 'Derived data',
            'Monthly EPA emissions {}.csv'.format(file_date))
epa = pd.read_csv(path)


# In[92]:


path = join(cwd, '..', 'Data storage', 'Facility labels',
            'Facility locations_RF.csv')
facility_labels = pd.read_csv(path)


# In[93]:


path = join(cwd, '..', 'Data storage', 'Derived data',
            'Facility gen fuels and CO2 {}.csv'.format(file_date))
eia_fac = pd.read_csv(path)


# In[94]:


co2, gen_fuels_state = facility_emission_gen(eia_facility=eia_fac, epa=epa,
                                              state_fuel_cat=state_fuel_cat,
                                              custom_fuel_cat=custom_fuel_cat,
                                              export_state_cats=True)


# ### Monthly CO₂ emissions

# In[95]:


co2 = add_facility_location(co2, facility_labels,
                            labels=['lat', 'lon', 'state', 'nerc', 'year'])


# In[96]:


co2_nerc = co2.groupby(['year', 'nerc', 'month'])['final co2 (kg)'].sum()


# ### Monthly generation

# In[97]:


gen_fuels_nerc = add_facility_location(gen_fuels_state,
                                       facility_labels, labels=['nerc', 'year'])
gen_fuels_nerc = (gen_fuels_nerc
                  .groupby(['year', 'nerc', 'month', 'type'])
                  ['generation (mwh)'].sum())


# ## Combine generation

# In[98]:


total_gen = gen_fuels_nerc.copy()


# In[99]:


total_gen.loc[idx[2016:, :, :, :]] = (total_gen.loc[2016:] 
                                     .add(extra_nerc.loc[:, 'generation (mwh)']
                                          , fill_value=0))
total_gen = total_gen.reset_index()
add_datetime(total_gen)


# In[100]:


final = group_fuel_cats(total_gen,
                        custom_fuel_cat, 'type',
                        'fuel category', extra_group_cols=['nerc', 'datetime'])
final.set_index(['nerc', 'fuel category', 'datetime'], inplace=True)


# In[101]:


total = final.groupby(['nerc', 'datetime']).sum()


# In[102]:


nercs = total.index.get_level_values('nerc').unique()


# ### Percent generation by fuel
#     

# In[103]:


df_list = []
for nerc in nercs:
    percent_gen = final.loc[nerc].divide(total.loc[nerc], level='datetime')
    percent_gen['nerc'] = nerc
    percent_gen.set_index('nerc', append=True, inplace=True)
    df_list.append(percent_gen)
    
percent_gen = pd.concat(df_list)
percent_gen.drop(['year', 'month'], axis=1, inplace=True)
percent_gen.columns = ['% generation']


# In[104]:


path = join(cwd, '..', 'Data storage', 'Final NERC data',
            'NERC percent gen {}.csv'.format(file_date))
percent_gen.to_csv(path)


# In[105]:


total_monthly_gen = final.groupby(['fuel category', 'year', 'nerc', 'month']).sum()
total_monthly_gen.sort_index(inplace=True)


# In[106]:


path = join(cwd, '..', 'Data storage', 'Final NERC data',
            'NERC generation {}.csv'.format(file_date))
total_monthly_gen.to_csv(path)


# ## CO₂ emissions intensity by NERC region

# In[107]:


total_monthly_gen = total_monthly_gen.groupby(['year', 'nerc', 'month']).sum()


# In[108]:


nerc_index = pd.concat([co2_nerc.sort_index(), total_monthly_gen.sort_index()], axis=1)
nerc_index['index'] = nerc_index['final co2 (kg)'] / nerc_index['generation (mwh)']
nerc_index = nerc_index.reset_index()
add_datetime(nerc_index)


# ### Write data to file

# In[109]:


path = join(cwd, '..', 'Data storage', 'Final NERC data',
            'NERC gen emissions and index {}.csv'.format(file_date))
nerc_index.to_csv(path, index=False)


# In[ ]: