#!/usr/bin/env python
# coding: utf-8

# # Democratic Prospects for the 2018 Midterm
# 
# Below, I look at the prospects for each party in the 2018 election.  Overall, the House actually looks pretty good for Democrats, with many Republicans up for re-election in districts that Clinton won.  The Senate is a different story, with many Democrats running in states that Trump won by a considerable margin.  But who knows what will happen if the election becomes a referendum on Trump.
# 
# The data for this post come from [The DailyKos](https://docs.google.com/spreadsheets/d/1oRl7vxEJUUDWJCyrjo62cELJD2ONIVl-D9TSUKiK9jk/edit#gid=2132957126).  If you're viewing this notebook on Github, view it in NBViewer [here](http://nbviewer.jupyter.org/gist/psthomas/ed54ed0586f224544c5ad83763cb7267) instead to see the interactive plots and tables.  I don't have an in-depth understanding of politics, so it's possible I'm missing some good indicators for midterm success.  Let me know if you have any ideas for additions.

# In[2]:


get_ipython().run_line_magic('matplotlib', 'inline')

import pandas as pd 
import numpy as np
import json
 
from IPython.display import HTML


# # House Seats at Risk
# 
# The plot below shows the 2016 Deocratic presidential lead on the x-axis, and the 2016 Democratic House lead on the y-axis.  Any point near the x-axis was a close race, with those just above the axis won by Democrats and those just below won by Republicans.  Points in the top left quadrant are seats that are especially at risk for Democrats because Trump won these districts.  
# 
# In the bottom right are districts that were won by Clinton in the presidential race, yet still held by House Republicans.  There seem to be more Republican House seats at risk in 2018, which could especially be dangerous if the election becomes a referendum on Trump.  

# In[3]:


house_df = pd.read_csv('house_data.csv', sep=',', encoding='utf-8')
house_df['demlead_house2016'] = house_df['housedem_2016'] - house_df['houserep_2016']
house_df['demlead_pres2016'] = ((house_df['clinton_2016'] - house_df['trump_2016'])/house_df['total_2016'])*100
house_df['sum'] = house_df['demlead_house2016'] + house_df['demlead_pres2016']
house_df = house_df.round(2)


# In[48]:


tooltip = '''
    '<b>District:</b> '+ d[keys.code] + 
    '<br><b>Name:</b> ' + d[keys.first] + ' ' + d[keys.last] + 
    '<br><b>President:</b> ' + d[keys.demlead_pres2016] + "%" + 
    '<br><b>House:</b> ' + d[keys.demlead_house2016] +"%" 
''' 

settings = {"x_label": "Democratic Lead, President 2016 (%)", 
            "y_label": "Democratic Lead, House 2016 (%)",
            "x": 'demlead_pres2016' ,
            "y": 'demlead_house2016', 
            "tooltip": tooltip}

interactive_scatter(house_df, settings=settings)


# ## Close House Seats for Democrats
# 
# The first two numeric columns below show the Democrat's lead in the House race and Presidential race for each district.  When the sum column of these two values is negative it indicates a district that had a larger margin for Trump than for the Democrat that was elected.  
# 
# Interestingly, Minnesota had a number of close seats, including the 7th district with a 35 point spread between Collin Peterson and Trump.  Someone needs to ask these guys in Minnesota how they appealed to Trump voters ([Peterson](https://en.wikipedia.org/wiki/Collin_Peterson) is a Blue Dog Democrat, and both [Nolan](https://en.wikipedia.org/wiki/Rick_Nolan) and [Walz](https://en.wikipedia.org/wiki/Rick_Nolan) are moderate Democrats).  I think Walz provides an especially good template for moderate Democrats to follow in contentious districts.   
# 
# I also include the [Partisan Voter Index](https://en.wikipedia.org/wiki/Cook_Partisan_Voting_Index) (pvi_2016), which combines the past two elections into an index of how the district votes relative to the country.  Currently, the table is sorted by the 'sum' column, but Bokeh outputs interactive tables so feel free to sort the columns by any of the indicators. 

# In[58]:


temp_df = house_df[(house_df['party'] == 'Democratic')].copy() 
temp_df.sort_values(by='sum', ascending=True, inplace=True)

temp_df = temp_df[['district', 'party', 'first', 'last', 'demlead_house2016', 
                   'demlead_pres2016', 'sum', 'pvi_2016']]

interactive_table(temp_df, width=800, height=500)


# ## Close House Seats for Republicans
# 
# I do the same thing below for House Republicans, but this time a positive sum indicates a district that had a larger margin for Clinton than for the Republican elected.  I also include a column showing whether or not the [Democratic Congressional Campaign Committee](http://dccc.org/) has indicated that this member of the House is a target for the 2018 election.  This shows that the 'sum' column is a fairly good indicator for a midterm target.
#  

# In[59]:


#http://action.dccc.org/pdf/dccc-on-offense.pdf
#59 total targets for 2018 elections
dccc_target = ["AL-02", "AR-02", "AZ-02", "CA-10", "CA-21", "CA-25", "CA-39", "CA-45", 
                "CA-48", "CA-49", "CO-03", "CO-06", "FL-18", "FL-25", "FL-26", "FL-27", 
                "GA-06", "IA-01", "IA-03", "IL-06", "IL-13", "IL-14", "KS-02", "KS-03", 
                "KY-06", "ME-02", "MI-07", "MI-08", "MI-11", "MN-02", "MN-03", "NC-08", 
                "NC-09", "NC-13", "NE-02", "NJ-02", "NJ-03", "NJ-07", "NJ-11", "NY-01", 
                "NY-11", "NY-19", "NY-22", "NY-24", "NY-27", "OH-01", "OH-07", "PA-06", 
                "PA-07", "PA-08", "PA-16", "TX-07", "TX-23", "TX-32", "VA-02", "VA-10", 
                "WA-03", "WA-08", "WV-02"]

#Build dataframe
temp_df = house_df[(house_df['party'] == 'Republican')].copy()  #.loc[:,:]
temp_df['dccc_target'] = temp_df['code'].isin(dccc_target)
temp_df.sort_values(by='sum', ascending=False, inplace=True)
temp_df = temp_df[['district', 'party', 'first', 'last', 'demlead_house2016',
         'demlead_pres2016', 'sum', 'dccc_target', 'pvi_2016']]

#Build Table
interactive_table(temp_df, width=800, height=500)


# # Senate Seats at Risk
# 
# Next, I look at the Senate seats at risk for each party.  
# 
# Below is a plot similar plot to the one above, except it only shows the Senators up for re-election in 2018.  This plot is also slightly different, as I am using the Democratic Lead in the 2012 Senate election for each candidate on the y-axis.  This might not be the best indicator of the state's support for the candidate as a lot has changed since 2012, but it's the best one I can think of.
# 
# This plot makes it clear that Senate Democrats have a tough 2018 coming up, as 9 senators are up for re-election in states that Trump won.  

# In[8]:


# Note, demlead_lastsenate refers to the 2010 election for 
# senators elected in 2016
senate_df = pd.read_csv('senate_data.csv', sep=',', encoding='utf-8') 
senate_df['demlead_pres2016'] = ((senate_df['clinton_2016'] - senate_df['trump_2016'])/senate_df['prestotal_2016'])*100
senate_df['demlead_lastsenate'] = senate_df['lastelect_dem'] - senate_df['lastelect_rep']
senate_df['sum'] = senate_df['demlead_lastsenate'] + senate_df['demlead_pres2016']

senate_df = senate_df.round(2)


# In[49]:


plot_df = senate_df[senate_df['class'] == 2012]

# p.circle(x='demlead_pres2016', y='demlead_lastsenate', size=8, 
#         color='red', source=source, fill_alpha=0.4)

tooltip = '''
    '<b>Class:</b> '+ d[keys.class] + 
    '<br><b>State:</b> '+ d[keys.state] + 
    '<br><b>Name:</b> ' + d[keys.first] + ' ' + d[keys.last] + 
    '<br><b>President:</b> ' + d[keys.demlead_pres2016] + "%" + 
    '<br><b>Last Senate:</b> ' + d[keys.demlead_lastsenate] +"%" 
    ''' 

settings = {"x_label": "Democratic Lead, President 2016 (%)", 
            "y_label": "Democratic Lead, Last Senate (%)",
            "x": 'demlead_pres2016' ,
            "y": 'demlead_lastsenate', 
            "tooltip": tooltip}

interactive_scatter(plot_df, settings=settings)


# ## Close Senate Seats for Democrats
# 
# The Senate is a little different because voting happens every six years.  I order this by the sum of the Democratic lead in the 2016 presidential election and the Senator's lead in their last election.  A district could shift quite a bit during a Senator's term, so this sum should be taken with a grain of salt.  
# 
# These data make it clear that Senate Democrats have a very difficult election ahead in 2018. 

# In[60]:


#Build Dataframe
temp_df = senate_df[(senate_df['party'] == 'Democratic') &
                    (senate_df['class'] == 2012)
                  ].sort_values(by='sum', ascending=True) 
temp_df = temp_df[['state', 'class', 'party', 'first', 'last', 'demlead_lastsenate',
                  'demlead_pres2016', 'sum', 'pvi_2016']]

#Build Table
interactive_table(temp_df, width=800, height=500)


# ## Close Senate Seat(s) for Republicans
# 
# Relative to the Democrats, Senate Republicans have an easier 2018.  Dean Heller's seat in Nevada is the only one in a state that was won by Clinton in 2016, and many of the other seats look pretty safe.

# In[61]:


temp_df = senate_df[(senate_df['party'] == 'Republican') &
                    (senate_df['class'] == 2012)
                  ].sort_values(by='sum', ascending=False) #by=  'demlead_pres2016'

temp_df = temp_df[['state', 'class', 'party', 'first', 'last', 'demlead_lastsenate',
                  'demlead_pres2016', 'sum','pvi_2016']]

#Build Table
interactive_table(temp_df, width=800, height=275)


# # Code for the Visualizations
# 
# Below are the two functions for the interactive scatter and tables.  I couldn't find a Python library with interactive features that I liked, so I decided to build the basics of my own.  The scatter is built with d3.js, and then embedded in an iframe along with the data.  I use plain javascript to add interactivity to the tables and then embed it in an iframe as well.  
# 
# I find this works pretty well, as I have complete control over the visualization, and can copy and paste the iframe wherever it's needed, data included.   

# In[47]:


def interactive_scatter(df, settings):
    srcdoc = r'''
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset="utf-8">
    <title>Zoom + Pan</title>
    <style>
    
    body {
      position: relative;
      width: 650px; /*960px*/
    }

    svg {font: 10px sans-serif;}

    rect {fill: #e5e5e5; }

    .label {
        font-size: 12px;
        /*stroke: #ddd;*/
        fill: #555;
    }

    .dot {
      /*stroke: #aaa;*/
      /*stroke: red;*/
      /*border: 1;*/
    }
    .dot:hover {fill-opacity: 0.4;}

    .axis path,
    .axis line {
      stroke: #fff; /*black;*/
      fill: none; 
      stroke-width: 1px;
    }
    
    #main-tick{
      fill: none;
      stroke: #777;  
      stroke-width: 1px;
      opacity: 1;
    }

    .buttons {
      position: absolute;
      right: 30px;
      top: 30px;
    }
    button {
      font: 16px sans-serif;
      display: block;
      border-radius: 0px;
      width: 25px;
      /*outline: none;*/
      /*outline:0;*/
      background-color: white;
      border: none;
    }
    
    button:hover {
      background-image:none;
      background-color:#d5d5d5;
    }
    
    button:focus{ 
    /*outline-color: #ddd;
    outline:none;
    outline: 1;*/
    outline-color: #b5b5b5;
    }
    
    div.tooltip {
      position: absolute;
      padding: 5px;
      font: 12px sans-serif;
      background: white;
      border: 0px;
      border-radius: 0px;
      pointer-events:none;
    }

    </style>
    </head>
    <body>
    
    <div class="buttons">
      <button data-zoom="+1">+</button>
      <button data-zoom="-1">-</button>
    </div>
    <script src="//d3js.org/d3.v3.min.js"></script>
    <script>

    var margin = {top: 20, right: 20, bottom: 40, left: 40},
        width = 650 - margin.left - margin.right,
        height = 515 - margin.top - margin.bottom;

    var keys = ||keys||;
    var data = ||datainsert||;
    
    var xName = "||x||";
    var yName = "||y||";
    var xLabel = "||x_label||";
    var yLabel = "||y_label||";
    
    var min_x = d3.min(data, function(d) { return +d[keys[xName]]; });
    var max_x = d3.max(data, function(d) { return +d[keys[xName]]; });
    var min_y = d3.min(data, function(d) { return +d[keys[yName]]; });
    var max_y = d3.max(data, function(d) { return +d[keys[yName]]; });
    var max = d3.max([min_x, min_y, max_x, max_y].map(Math.abs));

    var x = d3.scale.linear()
            .domain([-1.2*max, 1.2*max])
            .range([ 0, width ]);

    var y = d3.scale.linear()
            //.domain([min_y - Math.abs(0.2*min_y), max_y + Math.abs(0.2*max_y)])
            .domain([-1.2*max, 1.2*max])
            .range([ height, 0 ]);

    var xAxis = d3.svg.axis()
        .scale(x)
        .orient("bottom")
        .ticks(6)  //5
        .tickSize(-height);

    var yAxis = d3.svg.axis()
        .scale(y)
        .orient("left")
        .ticks(6)  //5
        .tickSize(-width);  //-width

    var zoom = d3.behavior.zoom()
        .x(x)
        .y(y)
        .scaleExtent([1, 10])
        .center([width / 2, height / 2])
        .size([width, height])
        .on("zoom", zoomed);

    var svg = d3.select("body").append("svg")
        .attr("width", width + margin.left + margin.right)
        .attr("height", height + margin.top + margin.bottom)
      .append("g")
        .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
        .call(zoom);

    //http://stackoverflow.com/questions/28723551
    //Create clip, then apply it to each dot
    var clip = svg.append("defs").append("svg:clipPath")
        .attr("id", "clip")
        .append("svg:rect")
        .attr("id", "clip-rect")
        .attr("x", "0")  //
        .attr("y", "0")  //
        .attr('width', width)
        .attr('height', height);

    svg.append("rect")
        .attr("width", width)
        .attr("height", height);

    svg.append("g")
        .attr("class", "x axis")
        .attr("transform", "translate(0," + height + ")")
        .call(xAxis);

    svg.append("g")
        .attr("class", "y axis")
        .call(yAxis);

    d3.selectAll('g.tick')
      .filter(function(d){ return d==0;} )
      .select('line') 
      .attr('id', 'main-tick');

    // Tooltips
    var div = d3.select("body")
        .append("div")  
        .attr("class", "tooltip")
        .style("opacity", 0); 

    svg.selectAll(".dot")
      .data(data)
    .enter().append("circle")
      .attr("class", "dot")
      .attr("clip-path", "url(#clip)")  //add the clip to each dot
      .attr("r", 4.0) //3.5  4.5 3*zoom.scale()
      .attr("cx", function(d) { return x(+d[keys[xName]]); })
      .attr("cy", function(d) { return y(+d[keys[yName]]); })
      .style("fill", 'red' ) 
      .attr('fill-opacity', 0.6)
      .on("mouseover", function(d) { drawTooltip(d); })
      .on("mouseout", function() {
        div.style("opacity", 0);
      });

    svg.append("text")
        .attr("class", "x label")
        .attr("text-anchor", "middle")
        .attr("x", width/2)
        .attr("y", height + 30)
        .text("||x_label||");

    svg.append("text")
        .attr("class", "y label")
        .attr("text-anchor", "middle")
        .attr("x", -height/2)
        .attr("y", -30) //-30
        .attr("transform", "rotate(-90)")
        .text("||y_label||");

    d3.selectAll("button[data-zoom]")
        .on("click", clicked);

    function zoomed() {
      svg.select(".x.axis").call(xAxis);
      svg.select(".y.axis").call(yAxis);

      //http://stackoverflow.com/questions/37573228
      svg.selectAll(".dot")
        //.attr("r", 3*zoom.scale())
        .attr("cx", function (d) {
            return x(+d[keys[xName]]);
        })
        .attr("cy", function (d) {
            return y(+d[keys[yName]]);
        });

        d3.selectAll('g.tick')
          .filter(function(d){ return d==0;} )
          .select('line') //grab the tick line
          .attr('id', 'main-tick');
    }

    function clicked() {
      svg.call(zoom.event); // https://github.com/mbostock/d3/issues/2387

      // Record the coordinates (in data space) of the center (in screen space).
      var center0 = zoom.center(), translate0 = zoom.translate(), coordinates0 = coordinates(center0);
      zoom.scale(zoom.scale() * Math.pow(2, +this.getAttribute("data-zoom")));

      // Translate back to the center.
      var center1 = point(coordinates0);
      zoom.translate([translate0[0] + center0[0] - center1[0], translate0[1] + center0[1] - center1[1]]);

      svg.transition().duration(750).call(zoom.event);

    }

    function coordinates(point) {
      var scale = zoom.scale(), translate = zoom.translate();
      return [(point[0] - translate[0]) / scale, (point[1] - translate[1]) / scale];
    }

    function point(coordinates) {
      var scale = zoom.scale(), translate = zoom.translate();
      return [coordinates[0] * scale + translate[0], coordinates[1] * scale + translate[1]];
    }

    function drawTooltip(d) {
        div.style("opacity", 1.0);
        div.html(||tooltip||)
            .style("left", (d3.event.pageX) + "px")
            .style("top", (d3.event.pageY ) + "px");
    }

    </script>
    </body>
    </html>
    ''' 

    width=960
    height=600

    srcdoc = srcdoc.replace('||datainsert||', df.to_json(orient="values"))
    
    key_list = list(df)
    key_dict = {i: key_list.index(i) for i in key_list}
    srcdoc = srcdoc.replace('||keys||', json.dumps(key_dict) )
    
    for s in settings.keys():
        srcdoc = srcdoc.replace('||{0}||'.format(s), str(settings[s]))
    
    srcdoc = srcdoc.replace('"', '&quot;')

    embed = HTML('<iframe srcdoc="{0}" '
                 'style="width: {1}px; height: {2}px; display:block; width: 100%; margin: 25px auto; '
                 'border: none"></iframe>'.format(srcdoc, width, height))
    return embed



# In[57]:


def interactive_table(df, width, height):
    srcdoc = r'''
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset="utf-8">
    <style>

    /*
    body {
        width: 800px;
    }*/

    table {
        font-size: 12px;
        border-collapse: collapse;
        border-top: 1px solid #ddd;
        border-right: 1px solid #ddd;
    }

    th {
        padding: 10px;
        cursor: pointer;
        background-color: #f2f2f2;
    }

    th, td {
        text-align: left;
        border-bottom: 1px solid #ddd;
        border-left: 1px solid #ddd;
    }

    td {
        padding: 5px 8px;
    }

    tr:nth-child(even) {
      background-color: #f9f9f9;
    }

    tr:hover {
      background-color: #F0F8FF; /*#f9f9f9;*/
    }

    </style>
    </head>

    <body>

    <div id ="tableInsert"></div>

    <script>
    //http://stackoverflow.com/questions/14267781/sorting-html-table-with-javascript

    function sortTable(table, col, reverse) {
        var tb = table.tBodies[0], 
            tr = Array.prototype.slice.call(tb.rows, 0), // put rows into array
            i;

        reverse = -((+reverse) || -1);
        tr = tr.sort(function (a, b) { 
            var first = a.cells[col].textContent.trim();
            var second = b.cells[col].textContent.trim();

            if (isNumeric(first) && isNumeric(second)) {        
                return reverse * (Number(first) - Number(second));
            } else {
                return reverse * first.localeCompare(second);
            };
        });
        for(i = 0; i < tr.length; ++i) {  // append each row in order
            tb.appendChild(tr[i]);
        }
    }

    //http://stackoverflow.com/questions/18082
    function isNumeric(n) {
      return !isNaN(parseFloat(n)) && isFinite(n);
    }

    function makeSortable(table) {
        var th = table.tHead, i;
        th && (th = th.rows[0]) && (th = th.cells);
        if (th) i = th.length;
        else return; // if no `<thead>` then do nothing
        while (--i >= 0) (function (i) {
            var dir = 1;
            th[i].addEventListener('click', function () {sortTable(table, i, (dir = 1 - dir))});
        }(i));
    }

    function makeAllSortable(parent) {
        parent = parent || document.body;
        var t = parent.getElementsByTagName('table'), i = t.length;
        while (--i >= 0) makeSortable(t[i]);
    }

    function addTable() {
        var tableDiv = document.getElementById("tableInsert")
        var table = document.createElement('table')
        var tableHead = document.createElement('thead')
        var tableBody = document.createElement('tbody')

        table.appendChild(tableHead)
        table.appendChild(tableBody);

        var heading = ||headinginsert||;
        var data = ||datainsert||;

        //TABLE HEAD
        var tr = document.createElement('tr');
        tableHead.appendChild(tr);
        for (i = 0; i < heading.length; i++) {
            var th = document.createElement('th')
            //th.width = '75';
            th.appendChild(document.createTextNode(heading[i]));
            tr.appendChild(th);
        }

        //TABLE ROWS
        for (i = 0; i < data.length; i++) {
            var tr = document.createElement('TR');
            for (j = 0; j < data[i].length; j++) {
                var td = document.createElement('TD')
                td.appendChild(document.createTextNode(data[i][j]));
                tr.appendChild(td)
            }
            tableBody.appendChild(tr);
        }  
        tableDiv.appendChild(table)

    }

    window.onload = function () {addTable(); makeAllSortable(); };
    // use callback makeAllSortable(); at end?
    // window.onload = function () {addTable(makeAllSortable);  };  

    </script>

    </body>
    </html>'''

    srcdoc = srcdoc.replace('||headinginsert||', json.dumps(list(df)))
    srcdoc = srcdoc.replace('||datainsert||', df.to_json(orient="values"))
    srcdoc = srcdoc.replace('"', '&quot;')

    html = '''<iframe srcdoc="{0}" style="width: {1}px; height: {2}px; 
            display:block; margin: 25px; border: none"></iframe>
            '''.format(srcdoc, width, height)  #width: 100%;  margin: 25px auto;

    embed = HTML(html)
    return embed