import re
import os
import glob
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import netCDF4 as nc
import seaborn as sns
import xarray as xr
import functools
from IPython.display import clear_output
import datetime
import random
from salishsea_tools import geo_tools, geo_tools, tidetools, data_tools
%matplotlib inline
plt.rcParams['image.cmap'] = 'jet'
# converts lons/lats like this "124° 01.400" to decimal format
def strLonLatToDec(lstr):
if type(lstr) is not str:
return(np.NaN)
lstr = lstr.replace(u'°',' ')
lstr = lstr.split()
return (int(lstr[0])+float(lstr[1])/60.0)
# Grab tracer data
# '/ocean/rich/home/metro/venus_adcp/PSF/bottle/PSF_2015bottle_RPedit.xls'
tracer_df = pd.read_excel('/ocean/rich/home/metro/venus_adcp/PSF/bottle/PSF_2015bottle_RPeditfinal.xls')
tracer_df = tracer_df[~pd.isnull(tracer_df["Date"])]
#tracer_df["Date"] = pd.to_datetime(tracer_df["Date"])
tracer_df = tracer_df.sort_values("Date")
tracer_df["DECIMAL_LON"] = (tracer_df["Longitude"].apply(strLonLatToDec))*(-1)
tracer_df["DECIMAL_LAT"] = (tracer_df["Latitude"].apply(strLonLatToDec))
tracer_df["STATION_LAT"] = tracer_df.groupby("Station ID")["DECIMAL_LAT"].transform(np.median)
tracer_df["STATION_LON"] = tracer_df.groupby("Station ID")["DECIMAL_LON"].transform(np.median)
# Create smaller dataframe with location of stations
station_lon_lat = tracer_df[["Station ID", "STATION_LON", "STATION_LAT"]].drop_duplicates()
station_lon_lat.columns = ["STATION", "LON", "LAT"]
# Add corresponding model points to station data frame
grid_B = nc.Dataset('/data/nsoontie/MEOPAR/NEMO-forcing/grid/bathy_meter_SalishSea2.nc')
bathy, X, Y = tidetools.get_bathy_data(grid_B)
def closest_model_point_wrapper(lon, lat):
try:
model_point = geo_tools.find_closest_model_point(lon, lat, model_lons = X, model_lats = Y, grid = 'NEMO', land_mask = bathy.mask)
return(model_point)
except:
print("ERROR" + str(lon) + " " + str(lat))
return(np.nan)
station_lon_lat["MODEL_POINT"] = station_lon_lat.apply(lambda row: closest_model_point_wrapper(row[1], row[2]) , axis = 1)
station_lon_lat["MODEL_J"] = station_lon_lat["MODEL_POINT"].apply(lambda x: x[0])
station_lon_lat["MODEL_I"] = station_lon_lat["MODEL_POINT"].apply(lambda x: x[1])
# Choose which values to add to nowcast dataframe
tracers = ["NO3"]
plot_months = ["feb", "mar", "apr", "may", "jun", "jul", "aug", "sep"]
plot_hours = np.array([12])
max_depth = 20
result_depths = xr.open_dataset('/data/nsoontie/MEOPAR/NEMO-forcing/grid/deptht_428m.nc').deptht.values
depth_indices = np.where(result_depths < max_depth)[0]
station_points = station_lon_lat["MODEL_POINT"]
station_names = station_lon_lat["STATION"]
stations = dict(zip(station_names, station_points))
nowcast_df = pd.read_pickle("/ocean/jpetrie/MEOPAR/analysis-james/nowcast_green_subset/2016_8_4_12_30.pkl")
nowcast_df["DATE"] = pd.to_datetime(nowcast_df["DATE"], infer_datetime_format=True)
#nowcast_df = data_tools.load_nowcast_station_tracers(tracers, stations, months, hours, depth_indices)
nitrate_df = tracer_df[["Station ID", "Date", "DECIMAL_LON", "DECIMAL_LAT","Depth", "Nitrate + Nitrite"]]
nitrate_df.columns = ["STATION", "DATE", "DECIMAL_LON", "DECIMAL_LAT", "DEPTH", "NO3"]
nitrate_df["DATA_TYPE"] = "Measured"
nitrate_df["DAY_OF_YEAR"] = nitrate_df["DATE"].apply(lambda x: x.dayofyear)
/home/jpetrie/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:3: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy app.launch_new_instance() /home/jpetrie/anaconda3/lib/python3.5/site-packages/ipykernel/__main__.py:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
nowcast_df["DAY_OF_YEAR"] = nowcast_df["DATE"].apply(lambda x: x.dayofyear)
nowcast_df["DATA_TYPE"] = "Nowcast"
combined = pd.concat([nitrate_df, nowcast_df.query("(DEPTH - 2.500011)**2 < 0.0001 | (DEPTH - 19.481785)**2 < 0.0001")])
combined["IDENTIFIER"] = combined["DATA_TYPE"] + ", Depth = " + combined["DEPTH"].apply(str)
combined.STATION = combined.STATION.apply(lambda x: x.strip())
combined = combined.sort_values(["STATION", "DATE"])
# Plot nowcast and measured nitrate data against day of year
# (simulation is for 2016 and data was collected in 2015)
colors = ["red","burnt orange", "dark grey blue", "cerulean",]
cmap = sns.xkcd_palette(colors)
sns.set(font_scale = 3)
fg = sns.FacetGrid(data = combined[(combined["DAY_OF_YEAR"] <= max(nowcast_df["DAY_OF_YEAR"]))], col = "STATION", col_wrap=3, hue = "IDENTIFIER", size =8, sharex = False, sharey = True, palette= cmap)
fg.map(plt.plot,"DAY_OF_YEAR", "NO3").add_legend()
#plt.xticks(rotation=45)
plt.subplots_adjust(top=0.9)
tracers = ["vosaline", "votemper"]
months = ['mar', 'apr', 'may']
hours = [12]
depth_indices = range(30)
nowcast_dir = "/results/SalishSea/nowcast-green/"
temp_salin = data_tools.load_nowcast_station_tracers(tracers, stations, months, hours, depth_indices, "grid_T.nc")
temp_salin["STATION"] = temp_salin["STATION"].apply(lambda x: x.strip())
temp_salin["DAY"] = [x.day for x in temp_salin.DATE]
temp_salin["DATE_STR"] = temp_salin["DATE"].apply(lambda temp_salin: temp_salin.strftime('%Y-%m-%d'))
Files loaded:1 Date of most recent nowcast load: 20160301 Time loading: 0:00:33.390929 Files loaded:2 Date of most recent nowcast load: 20160302 Time loading: 0:00:41.534818 Files loaded:3 Date of most recent nowcast load: 20160303 Time loading: 0:00:49.773222 Files loaded:4 Date of most recent nowcast load: 20160304 Time loading: 0:00:57.634617 Files loaded:5 Date of most recent nowcast load: 20160305 Time loading: 0:01:05.550708 Files loaded:6 Date of most recent nowcast load: 20160306 Time loading: 0:01:13.452218 Files loaded:7 Date of most recent nowcast load: 20160307 Time loading: 0:01:21.297703 Files loaded:8 Date of most recent nowcast load: 20160308 Time loading: 0:01:29.169913 Files loaded:9 Date of most recent nowcast load: 20160309 Time loading: 0:01:37.132858 Files loaded:10 Date of most recent nowcast load: 20160310 Time loading: 0:01:44.986017 Files loaded:11 Date of most recent nowcast load: 20160311 Time loading: 0:01:52.868984 Files loaded:12 Date of most recent nowcast load: 20160312 Time loading: 0:02:00.953718 Files loaded:13 Date of most recent nowcast load: 20160313 Time loading: 0:02:08.827574 Files loaded:14 Date of most recent nowcast load: 20160314 Time loading: 0:02:16.656961 Files loaded:15 Date of most recent nowcast load: 20160315 Time loading: 0:02:24.563564 Files loaded:16 Date of most recent nowcast load: 20160316 Time loading: 0:02:32.402816 Files loaded:17 Date of most recent nowcast load: 20160317 Time loading: 0:02:40.463418 Files loaded:18 Date of most recent nowcast load: 20160318 Time loading: 0:02:51.228962 Files loaded:19 Date of most recent nowcast load: 20160319 Time loading: 0:02:59.119864 Files loaded:20 Date of most recent nowcast load: 20160320 Time loading: 0:03:07.105026 Files loaded:21 Date of most recent nowcast load: 20160321 Time loading: 0:03:15.238107 Files loaded:22 Date of most recent nowcast load: 20160322 Time loading: 0:03:23.269601 Files loaded:23 Date of most recent nowcast load: 20160323 Time loading: 0:03:31.512268 Files loaded:24 Date of most recent nowcast load: 20160324 Time loading: 0:03:39.882798 Files loaded:25 Date of most recent nowcast load: 20160325 Time loading: 0:03:48.054518 Files loaded:26 Date of most recent nowcast load: 20160326 Time loading: 0:03:56.227368 Files loaded:27 Date of most recent nowcast load: 20160327 Time loading: 0:04:04.222284 Files loaded:28 Date of most recent nowcast load: 20160328 Time loading: 0:04:12.405696 Files loaded:29 Date of most recent nowcast load: 20160329 Time loading: 0:04:20.274285 Files loaded:30 Date of most recent nowcast load: 20160330 Time loading: 0:04:28.247922 Files loaded:31 Date of most recent nowcast load: 20160331 Time loading: 0:04:36.124121 Files loaded:32 Date of most recent nowcast load: 20160401 Time loading: 0:04:43.988698 Files loaded:33 Date of most recent nowcast load: 20160402 Time loading: 0:04:51.913854 Files loaded:34 Date of most recent nowcast load: 20160403 Time loading: 0:04:59.894085 Files loaded:35 Date of most recent nowcast load: 20160404 Time loading: 0:05:30.379882 Files loaded:36 Date of most recent nowcast load: 20160405 Time loading: 0:05:38.509861 Files loaded:37 Date of most recent nowcast load: 20160406 Time loading: 0:05:46.619354 Files loaded:38 Date of most recent nowcast load: 20160407 Time loading: 0:05:54.575349 Files loaded:39 Date of most recent nowcast load: 20160408 Time loading: 0:06:02.469418 Files loaded:40 Date of most recent nowcast load: 20160409 Time loading: 0:06:10.471885 Files loaded:41 Date of most recent nowcast load: 20160410 Time loading: 0:06:18.550153 Files loaded:42 Date of most recent nowcast load: 20160411 Time loading: 0:06:26.336566 Files loaded:43 Date of most recent nowcast load: 20160412 Time loading: 0:06:34.430981 Files loaded:44 Date of most recent nowcast load: 20160413 Time loading: 0:06:42.805053 Files loaded:45 Date of most recent nowcast load: 20160414 Time loading: 0:06:50.870813 Files loaded:46 Date of most recent nowcast load: 20160415 Time loading: 0:06:58.832469 Files loaded:47 Date of most recent nowcast load: 20160416 Time loading: 0:07:07.089158 Files loaded:48 Date of most recent nowcast load: 20160417 Time loading: 0:07:15.108011 Files loaded:49 Date of most recent nowcast load: 20160418 Time loading: 0:07:23.149164 Files loaded:50 Date of most recent nowcast load: 20160419 Time loading: 0:07:31.350820 Files loaded:51 Date of most recent nowcast load: 20160420 Time loading: 0:07:39.474192 Files loaded:52 Date of most recent nowcast load: 20160421 Time loading: 0:07:47.650483 Files loaded:53 Date of most recent nowcast load: 20160422 Time loading: 0:07:55.828879 Files loaded:54 Date of most recent nowcast load: 20160423 Time loading: 0:08:03.954605 Files loaded:55 Date of most recent nowcast load: 20160424 Time loading: 0:08:12.573875 Files loaded:56 Date of most recent nowcast load: 20160425 Time loading: 0:08:20.830778 Files loaded:57 Date of most recent nowcast load: 20160426 Time loading: 0:08:28.768912 Files loaded:58 Date of most recent nowcast load: 20160427 Time loading: 0:08:36.633761 Files loaded:59 Date of most recent nowcast load: 20160428 Time loading: 0:08:44.589430 Files loaded:60 Date of most recent nowcast load: 20160429 Time loading: 0:08:52.590952 Files loaded:61 Date of most recent nowcast load: 20160430 Time loading: 0:09:00.475825 Files loaded:62 Date of most recent nowcast load: 20160501 Time loading: 0:09:08.440969 Files loaded:63 Date of most recent nowcast load: 20160502 Time loading: 0:09:16.339369 Files loaded:64 Date of most recent nowcast load: 20160503 Time loading: 0:09:24.296422 Files loaded:65 Date of most recent nowcast load: 20160504 Time loading: 0:09:32.241396 Files loaded:66 Date of most recent nowcast load: 20160505 Time loading: 0:09:40.213207 Files loaded:67 Date of most recent nowcast load: 20160506 Time loading: 0:09:48.106665 Files loaded:68 Date of most recent nowcast load: 20160507 Time loading: 0:09:56.058041 Files loaded:69 Date of most recent nowcast load: 20160508 Time loading: 0:10:03.964471 Files loaded:70 Date of most recent nowcast load: 20160509 Time loading: 0:10:12.167097 Files loaded:71 Date of most recent nowcast load: 20160510 Time loading: 0:10:20.463456 Files loaded:72 Date of most recent nowcast load: 20160511 Time loading: 0:10:28.452035 Files loaded:73 Date of most recent nowcast load: 20160512 Time loading: 0:10:36.271314 Files loaded:74 Date of most recent nowcast load: 20160513 Time loading: 0:10:44.219928 Files loaded:75 Date of most recent nowcast load: 20160514 Time loading: 0:10:52.072009 Files loaded:76 Date of most recent nowcast load: 20160515 Time loading: 0:10:59.989627 Files loaded:77 Date of most recent nowcast load: 20160516 Time loading: 0:11:07.880030 Files loaded:78 Date of most recent nowcast load: 20160517 Time loading: 0:11:15.826886 Files loaded:79 Date of most recent nowcast load: 20160518 Time loading: 0:11:23.743298 Files loaded:80 Date of most recent nowcast load: 20160519 Time loading: 0:11:31.701829 Files loaded:81 Date of most recent nowcast load: 20160520 Time loading: 0:11:39.556113 Files loaded:82 Date of most recent nowcast load: 20160521 Time loading: 0:11:47.497138 Files loaded:83 Date of most recent nowcast load: 20160522 Time loading: 0:11:55.402111 Files loaded:84 Date of most recent nowcast load: 20160523 Time loading: 0:12:04.569239 Files loaded:85 Date of most recent nowcast load: 20160524 Time loading: 0:12:12.689877 Files loaded:86 Date of most recent nowcast load: 20160525 Time loading: 0:12:20.693772 Files loaded:87 Date of most recent nowcast load: 20160526 Time loading: 0:12:28.657720 Files loaded:88 Date of most recent nowcast load: 20160527 Time loading: 0:12:36.598409 Files loaded:89 Date of most recent nowcast load: 20160528 Time loading: 0:12:45.429176 Files loaded:90 Date of most recent nowcast load: 20160529 Time loading: 0:12:53.571560 Files loaded:91 Date of most recent nowcast load: 20160530 Time loading: 0:13:01.563528 Files loaded:92 Date of most recent nowcast load: 20160531 Time loading: 0:13:10.028250 Files loaded:92
value_vars = ["vosaline", "votemper"]
long_format = pd.melt(temp_salin, id_vars = list(temp_salin.columns[~temp_salin.columns.isin(value_vars)]), value_vars = value_vars)
long_format["IDENTIFIER"] = long_format["variable"] + ": " + long_format["DATE_STR"]
long_format = long_format.sort_values(["STATION", "DATE"])
sns.set(font_scale = 3)
fg = sns.FacetGrid(data=long_format.query("DAY == 15 and DEPTH < 45"), hue="IDENTIFIER",col = "STATION",col_wrap = 3, size =8, sharey = True)
fg.map(plt.plot,"value", "DEPTH" ).add_legend()
fg.set_xticklabels(rotation=-35)
plt.subplots_adjust(top=0.9)
plt.gca().invert_yaxis()
dict_keys(['VC7', 'LD1', 'NQ6', 'SN1', 'IS4', 'NQ1', 'CR4', 'IS3', 'VC3', 'BS1', 'LD2', 'PR7', 'BS9', 'LS5', 'PR8', 'LD3', 'CBE2', 'CBC3', 'BS3', 'BS7', 'CR2', 'CBC2', 'VC1', 'LS3', 'VC6', 'PR6', 'LS2', 'BS11', 'CBC1', 'NQ5', 'NQ4', 'IS2 ', 'IS2', 'BS6', 'CBW2', 'SN6', 'CR8', 'NQ3', 'SN3', ' CR2'])