from erddapy.multiple_server_search import advanced_search_servers
min_time = "2017-07-01T00:00:00Z"
max_time = "2017-09-01T00:00:00Z"
min_lon, max_lon = -127, -123.75
min_lat, max_lat = 43, 48
standard_name = "sea_water_practical_salinity"
kw = {
"standard_name": standard_name,
"min_lon": min_lon,
"max_lon": max_lon,
"min_lat": min_lat,
"max_lat": max_lat,
"min_time": min_time,
"max_time": max_time,
"cdm_data_type": "timeseries", # let's exclude AUV's tracks
}
servers = {
"ooi": "https://erddap.dataexplorer.oceanobservatories.org/erddap/",
"ioos": "https://erddap.sensors.ioos.us/erddap/",
}
df = advanced_search_servers(servers_list=servers.values(), **kw)
df.head()
Title | Institution | Dataset ID | Server url | |
---|---|---|---|---|
0 | Coastal Endurance: Oregon Inshore Surface Moor... | Ocean Observatories Initiative (OOI) | ooi-ce01issm-rid16-02-flortd000 | https://erddap.dataexplorer.oceanobservatories... |
1 | Coastal Endurance: Oregon Inshore Surface Moor... | Ocean Observatories Initiative (OOI) | ooi-ce01issm-rid16-03-ctdbpc000 | https://erddap.dataexplorer.oceanobservatories... |
2 | Coastal Endurance: Oregon Inshore Surface Moor... | Ocean Observatories Initiative (OOI) | ooi-ce01issm-rid16-03-dostad000 | https://erddap.dataexplorer.oceanobservatories... |
3 | Coastal Endurance: Oregon Inshore Surface Moor... | Ocean Observatories Initiative (OOI) | ooi-ce01issm-rid16-07-nutnrb000 | https://erddap.dataexplorer.oceanobservatories... |
4 | Coastal Endurance: Oregon Inshore Surface Moor... | Ocean Observatories Initiative (OOI) | ooi-ce01issm-rid16-06-phsend000 | https://erddap.dataexplorer.oceanobservatories... |
Now that we know all the dataset_ids we can loop over them and fetch the position data (longitude and latitude).
from pathlib import Path
import pickle
import numpy as np
from urllib.error import HTTPError
from tqdm.notebook import tqdm
import pandas as pd
from erddapy import ERDDAP
path = Path("positions.p")
kw = {
"longitude>=": min_lon,
"longitude<=": max_lon,
"latitude>=": min_lat,
"latitude<=": max_lat,
"time>=": min_time,
"time<=": max_time,
}
def request_positions(server, dataset_id):
e = ERDDAP(server, protocol="tabledap")
download_url = e.get_download_url(
dataset_id=dataset_id,
response="csvp",
variables=["longitude", "latitude"],
constraints=kw,
distinct=True,
)
df = pd.read_csv(download_url)
pos = df.values.squeeze().tolist()
return pos
if not path.exists():
# this step is slow b/c we are hitting two servers multiple times to request the lon, lat of the stations
positions = []
for idx, row in tqdm(df.iterrows()):
try:
pos = request_positions(row["Server url"], row["Dataset ID"])
except HTTPError as e:
print(f"Failed downloading {row['Dataset ID']} from {row['Server url']}\n{e}")
pos = [np.NaN, np.NaN]
positions.append(pos)
with open(path, "wb") as f:
positions = pickle.dump(f)
else:
with open(path, "rb") as f:
positions = pickle.load(f)
Split the DataFrame into two based on the servers we requested and add the position data to them.
df.loc[:, "lon"] = np.array(positions)[:, 0]
df.loc[:, "lat"] = np.array(positions)[:, 1]
df.dropna(inplace=True)
df_ioos = df.loc[df["Server url"] == servers["ioos"]]
df_ooi = df.loc[df["Server url"] == servers["ooi"]]
from ipyleaflet import AwesomeIcon, Marker, Map, LegendControl, FullScreenControl, Popup, Polyline, Rectangle
from ipywidgets import HTML
colors = {
"IOOS": "blue",
"OOI": "orange",
}
legend = LegendControl(
colors,
name="Dataset locations",
position="bottomright",
)
def make_popup(row):
classes = "table table-striped table-hover table-condensed table-responsive"
return pd.DataFrame(row[["Institution", "Dataset ID"]]).to_html(
classes=classes
)
m = Map(center=(min_lat, max_lon), zoom=4)
m.add_control(FullScreenControl())
m.add_control(legend)
rectangle = Rectangle(
bounds=((min_lat, min_lon), (max_lat, max_lon)),
color="red",
fill=False,
)
m.add_layer(rectangle)
# IOOS
for k, row in df_ioos.iterrows():
marker = Marker(
icon=AwesomeIcon(name="life-ring", marker_color=colors["IOOS"]),
location=(row["lat"], row["lon"]),
)
msg = HTML()
msg.value = make_popup(row)
marker.popup = msg
m.add_layer(marker)
# OOI
for k, row in df_ooi.iterrows():
marker = Marker(
icon=AwesomeIcon(name="life-ring", marker_color=colors["OOI"]),
location=(row["lat"], row["lon"]),
)
msg = HTML()
msg.value = make_popup(row)
marker.popup = msg
m.add_layer(marker)
m
Map(center=[43, -123.75], controls=(ZoomControl(options=['position', 'zoom_in_text', 'zoom_in_title', 'zoom_ou…
Now we can try to find the two stations closest to each other and compare their salinity data.
import seawater as sw
# brute force but OK for this small data set
dist = 1e3
for k_i, (lon_i, lat_i) in df_ioos[["lon", "lat"]].iterrows():
for k_o, (lon_o, lat_o) in df_ooi[["lon", "lat"]].iterrows():
lats = lat_i, lat_o
lons = lon_i, lon_o
new_dist, angle = sw.dist(lats, lons)
if new_dist < dist:
dist = new_dist
ki, ko = k_i, k_o
print(f"The stations closest ({dist.squeeze():.2f} km) to each other are:")
The stations closest (0.65 km) to each other are:
df.loc[ki]
Title 46098 - Moored Buoy Institution Observing System Monitoring Center (OSMC) Dataset ID osmc_46098 Server url https://erddap.sensors.ioos.us/erddap/ lon -124.95 lat 44.383335 Name: 65, dtype: object
df.loc[ko]
Title Coastal Endurance: Oregon Offshore Surface Moo... Institution Ocean Observatories Initiative (OOI) Dataset ID ooi-ce04ossm-rid27-02-flortd000 Server url https://erddap.dataexplorer.oceanobservatories... lon -124.94508 lat 44.37868 Name: 15, dtype: object
def download_salinity(server, dataset_id):
print(f"Downloading {dataset_id=}.")
e = ERDDAP(server, protocol="tabledap")
e.dataset_id = dataset_id
e.constraints = {"time>=": min_time,"time<=": max_time}
e.response = "csv"
e.variables = [
"time",
e.get_var_by_attr(dataset_id=dataset_id, standard_name=standard_name)[0]
]
col = "time (UTC)"
return e.to_pandas(parse_dates=[col], index_col=col)
dataset_id = df.loc[ki]["Dataset ID"]
sal_ioos = download_salinity(servers["ioos"], dataset_id)
Downloading dataset_id='osmc_46098'.
dataset_id = df.loc[ko]["Dataset ID"]
sal_ooi = download_salinity(servers["ooi"], dataset_id)
Downloading dataset_id='ooi-ce04ossm-rid27-02-flortd000'.
%matplotlib notebook
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
fig, ax = plt.subplots(figsize=(9, 3.75))
ax.plot(sal_ioos.index, sal_ioos, label="IOOS")
ax.plot(sal_ooi.index, sal_ooi, label="OOI", alpha=0.75)
fig.autofmt_xdate()
fig.legend(loc=[0.5, 0.25]);