%matplotlib inline
from dask.distributed import Client, progress, LocalCluster
from dask_kubernetes import KubeCluster
import xarray as xr
import s3fs
cluster = KubeCluster.from_yaml('/home/jovyan/custom-worker-template.yaml')
cluster.scale(40);
cluster
client = Client(cluster)
client
# jetstream s3
# url='https://iu.jetstream-cloud.org:8080'
# fs = s3fs.S3FileSystem(client_kwargs=dict(endpoint_url=url), anon=True)
# s3map = s3fs.S3Map('rsignell/nwm/test_week', s3=fs)
# AWS s3
fs = s3fs.S3FileSystem(anon=True)
s3map = s3fs.S3Map('rsignell/adcirc_test01', s3=fs)
#s3map = s3fs.S3Map('rsignell/nwm/tiny3a', s3=fs)
ds = xr.open_zarr(s3map)
ds
var='zeta'
ds[var]
ds[var].nbytes/1.e9
ds[var].mean(dim='time')
max_var = ds[var].max(dim='time').persist()
progress(max_var)
import numpy as np
import datashader as dshade
import holoviews as hv
import geoviews as gv
import cartopy.crs as ccrs
from holoviews.operation.datashader import datashade, rasterize
from colorcet import cm_n
datashade.precompute = True
hv.extension('bokeh')
%opts Image RGB VectorField [width=800 height=600]
import pandas as pd
v = np.vstack((ds['x'], ds['y'], max_var)).T
verts = pd.DataFrame(v, columns=['x','y','z'])
points = gv.operation.project_points(gv.Points(verts, vdims=['z']))
tris = pd.DataFrame(ds['element'].values.astype('int')-1, columns=['v0','v1','v2'])
tiles = gv.WMTS('https://maps.wikimedia.org/osm-intl/{Z}/{X}/{Y}@2x.png')
trimesh = gv.TriMesh((tris, points))
adcirc_shade = tiles * datashade(trimesh, aggregator=dshade.mean('z'))
adcirc_shade
value = 'max water level'
label = '{} (m)'.format(value)
hv.TriMesh((tris, points), label=label)
tiles = gv.WMTS('https://maps.wikimedia.org/osm-intl/{Z}/{X}/{Y}@2x.png')
trimesh = gv.TriMesh((tris, points))
chesapeake_interp = tiles * datashade(trimesh, aggregator=ds.mean('z'))
chesapeake_interp
%%time
ds1d = ds[var][:,2000,2000]
ds1d.plot()