from dask_kubernetes import KubeCluster
n_workers = 10
cluster = KubeCluster.from_yaml(
'/home/jovyan/myworker.yml',
n_workers=n_workers
)
cluster
Failed to display Jupyter Widget of type VBox
.
If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean that the widgets JavaScript is still loading. If this message persists, it likely means that the widgets JavaScript library is either not installed or not enabled. See the Jupyter Widgets Documentation for setup instructions.
If you're reading this message in another frontend (for example, a static rendering on GitHub or NBViewer), it may mean that your frontend doesn't currently support widgets.
from dask.distributed import Client
client = Client(cluster)
client
Client
|
Cluster
|
import gcsfs
import xarray as xr
fs = gcsfs.GCSFileSystem(project='pangeo-181919', access='read_only')
gcsmap = gcsfs.mapping.GCSMap(
'pangeo-data/rsignell/ocean_his_tide_zeta',
gcs=fs, check=False, create=False
)
ds = xr.open_zarr(gcsmap, decode_times=False)
ds
<xarray.Dataset> Dimensions: (eta_rho: 324, ocean_time: 1441, xi_rho: 1542) Coordinates: lat_rho (eta_rho, xi_rho) float64 dask.array<shape=(324, 1542), chunksize=(81, 771)> lon_rho (eta_rho, xi_rho) float64 dask.array<shape=(324, 1542), chunksize=(81, 771)> * ocean_time (ocean_time) float64 0.0 1.8e+03 3.6e+03 5.4e+03 7.2e+03 ... Dimensions without coordinates: eta_rho, xi_rho Data variables: zeta (ocean_time, eta_rho, xi_rho) float32 dask.array<shape=(1441, 324, 1542), chunksize=(91, 41, 193)> Attributes: CPP_options: GSB, ADD_FSOBC, ADD_M2OBC, ANA_BSFLUX, ANA_BTFLUX, ANA... Conventions: CF-1.4 NCO: 4.7.3 NLM_LBC: \nEDGE: WEST SOUTH EAST NORTH \nzeta: Cha C... ana_file: ROMS/Functionals/ana_btflux.h, ROMS/Functionals/ana_fs... code_dir: /cxfs/projects/usgs/hazards/cmgp/woodshole/aaretxabale... compiler_command: /opt/intel/impi/5.0.1.035/intel64/bin/mpif90 compiler_flags: -heap-arrays -fp-model precise -ip -O3 -xW -free compiler_system: ifort cpu: x86_64 file: ocean_his_gsb_tides_55nb.nc format: netCDF-3 64bit offset file frc_file_01: ../forcings/tide_forc_GSB_55.nc grd_file: ../grids/GSB_55nb.nc header_dir: /cxfs/projects/usgs/hazards/cmgp/woodshole/aaretxabale... header_file: gsb.h his_file: ocean_his_gsb_tides_55nb.nc history: Mon Mar 12 09:36:34 2018: ncks -O -v ocean_time,zeta,l... os: Linux rst_file: ocean_rst.nc script_file: svn_rev: svn_url: https:://myroms.org/svn/src tiling: 036x010 title: Great south Bay type: ROMS/TOMS history file var_info: varinfo.dat
ds['zeta'].nbytes / 1e9
2.879740512
%%time
isub = 2
zeta = ds['zeta'][:,::isub,::isub].load()
CPU times: user 7.43 s, sys: 2.58 s, total: 10 s Wall time: 17.1 s
zeta
<xarray.DataArray 'zeta' (ocean_time: 1441, eta_rho: 162, xi_rho: 771)> array([[[ 0.000000e+00, 0.000000e+00, ..., 0.000000e+00, 0.000000e+00], [ 0.000000e+00, 0.000000e+00, ..., 0.000000e+00, 0.000000e+00], ..., [ 0.000000e+00, 0.000000e+00, ..., 0.000000e+00, 0.000000e+00], [ 0.000000e+00, 0.000000e+00, ..., 0.000000e+00, 0.000000e+00]], [[ 2.843082e-03, 2.836426e-03, ..., 1.376264e-03, 1.358796e-03], [ 2.846219e-03, 2.842475e-03, ..., 1.373103e-03, 1.356077e-03], ..., [ 0.000000e+00, -4.496790e+00, ..., -1.292400e+01, -1.292400e+01], [ 0.000000e+00, -4.496790e+00, ..., -1.292400e+01, -1.292400e+01]], ..., [[ 4.035858e-01, 4.035637e-01, ..., 2.775990e-01, 2.774480e-01], [ 4.036383e-01, 4.033978e-01, ..., 2.773239e-01, 2.771884e-01], ..., [ 0.000000e+00, -4.496790e+00, ..., -1.292400e+01, -1.292400e+01], [ 0.000000e+00, -4.496790e+00, ..., -1.292400e+01, -1.292400e+01]], [[ 3.238091e-01, 3.237954e-01, ..., 2.161487e-01, 2.160328e-01], [ 3.238123e-01, 3.234939e-01, ..., 2.157554e-01, 2.156571e-01], ..., [ 0.000000e+00, -4.496790e+00, ..., -1.292400e+01, -1.292400e+01], [ 0.000000e+00, -4.496790e+00, ..., -1.292400e+01, -1.292400e+01]]], dtype=float32) Coordinates: lat_rho (eta_rho, xi_rho) float64 40.21 40.21 40.21 40.21 40.21 ... lon_rho (eta_rho, xi_rho) float64 -73.79 -73.79 -73.78 -73.78 -73.78 ... * ocean_time (ocean_time) float64 0.0 1.8e+03 3.6e+03 5.4e+03 7.2e+03 ... Dimensions without coordinates: eta_rho, xi_rho Attributes: field: free-surface, scalar, series grid: grid location: face long_name: free-surface time: ocean_time units: meter
#client.get_versions(check=True)
stacked = zeta.stack(points=['eta_rho', 'xi_rho'])
stacked
<xarray.DataArray 'zeta' (ocean_time: 1441, points: 124902)> array([[ 0.000000e+00, 0.000000e+00, 0.000000e+00, ..., 0.000000e+00, 0.000000e+00, 0.000000e+00], [ 2.843082e-03, 2.836426e-03, 2.823583e-03, ..., -1.292400e+01, -1.292400e+01, -1.292400e+01], [ 1.505562e-03, 1.505880e-03, 1.502692e-03, ..., -1.292400e+01, -1.292400e+01, -1.292400e+01], ..., [ 4.532547e-01, 4.532280e-01, 4.531393e-01, ..., -1.292400e+01, -1.292400e+01, -1.292400e+01], [ 4.035858e-01, 4.035637e-01, 4.034485e-01, ..., -1.292400e+01, -1.292400e+01, -1.292400e+01], [ 3.238091e-01, 3.237954e-01, 3.236695e-01, ..., -1.292400e+01, -1.292400e+01, -1.292400e+01]], dtype=float32) Coordinates: lat_rho (points) float64 40.21 40.21 40.21 40.21 40.21 40.21 40.21 ... lon_rho (points) float64 -73.79 -73.79 -73.78 -73.78 -73.78 -73.78 ... * ocean_time (ocean_time) float64 0.0 1.8e+03 3.6e+03 5.4e+03 7.2e+03 ... * points (points) MultiIndex - eta_rho (points) int64 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ... - xi_rho (points) int64 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 ... Attributes: field: free-surface, scalar, series grid: grid location: face long_name: free-surface time: ocean_time units: meter
groups = stacked.groupby('points')
from utide import solve
time = ds['ocean_time'].values / (3600*24)
lat = ds['lat_rho'].mean().values
kw = {
'trend': False,
'nodal': False,
'lat': lat,
'verbose': False,
'Rayleigh_min': 0.95,
'method': 'ols',
'conf_int': 'linear',
}
psolve = lambda u: solve(time, u, None, **kw)
%%time
tides = []
for g in groups:
u = g[1].values
res = client.submit(psolve, u)
tides.append(res)
CPU times: user 6min 40s, sys: 33.1 s, total: 7min 13s Wall time: 6min 45s
%matplotlib inline
zeta[0].plot();
tides[0].result()
{'A': array([6.04506815e-01, 1.43026754e-01, 1.04772316e-01, 8.50216093e-02, 6.01551358e-02, 1.39807474e-02, 1.12658421e-02, 6.34471918e-03, 5.66794317e-03, 4.50432438e-03, 4.12241279e-03, 3.75482085e-03, 3.50733153e-03, 3.27444395e-03, 3.16475951e-03, 2.92814698e-03, 2.92308193e-03, 2.72557681e-03, 2.16404393e-03, 2.00910647e-03, 2.00012035e-03, 1.71861859e-03, 1.16177493e-03, 1.14196886e-03, 7.34160722e-04, 7.30638929e-04, 7.29099218e-04, 6.48429645e-04, 4.93227153e-04]), 'A_ci': array([4.54164709e-03, 4.54074074e-03, 4.54122336e-03, 1.37100765e-03, 1.36991119e-03, 4.54373360e-03, 2.75668653e-04, 7.37937959e-04, 1.36955798e-03, 1.37084650e-03, 1.37098525e-03, 1.36943781e-03, 1.37096401e-03, 1.37105397e-03, 1.01649303e-03, 1.01650672e-03, 1.01652748e-03, 1.01659976e-03, 7.37825561e-04, 7.37891958e-04, 1.00862817e-03, 7.37921444e-04, 2.87203900e-04, 2.87209282e-04, 8.49705099e-05, 2.75733077e-04, 2.75689334e-04, 2.75730137e-04, 3.54087853e-05]), 'aux': {'frq': array([0.0805114 , 0.07899925, 0.08333333, 0.04178075, 0.03873065, 0.08507364, 0.2415342 , 0.1610228 , 0.0372185 , 0.04634299, 0.04483084, 0.0432929 , 0.0402686 , 0.03570635, 0.11924205, 0.1207671 , 0.12511408, 0.12229215, 0.15951065, 0.16384473, 0.00282193, 0.16666667, 0.20280355, 0.20844741, 0.28331495, 0.24435613, 0.24002205, 0.24717807, 0.3220456 ]), 'lat': array(40.70228015), 'lind': array([ 47, 41, 56, 20, 12, 60, 105, 81, 10, 28, 27, 24, 15, 8, 67, 68, 73, 71, 78, 85, 5, 88, 95, 98, 119, 109, 102, 112, 124]), 'opt': {'RunTimeDisp': False, 'cnstit': 'auto', 'conf_int': True, 'diagnminsnr': 2, 'diagnplots': 0, 'epoch': 'python', 'equi': True, 'gwchlint': False, 'gwchnone': False, 'infer': None, 'inferaprx': 0, 'linci': True, 'lsfrqosmp': 1, 'method': 'ols', 'newopts': {'MC_n': 200, 'Rayleigh_min': 0.95, 'conf_int': 'linear', 'constit': 'auto', 'epoch': 'python', 'infer': None, 'method': 'ols', 'nodal': False, 'phase': 'Greenwich', 'robust_kw': {'weight_function': 'cauchy'}, 'trend': False, 'verbose': False, 'white': False}, 'nodesatlint': False, 'nodesatnone': False, 'nodiagn': 0, 'nodsatlint': 0, 'nodsatnone': True, 'notrend': True, 'nrlzn': 200, 'ordercnstit': None, 'prefilt': [], 'rmin': 0.95, 'runtimedisp': 'yyy', 'tunrdn': 1, 'twodim': False, 'white': False}, 'reftime': 15.0}, 'diagn': {'PE': array([8.95144627e+01, 5.01101119e+00, 2.68895725e+00, 1.77071819e+00, 8.86413231e-01, 4.78797179e-02, 3.10898144e-02, 9.86087870e-03, 7.86940290e-03, 4.96992833e-03, 4.16287875e-03, 3.45357784e-03, 3.01331440e-03, 2.62643060e-03, 2.45342188e-03, 2.10027688e-03, 2.09301713e-03, 1.81973295e-03, 1.14715724e-03, 9.88773257e-04, 9.79948079e-04, 7.23518792e-04, 3.30624328e-04, 3.19447382e-04, 1.32030144e-04, 1.30766477e-04, 1.30215916e-04, 1.02995092e-04, 5.95915609e-05]), 'SNR': array([6.80593468e+04, 3.81147693e+03, 2.04484083e+03, 1.47737597e+04, 7.40751835e+03, 3.63703263e+01, 6.41600503e+03, 2.83986209e+02, 6.57964214e+01, 4.14757078e+01, 3.47335789e+01, 2.88805829e+01, 2.51428021e+01, 2.19118053e+01, 3.72378635e+01, 3.18769955e+01, 3.17655128e+01, 2.76139789e+01, 3.30473693e+01, 2.84795087e+01, 1.51064430e+01, 2.08377532e+01, 6.28601593e+01, 6.07328591e+01, 2.86786026e+02, 2.69736690e+01, 2.68686273e+01, 2.12456192e+01, 7.45390818e+02]), 'name': array(['M2', 'N2', 'S2', 'K1', 'O1', 'ETA2', 'M6', 'M4', 'Q1', 'UPS1', 'OO1', 'J1', 'NO1', '2Q1', 'MO3', 'M3', 'SK3', 'MK3', 'MN4', 'MS4', 'MSF', 'S4', '2MK5', '2SK5', '3MK7', '2MS6', '2MN6', '2SM6', 'M8'], dtype=object)}, 'g': array([2.08187064e+02, 2.86119829e+02, 1.53645845e-01, 2.25376423e+02, 3.44678374e+02, 1.47386253e+01, 2.95949645e+02, 2.61772233e+02, 6.24180677e+01, 3.56097383e+02, 3.15249678e+02, 7.44775928e+01, 3.56069035e+02, 2.14943642e+02, 4.66314674e+01, 9.15087672e+01, 2.25967709e+02, 1.42278949e+02, 2.44022850e+01, 1.40347023e+02, 8.60883084e+01, 2.31878012e+02, 7.37842457e+01, 2.44633966e+02, 3.52916978e+02, 8.06907642e+01, 3.27428923e+02, 1.54006104e+02, 2.69016482e+02]), 'g_ci': array([ 0.43043291, 1.81960046, 2.4837086 , 0.92390172, 1.30686072, 18.60272609, 1.40216135, 6.66251725, 13.87356623, 17.44120466, 19.05507737, 20.94412967, 22.39711613, 23.98848721, 18.40358177, 19.89043814, 19.92449685, 21.36677807, 19.53668349, 21.04141201, 29.33358226, 24.59693369, 14.15860256, 14.40389617, 6.62903052, 21.61510449, 21.66418938, 24.35577303, 4.11246921]), 'mean': 0.0011312695256327603, 'nI': 0, 'nNR': 29, 'nR': 0, 'name': array(['M2', 'N2', 'S2', 'K1', 'O1', 'ETA2', 'M6', 'M4', 'Q1', 'UPS1', 'OO1', 'J1', 'NO1', '2Q1', 'MO3', 'M3', 'SK3', 'MK3', 'MN4', 'MS4', 'MSF', 'S4', '2MK5', '2SK5', '3MK7', '2MS6', '2MN6', '2SM6', 'M8'], dtype=object), 'weights': array([1., 1., 1., ..., 1., 1., 1.])}
%%time
m2amp = [f.result()['A'][0] for f in tides]
CPU times: user 8min 23s, sys: 51.7 s, total: 9min 15s Wall time: 11min 39s
import matplotlib.pyplot as plt
import numpy as np
M2 = np.array(m2amp).reshape(zeta.shape[1:])
fig, ax = plt.subplots()
ax.pcolormesh(M2);
M2.shape
(162, 771)