Please follow these steps to running this script
#install required libs
!pip install arm_pyart cartopy adjustText
import os
from glob import glob
from datetime import datetime, timedelta
import tempfile #used to create temporary folders to store data
import zipfile #used to extract tar files
import urllib
import pandas
from matplotlib import pyplot as plt
import cartopy
import cartopy.crs as ccrs # A toolkit for map projections
import numpy as np
from tqdm import tqdm
from adjustText import adjust_text
from IPython.display import display, FileLink
from PIL import Image
import pyart
import warnings
warnings.filterwarnings('ignore')
warnings.simplefilter('ignore')
Requirement already satisfied: arm_pyart in /usr/local/lib/python3.10/dist-packages (1.17.0) Requirement already satisfied: cartopy in /usr/local/lib/python3.10/dist-packages (0.22.0) Requirement already satisfied: adjustText in /usr/local/lib/python3.10/dist-packages (1.0.4) Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (1.25.2) Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (1.11.4) Requirement already satisfied: netCDF4 in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (1.6.5) Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (3.7.1) Requirement already satisfied: pooch in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (1.8.1) Requirement already satisfied: cftime in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (1.6.3) Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (2024.2.0) Requirement already satisfied: s3fs in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (2024.2.0) Requirement already satisfied: open-radar-data in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (0.0.9) Requirement already satisfied: xradar in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (0.4.3) Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (1.5.3) Requirement already satisfied: xarray!=0.21.0 in /usr/local/lib/python3.10/dist-packages (from arm_pyart) (2024.2.0) Requirement already satisfied: shapely>=1.7 in /usr/local/lib/python3.10/dist-packages (from cartopy) (2.0.3) Requirement already satisfied: packaging>=20 in /usr/local/lib/python3.10/dist-packages (from cartopy) (23.2) Requirement already satisfied: pyshp>=2.1 in /usr/local/lib/python3.10/dist-packages (from cartopy) (2.3.1) Requirement already satisfied: pyproj>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from cartopy) (3.6.1) Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (1.2.0) Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (0.12.1) Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (4.49.0) Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (1.4.5) Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (9.4.0) Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (3.1.1) Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->arm_pyart) (2.8.2) Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from pyproj>=3.1.0->cartopy) (2024.2.2) Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->arm_pyart) (2023.4) Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch->arm_pyart) (4.2.0) Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from pooch->arm_pyart) (2.31.0) Requirement already satisfied: aiobotocore<3.0.0,>=2.5.4 in /usr/local/lib/python3.10/dist-packages (from s3fs->arm_pyart) (2.12.1) Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.10/dist-packages (from s3fs->arm_pyart) (3.9.3) Requirement already satisfied: cmweather in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (0.3.2) Requirement already satisfied: dask in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (2023.8.1) Requirement already satisfied: h5netcdf in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (1.3.0) Requirement already satisfied: h5py in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (3.9.0) Requirement already satisfied: lat-lon-parser in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (1.3.0) Requirement already satisfied: xarray-datatree>=0.0.10 in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (0.0.14) Requirement already satisfied: xmltodict in /usr/local/lib/python3.10/dist-packages (from xradar->arm_pyart) (0.13.0) Requirement already satisfied: botocore<1.34.52,>=1.34.41 in /usr/local/lib/python3.10/dist-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs->arm_pyart) (1.34.51) Requirement already satisfied: wrapt<2.0.0,>=1.10.10 in /usr/local/lib/python3.10/dist-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs->arm_pyart) (1.14.1) Requirement already satisfied: aioitertools<1.0.0,>=0.5.1 in /usr/local/lib/python3.10/dist-packages (from aiobotocore<3.0.0,>=2.5.4->s3fs->arm_pyart) (0.11.0) Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs->arm_pyart) (1.3.1) Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs->arm_pyart) (23.2.0) Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs->arm_pyart) (1.4.1) Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs->arm_pyart) (6.0.5) Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs->arm_pyart) (1.9.4) Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->s3fs->arm_pyart) (4.0.3) Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib->arm_pyart) (1.16.0) Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch->arm_pyart) (3.3.2) Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch->arm_pyart) (3.6) Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch->arm_pyart) (2.0.7) Requirement already satisfied: click>=8.0 in /usr/local/lib/python3.10/dist-packages (from dask->xradar->arm_pyart) (8.1.7) Requirement already satisfied: cloudpickle>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from dask->xradar->arm_pyart) (2.2.1) Requirement already satisfied: partd>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from dask->xradar->arm_pyart) (1.4.1) Requirement already satisfied: pyyaml>=5.3.1 in /usr/local/lib/python3.10/dist-packages (from dask->xradar->arm_pyart) (6.0.1) Requirement already satisfied: toolz>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from dask->xradar->arm_pyart) (0.12.1) Requirement already satisfied: importlib-metadata>=4.13.0 in /usr/local/lib/python3.10/dist-packages (from dask->xradar->arm_pyart) (7.0.1) Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.34.52,>=1.34.41->aiobotocore<3.0.0,>=2.5.4->s3fs->arm_pyart) (1.0.1) Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata>=4.13.0->dask->xradar->arm_pyart) (3.17.0) Requirement already satisfied: locket in /usr/local/lib/python3.10/dist-packages (from partd>=1.2.0->dask->xradar->arm_pyart) (1.0.0)
# @title
# BASIC CONFIGURATION
rid = 50 # radar identification number, integer
rdate = '2014/11/27' # UTC date to start the image generation (format: YYYY/MM/DD), string
rtime = '05:00' # UTC time to start the image generation (format: HH:MM), string
rdur = 1 # number of hours from start date/time for image generation (MUST NOT EXTEND TO THE NEXT DAY), integer
prng = 150 # number of kilometers from the radar site to plot, integer
field = 'reflectivity_horizontal' # product to plot. Reflectivity = 'reflectivity_horizontal', Doppler Velocity = 'velocity_horizontal'
# (Others/Polarimetric:differential_reflectivity, cross_correlation_ratio, specific_differential_phase, differential_phase, spectrum_width), string
# MAP CONFIGURATION
hide_place_names = [''] #list of placenames you wish to hide from the map (they are auto generated), list of string
nice_radar_name = 'Marburg' #name of radar to use for title, string
nice_colorbar_name = field #name to use for colorbar
range_ring_list = [50, 100] #list of range ring disances from the radar site (km), list of numbers
# ADVANCED CONFIGURATION
tilt = 1 # PPI index in volume to plot, first/lowest PPI is number 0, integer
# ADVANCED CONFIGURATION - Reflectivity
vmin_ref = 20 # minimum of colourmap (values less than will be removed), integer
vmax_ref = 65 # maximum of colormap (values greater than will be removed), integer
cmap_ref = 'pyart_HomeyerRainbow' # colormap to use. Another option is 'pyart_NWSRef' for more see https://arm-doe.github.io/pyart/API/generated/pyart.graph.html, string
# ADVANCED CONFIGURATION - Doppler Velocity
vmin_vel =-26 # minimum of colormap, integer
vmax_vel = 26 # maximum of colormap, integer
cmap_vel = 'pyart_BuDRd18' # colormap to use, string
# ADVANCED CONFIGURATION - Other/Polarimetric
vmin_pol = -1 # minimum of colormap, integer
vmax_pol = 5 # maximum of colormap, integer
cmap_pol = 'pyart_HomeyerRainbow' # olormap to use, string
# now run the last cell to create the animation
# @title
#########################################################################
def get_date_from_filename(filename, delimiter='_', date_fmt='%Y%m%d_%H%M%S'):
"""
INPUT:
filename (str):
can contain path, has no impact on this function.
filename must use the convention IDDDD_DATE. delimiter + everything else
DATE must have same format as date
OUTPUT:
radar_id (int)
"""
if not isinstance(filename, str):
raise ValueError(f"get_id_from_filename: filename is not a string: {filename}")
return None
if delimiter not in filename:
raise ValueError(f"get_id_from_filename: Delimiter not found in filename: {filename}")
return None
fn = os.path.basename(filename)
fn_parts = fn.split(delimiter)
try:
dtstr = fn_parts[1] + '_' + fn_parts[2].split('.')[0]
dt = datetime.strptime(dtstr, date_fmt)
return dt
except:
raise ValueError(f"get_id_from_filename: Failed to extract radar if from: {filename}")
return None
def get_id_from_filename(filename, delimiter='_'):
"""
INPUT:
filename (str):
can contain path, has no impact on this function.
filename must use the convention IDDDD + delimiter + everything else
OUTPUT:
radar_id (int)
"""
if not isinstance(filename, str):
raise ValueError(f"get_id_from_filename: filename is not a string: {filename}")
return None
if delimiter not in filename:
raise ValueError(f"get_id_from_filename: Delimiter not found in filename: {filename}")
return None
fn = os.path.basename(filename)
fn_parts = fn.split(delimiter)
try:
radar_id = int(fn_parts[0])
return radar_id
except:
raise ValueError(f"get_id_from_filename: Failed to extract radar if from: {filename}")
return None
def make_gif(files, output, delay=100):
frames = [Image.open(image) for image in files]
frame_one = frames[0]
frame_one.save(output, format="GIF", append_images=frames,
save_all=True, duration=delay, loop=0)
def _read_csv(csv_ffn, header_line):
"""
CSV reader used for the radar locations file (comma delimited)
"""
df = pandas.read_csv(csv_ffn, header=header_line)
as_dict = df.to_dict(orient='list')
return as_dict
# Function which generates a plot for each sweep
def _fast_plot(odim_ffn, cdict, img_path):
#open figure
fig = plt.figure(figsize=(10, 8), facecolor='w')
#load radar object
my_radar = pyart.aux_io.read_odim_h5(odim_ffn)
#find limits
radar_lat = my_radar.latitude['data'][0]
radar_lon = my_radar.longitude['data'][0]
plot_range = cdict['prng'] * 1000 #convert to m
min_lon, min_lat = pyart.core.cartesian_to_geographic_aeqd(-plot_range, -plot_range, radar_lon, radar_lat)
max_lon, max_lat = pyart.core.cartesian_to_geographic_aeqd(plot_range, plot_range, radar_lon, radar_lat)
# Set up the GIS projection
projection = ccrs.Mercator(
central_longitude=radar_lon,
min_latitude=min_lat, max_latitude=max_lat)
#load diplay class for radar using cartopy
display = pyart.graph.RadarMapDisplay(my_radar)
#set plotting options
if cdict['field']=='reflectivity_horizontal':
vmin = cdict['vmin_ref']
vmax = cdict['vmax_ref']
cmap = cdict['cmap_ref']
elif cdict['field']=='velocity_horizontal':
vmin = cdict['vmin_vel']
vmax = cdict['vmax_vel']
cmap = cdict['cmap_vel']
else:
vmin = cdict['vmin_pol']
vmax = cdict['vmax_pol']
cmap = cdict['cmap_pol']
#create plot
display.plot_ppi_map(cdict['field'], cdict['tilt'],
projection=projection, colorbar_flag=True,
min_lon=min_lon, max_lon=max_lon, min_lat=min_lat, max_lat=max_lat,
vmin=vmin, vmax=vmax, cmap=cmap,
resolution='10m', mask_outside=True)
#add city markers
ax = plt.gca()
fname = cartopy.io.shapereader.natural_earth(resolution='10m', category='cultural', name='populated_places')
reader = cartopy.io.shapereader.Reader(fname)
city_list = list(reader.records())
texts_list = []
for city in city_list:
if (((city.attributes['LATITUDE'] >= min_lat) and (city.attributes['LATITUDE'] <= max_lat))
and ((city.attributes['LONGITUDE'] >= min_lon) and (city.attributes['LONGITUDE'] <= max_lon))
and (city.attributes['NAME'] not in cdict['hide_place_names'])):
ax.scatter(city.attributes['LONGITUDE'], city.attributes['LATITUDE'], s=6, color='black',
transform=ccrs.PlateCarree(), zorder=5)
texts_list.append(ax.text(city.attributes['LONGITUDE']+0.01, city.attributes['LATITUDE']+0.01,
city.attributes['NAME'], fontsize=10, transform=ccrs.PlateCarree()))
#optimise the location of text
adjust_text(texts_list)
#add range rings
display.plot_range_rings(np.array(cdict['range_ring_list'],dtype=float),
ax=ax, col='k', ls='-', lw=0.5)
#Now we add lat lon lines
gl = display.ax.gridlines(draw_labels=True,
linewidth=1, color='gray', alpha=0.5,
linestyle='--')
gl.xlabel_style = {'size': 12}
gl.ylabel_style = {'size': 12}
gl.xlabels_top = False
gl.ylabels_right = False
#fix title
current_title = ax.get_title()
new_title = cdict['nice_radar_name'] + ' ' + current_title
ax.set_title(new_title)
out_ffn = f'{img_path}/{os.path.basename(odim_ffn)[:-3]}_{cdict["field"]}.png'
plt.savefig(out_ffn, dpi=100) # Saving figure.
plt.close() # Release memory
fig.clf() # Clear figure
#todo
#-fix title
def build_animation(cdict):
#config
base_url = 'http://dapds00.nci.org.au/thredds/fileServer/rq0' #base url for NCI dataset
#parse inputs
radar_id = cdict['rid']
start_dt = datetime.strptime(cdict['rdate'] + ' ' + cdict['rtime'], '%Y/%m/%d %H:%M')
end_dt = start_dt + timedelta(hours = cdict['rdur'])
#build request filename url
zip_fn = str(radar_id) + '_' + start_dt.strftime('%Y%m%d') + '.pvol.zip'
zip_ffn = '/tmp/' + zip_fn
request_url = '/'.join([base_url, str(radar_id), start_dt.strftime('%Y'), 'vol', zip_fn])
#download the zip file
if not os.path.isfile(zip_ffn):
print('Fetching:', request_url)
urllib.request.urlretrieve(request_url, zip_ffn)
else:
print('File already downloaded:', request_url)
#extract the zip file to a temporary directory
temp_dir = tempfile.mkdtemp()
zip_fh = zipfile.ZipFile(zip_ffn)
zip_fh.extractall(path = temp_dir)
zip_fh.close()
#list all the volumes extracted from the zip file
file_list = sorted(glob(temp_dir + '/*'))
#now let's read the datetime numbers of all the volumes for comparision
file_dt_list = []
for i, fname in enumerate(file_list):
file_dt_list.append(get_date_from_filename(fname))
#find the index of volumes within our start and end times
file_dt_array = np.array(file_dt_list)
filter_file_list = []
for i, file_dt in enumerate(file_dt_array):
if file_dt >= start_dt and file_dt <= end_dt:
filter_file_list.append(file_list[i])
#generate images
print('Generating Images')
img_path = tempfile.mkdtemp()
n_img = len(filter_file_list)
for odim_ffn in tqdm(filter_file_list, total=n_img):
_fast_plot(odim_ffn, cdict, img_path)
#create image folder if requires
if not os.path.exists('images'):
os.mkdir('images')
#create animation
now = datetime.now()
gif_ffn = './images/' + f'{radar_id}_{start_dt.strftime("%Y-%m-%d")}_{cdict["field"]}.animation.{now.strftime("%d-%m-%Y_%H:%M:%S")}.gif'
files_to_animate = sorted(glob(img_path + '/*'))
make_gif(files_to_animate, gif_ffn, delay=100)
#zip files and move to local directory
now = datetime.now()
img_zip_fn = f'{radar_id}_{start_dt.strftime("%Y-%m-%d")}_{cdict["field"]}.image_request.{now.strftime("%d-%m-%Y_%H:%M:%S")}.zip'
img_zip_ffn = './images/' + img_zip_fn
#zip up
zipf = zipfile.ZipFile(img_zip_ffn, 'w', zipfile.ZIP_DEFLATED)
files_to_zip = glob(img_path + '/*')
for item in files_to_zip:
zipf.write(item)
zipf.close()
#remove temp images
os.system('rm -rf ' + img_path)
local_file = FileLink(img_zip_ffn, result_html_prefix="Click here to download zip of images: ")
display(local_file)
local_file = FileLink(gif_ffn, result_html_prefix="Click here to download animation: ")
display(local_file)
cdict = {
'rid':rid, 'rdate':rdate, 'rtime':rtime, 'rdur':rdur, 'prng':prng, 'field':field,
'hide_place_names':hide_place_names,'nice_radar_name':nice_radar_name, 'range_ring_list':range_ring_list,
'tilt':tilt,
'vmin_ref':vmin_ref, 'vmax_ref':vmax_ref, 'cmap_ref':cmap_ref,
'vmin_vel':vmin_vel, 'vmax_vel':vmax_vel, 'cmap_vel':cmap_vel,
'vmin_pol':vmin_pol, 'vmax_pol':vmax_pol, 'cmap_pol':cmap_pol
}
build_animation(cdict)
Fetching: http://dapds00.nci.org.au/thredds/fileServer/rq0/50/2014/vol/50_20141127.pvol.zip Generating Images
100%|██████████| 6/6 [02:31<00:00, 25.28s/it]