Copyright (c) 2021, ETH Zurich, Computer Engineering Group (TEC)
This script visualizes the health data of the STeC deployments for the last week both on DH and ETZ (test deployment).
import datetime as dt
import pandas as pd
import sys
import plotly.express as px
sys.path.append('../') # FIXME: Work around if not built as a package
from data_management.data_manager import DataManager
from data_management.data_processing import DataProcessor
# Settings
TEST_SERVER = 'http://tpbl.permasense.ethz.ch/'
DEPLOYMENT_OUTDOOR = 'dirruhorn'
DEPLOYMENT_TEST = 'etz'
VS_ACOUSTIC_METADATA = '_dpp_geophone_acq__conv'
VS_ACOUSTIC_AGGR = '_dpp_geophone_acq_min_aggr__mapped'
VS_ACOUSTIC_DATA = '_dpp_geophone_adcData__conv'
VS_HEALTH_MIN_DATA = '_dpp_health_min__conv'
DATA_START_TIME = dt.datetime.today() - dt.timedelta(days=7)
DATA_END_TIME = dt.datetime.today()
S_TO_US = 1000 * 1000
FALLBACK_BS_IDS_OUTDOOR = [106, 110]
FALLBACK_BS_IDS_TEST = [103, 107, 109]
# Print settings
print('Going to fetch STeC Health data at deployment site {0:s} and {1:s} from {2:s} - {3:s}'.format(DEPLOYMENT_OUTDOOR.capitalize(), DEPLOYMENT_TEST.capitalize(), DATA_START_TIME.strftime('%d/%m/%Y'), DATA_END_TIME.strftime('%d/%m/%Y')))
# Adjust display width for variables
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
Going to fetch STeC Health data at deployment site Dirruhorn and Etz from 04/11/2021 - 11/11/2021
# Create necessary objects
DataMgr = DataManager(deployment=DEPLOYMENT_OUTDOOR, config_file='../stec.conf', project_name='stec', start_time=DATA_START_TIME, end_time=DATA_END_TIME)
DataProc = DataProcessor(config_file='../stec.conf', project_name='stec')
# Fetch data
# Create URL including conditions
url_health_outdoor = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA)
url_aggr_outdoor = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR)
url_health_test = DataMgr.assemble_gsn_url(VS_HEALTH_MIN_DATA, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
url_aggr_test = DataMgr.assemble_gsn_url(VS_ACOUSTIC_AGGR, server_url=TEST_SERVER, deployment=DEPLOYMENT_TEST)
# Fetch data
df_health_outdoor = DataMgr.fetch_csv_data(url_health_outdoor, description="Health", cache=False)
df_aggr_outdoor = DataMgr.fetch_csv_data(url_aggr_outdoor, description="Aggregation", cache=False)
df_health_test = DataMgr.fetch_csv_data(url_health_test, description="Health", abort=False, cache=False)
df_aggr_test = DataMgr.fetch_csv_data(url_aggr_test, description="Aggregation", abort=False, cache=False)
17:40:05 - DataManager - INFO - No Aggregation data found for CSV from path: http://data.permasense.ch/multidata?field[0]=All&vs[0]=dirruhorn_dpp_geophone_acq_min_aggr__mapped&time_format=iso&timeline=generation_time&from=04/11/2021+17:40:05&to=11/11/2021+17:40:05 17:40:05 - DataManager - INFO - No Health data found for CSV from path: http://tpbl.permasense.ethz.ch//multidata?field[0]=All&vs[0]=etz_dpp_health_min__conv&time_format=iso&timeline=generation_time&from=04/11/2021+17:40:05&to=11/11/2021+17:40:05 17:40:05 - DataManager - INFO - No Aggregation data found for CSV from path: http://tpbl.permasense.ethz.ch//multidata?field[0]=All&vs[0]=etz_dpp_geophone_acq_min_aggr__mapped&time_format=iso&timeline=generation_time&from=04/11/2021+17:40:05&to=11/11/2021+17:40:05
# Prepare data for plotting
# Formatting timestamps, add column indicating whether the packet was received over SF7 or SF10
if df_health_outdoor is not None:
df_health_outdoor['generation_time'] = pd.to_datetime(df_health_outdoor['generation_time_microsec'], unit='us')
df_health_outdoor['device_id_str'] = df_health_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_outdoor['fallback'] = df_health_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_health_outdoor['fallback_str'] = df_health_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_outdoor is not None:
df_aggr_outdoor['generation_time'] = pd.to_datetime(df_aggr_outdoor['generation_time_microsec'], unit='us')
df_aggr_outdoor['device_id_str'] = df_aggr_outdoor['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_outdoor['fallback'] = df_aggr_outdoor['target_id'].isin(FALLBACK_BS_IDS_OUTDOOR)
df_aggr_outdoor['fallback_str'] = df_aggr_outdoor['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_health_test is not None:
df_health_test['generation_time'] = pd.to_datetime(df_health_test['generation_time_microsec'], unit='us')
df_health_test['device_id_str'] = df_health_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_health_test['fallback'] = df_health_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_health_test['fallback_str'] = df_health_test['fallback'].astype(str) # Required for correct colouring of scatter plots
if df_aggr_test is not None:
df_aggr_test['generation_time'] = pd.to_datetime(df_aggr_test['generation_time_microsec'], unit='us')
df_aggr_test['device_id_str'] = df_aggr_test['device_id'].astype(str) # Required for correct colouring of scatter plots
df_aggr_test['fallback'] = df_aggr_test['target_id'].isin(FALLBACK_BS_IDS_TEST)
df_aggr_test['fallback_str'] = df_aggr_test['fallback'].astype(str) # Required for correct colouring of scatter plots
# Color mapping for consistent display of fallback data
color_fallback_map = {'True': 'red', 'False': 'green'}
After having pre-processed the data, we now plot the data for visual inspection.
# Plotting Outdoor
if df_health_outdoor is not None:
# Health messages over time
fig = px.scatter(df_health_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id', 'position'], title='Outdoor Deployment (Dirruhorn)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_outdoor is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_outdoor.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
else:
print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')
if df_health_outdoor is not None:
# Uptime
fig = px.line(df_health_outdoor, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Sequence number
fig = px.line(df_health_outdoor, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Temperature
fig = px.line(df_health_outdoor, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Humidity
fig = px.line(df_health_outdoor, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_outdoor, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_outdoor, min_id=21006, reference_id=112)
# fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
# fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_outdoor)
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.matter)
fig.update_yaxes(dtick=1)
fig.show()
Skipping plotting aggregation data as no co-detections have been observed in the selected time frame
# Plotting Test Deployment
if df_health_test is not None:
# Health messages over time
fig = px.scatter(df_health_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Health from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['target_id'], title='Test Deployment (ETZ)')
fig.update_yaxes(type='category')
fig.show()
if df_aggr_test is not None:
# Aggregated messages over time
fig = px.scatter(df_aggr_test.sort_values('device_id', inplace=False), x='generation_time', y='device_id', labels={'generation_time': 'Time', 'device_id': 'Aggregated Acq from Device ID', 'device_id_str': 'Device ID', 'fallback_str': 'Fallback modulation', 'BaseStation ID:': 'target_id'}, color='fallback_str', color_discrete_map=color_fallback_map, hover_name='generation_time', hover_data=['block_cnt', 'seqnr', 'target_id'])
fig.update_yaxes(type='category')
fig.show()
else:
print('Skipping plotting aggregation data as no co-detections have been observed in the selected time frame')
if df_health_test is not None:
# Uptime
fig = px.line(df_health_test, x='generation_time', y='uptime', labels={'generation_time': 'Time', 'uptime': 'Uptime [s]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Sequence number
fig = px.line(df_health_test, x='generation_time', y='seqnr', labels={'generation_time': 'Time', 'seqnr': 'Sequence Number [#]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Temperature
fig = px.line(df_health_test, x='generation_time', y='temperature', labels={'generation_time': 'Time', 'temperature': 'Temperature [°C]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Humidity
fig = px.line(df_health_test, x='generation_time', y='humidity', labels={'generation_time': 'Time', 'humidity': 'Humidity [%]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.matter)
fig.show()
# Radio RSSI
fig = px.line(df_health_test, x='generation_time', y='radio_rssi', labels={'generation_time': 'Time', 'radio_rssi': 'Radio RSSI [dBm]', 'device_id': 'Device ID'}, color='device_id', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Clock drift
drift_data = DataProc.extract_clock_drift(df_health_test, min_id=21006, reference_id=108)
#fig = px.scatter(drift_data, x='generation_time', y='clock_drift_local', labels={'generation_time': 'Time', 'clock_drift_local': 'Local clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
#fig.show()
fig = px.scatter(drift_data, x='generation_time', y='clock_drift_global', labels={'generation_time': 'Time', 'clock_drift_global': 'Clock drift [us]', 'device_id': 'Device ID', 'device_id_str': 'Device ID'}, color='device_id_str', color_discrete_sequence=px.colors.sequential.deep)
fig.show()
# Base station stats
bs_data = DataProc.extract_bs_data(df_health_test)
fig = px.line(bs_data, x='generation_time', y='nr_reports', labels={'generation_time': 'Time', 'nr_reports': 'Number of reports received [#]', 'bs_id': 'BaseStation ID'}, color='bs_id', color_discrete_sequence=px.colors.sequential.deep)
fig.update_yaxes(dtick=1)
fig.show()
if df_health_test is None and df_aggr_test is None:
print('Skipping test data as not able to access from this machine (no connection to ETH-internal network)')
Skipping plotting aggregation data as no co-detections have been observed in the selected time frame Skipping test data as not able to access from this machine (no connection to ETH-internal network)