#!/usr/bin/env python # coding: utf-8 # # Host Explorer # # **Note: This notebook has been superceeded by the Entity Explorer - Host notebook and will be removed in a future update.
# Please migrate to the Host notebook: https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/Entity%20Explorer%20-%20Host.ipynb** # #
#  Details... # # **Notebook Version:** 2.0
# **Python Version:** Python 3.8 (including Python 3.8 - AzureML)
# **Required Packages**: msticpy, msticnb
# # **Data Sources Required**: # - Log Analytics - SecurityAlert, SecurityEvent (EventIDs 4688 and 4624/25), AzureNetworkAnalytics_CL, Heartbeat # - (Optional) - VirusTotal, AlienVault OTX, IBM XForce, Open Page Rank, (all require accounts and API keys) #
# # Brings together a series of queries and visualizations to help you determine the security state of the host that you are investigating. # # # # # # #

Contents

#
# #
# # # # # --- # ### Notebook initialization # The next cell: # - Checks versions and optionally installs required packages # - Imports the required packages into the notebook # - Sets a number of configuration options. # #
# More details... # # This should complete without errors. If you encounter errors or warnings look at the following two notebooks: # - [TroubleShootingNotebooks](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/TroubleShootingNotebooks.ipynb) # - [ConfiguringNotebookEnvironment](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/ConfiguringNotebookEnvironment.ipynb) # # If you are running in the Microsoft Sentinel Notebooks environment (Azure Notebooks or Azure ML) you can run live versions of these notebooks: # - [Run TroubleShootingNotebooks](./TroubleShootingNotebooks.ipynb) # - [Run ConfiguringNotebookEnvironment](./ConfiguringNotebookEnvironment.ipynb) # # You may also need to do some additional configuration to successfully use functions such as Threat Intelligence service lookup and Geo IP lookup. # There are more details about this in the `ConfiguringNotebookEnvironment` notebook and in these documents: # - [msticpy configuration](https://msticpy.readthedocs.io/en/latest/getting_started/msticpyconfig.html) # - [Threat intelligence provider configuration](https://msticpy.readthedocs.io/en/latest/data_acquisition/TIProviders.html#configuration-file) # #
# # In[ ]: from datetime import datetime, timedelta, timezone from IPython.display import HTML, display # %pip install msticpy --upgrade # %pip install msticnb --upgrade display(HTML("

Starting Notebook setup...

")) import msticpy as mp mp.init_notebook( additional_packages=["msticnb>=1.0"], ); # In[ ]: # papermill default parameters ws_name = "Default" host_name = "" # If user_name is supplied in this parameter then activty for that host will be limited to this user. user_name = None end = datetime.now(timezone.utc) start = end - timedelta(days=2) # ## Get WorkspaceId and Authenticate to Microsoft Sentinel #
# Details... # If you are using user/device authentication, run the following cell. # - Click the 'Copy code to clipboard and authenticate' button. # - This will pop up an Azure Active Directory authentication dialog (in a new tab or browser window). The device code will have been copied to the clipboard. # - Select the text box and paste (Ctrl-V/Cmd-V) the copied value. # - You should then be redirected to a user authentication page where you should authenticate with a user account that has permission to query your Log Analytics workspace. # # Use the following syntax if you are authenticating using an Azure Active Directory AppId and Secret: # ``` # %kql loganalytics://tenant(aad_tenant).workspace(WORKSPACE_ID).clientid(client_id).clientsecret(client_secret) # ``` # instead of # ``` # %kql loganalytics://code().workspace(WORKSPACE_ID) # ``` # # **Note:** you may occasionally see a JavaScript error displayed at the end of the authentication - you can safely ignore this.
# # On successful authentication you should see a ```popup schema``` button. # To find your Workspace Id go to [Log Analytics](https://ms.portal.azure.com/#blade/HubsExtension/Resources/resourceType/Microsoft.OperationalInsights%2Fworkspaces). Look at the workspace properties to find the ID. #
# In[ ]: print( "Configured workspaces: ", ", ".join(msticpy.settings.get_config("AzureSentinel.Workspaces").keys()), ) import ipywidgets as widgets ws_param = widgets.Combobox( description="Workspace Name", value=ws_name, options=list(msticpy.settings.get_config("AzureSentinel.Workspaces").keys()), ) ws_param # In[ ]: from msticpy.common.timespan import TimeSpan from msticpy.context.tilookup import TILookup # Authentication qry_prov = QueryProvider(data_environment="MSSentinel") qry_prov.connect(WorkspaceConfig(workspace=ws_param.value)) nb_timespan = TimeSpan(start, end) qry_prov.query_time.timespan = nb_timespan md("
") md("Confirm time range to search", "bold") qry_prov.query_time # ### Authentication and Configuration Problems # #
#
# Click for details about configuring your authentication parameters # # # The notebook is expecting your Microsoft Sentinel Tenant ID and Workspace ID to be configured in one of the following places: # - `config.json` in the current folder # - `msticpyconfig.yaml` in the current folder or location specified by `MSTICPYCONFIG` environment variable. # # For help with setting up your `config.json` file (if this hasn't been done automatically) see the [`ConfiguringNotebookEnvironment`](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/ConfiguringNotebookEnvironment.ipynb) notebook in the root folder of your Azure-Sentinel-Notebooks project. This shows you how to obtain your Workspace and Subscription IDs from the Microsoft Sentinel Portal. You can use the SubscriptionID to find your Tenant ID). To view the current `config.json` run the following in a code cell. # # ```%pfile config.json``` # # For help with setting up your `msticpyconfig.yaml` see the [Setup](#Setup) section at the end of this notebook and the [ConfigureNotebookEnvironment notebook](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/ConfiguringNotebookEnvironment.ipynb) #
# ## Import and initialize notebooklets # # This imports the **msticnb** package and the notebooklets classes. # # These are needed for the notebook operations # In[ ]: import msticnb as nb nb.init(query_provider=qry_prov) pivot.timespan = qry_prov.query_time.timespan # # Enter host name and query time window # Type the host name that you want to search for and the time bounds over which you want to search. # In[ ]: host_txt = nbwidgets.GetText( prompt="Enter the Host name to search for:", value=host_name ) display(host_txt) # # Review host overview # # The following cells runs the Host Summary Notebooklet to provide an overview of the host, and its activty within the timeframe specified. # Use the output of this cell to understand the context of its host and identify areas of further investigation. # In[ ]: host_nb = nb.nblts.azsent.host.HostSummary() md( "Note: Different result properties are populated depending on the account type", "large, bold", ) host_result = host_nb.run( value=host_txt.value, timespan=qry_prov.query_time.timespan, silent=True ) # # Review alerts # The following cell returns a list of all Microsoft Sentinel alerts reated to the host. You can browse and review these alerts. # In[ ]: host_result.notebooklet.browse_alerts() # Below is a timeline of the alerts related to the host. # In[ ]: if host_result.alert_timeline: display(host_result.display_alert_timeline()) else: md(f"No alerts for {host_txt.value}") # # Review bookmarks # If there are any bookmarks referencing this host they can be viewed by calling `host_result.related_bookmarks`.
# Review these bookmarks to see if this host has been flagged as part of a previous investigation or threat hunt. # In[ ]: if ( isinstance(host_result.related_bookmarks, pd.DataFrame) and not host_result.related_bookmarks.empty ): display(host_result.related_bookmarks) else: md(f"No bookmarks for {host_txt.value}") # # Summarize Host Events # # As there are likely to be a large number of log events for a host the below table is a summary of all the events from the host.
# You can use this table to idenfify addtional queries to run to review specific types of log entries. # # In[ ]: host_result.summary # # Noteable Host Events # Some log events such as those of a high severity are considered to be "noteable" events. # Review these events and combined with the summary of all events you can identify additional queries to run to review specific types of log entries. # # To access the DataFrames output by this code call `host_result.scheduled_tasks`, `host_result.account_actions` or `host_result.notable_events` to access to data. # In[ ]: schld_source_columns = [ "Service", "ServiceType", "ServiceStartType", "ScheduledTaskDetails", "Account", "TimeGenerated", "Activity", ] account_source_columns = ["TargetAccount", "Activity", "TimeGenerated", "Account"] notable_source_columns = ["Account", "TimeGenerated", "Activity"] if host_result.host_entity.OSFamily.name == "Linux": schld_source_columns = ["CMD", "User", "CronUser", "EditStatus", "TimeGenerated"] account_source_columns = ["User", "Group", "TimeGenerated", "UserGroupAction"] notable_source_columns = ["Facility", "TimeGenerated", "SeverityLevel"] if not host_result.scheduled_tasks.empty: host_result.scheduled_tasks.mp_plot.timeline( group_by="Type", source_columns=schld_source_columns, title="Service and Scheduled Task Events", ) md("Events related to Services and Scheduled Tasks:", "bold") display(host_result.scheduled_tasks) if not host_result.account_actions.empty: host_result.account_actions.mp_plot.timeline( group_by="EventID", source_columns=account_source_columns, title="Account modification Events", ) md("Events related to account modifications:", "bold") display(host_result.account_actions) if not host_result.notable_events.empty: host_result.notable_events.mp_plot.timeline( group_by="EventID", source_columns=notable_source_columns, title="Other Events" ) md("Other Events", "Bold") display(host_result.notable_events) # # Review Host Logons # Host activity is often driven by user actions. The following cell runs the Host Logon Notebooklet that summarizes logon sessions related to the host.
# Review the output of this notebooklet to identify logon sessions of note. # In[ ]: host_logons_nb = nb.nblts.azsent.host.HostLogonsSummary() md( "Note: Different result properties are populated depending on the account type", "large, bold", ) host_logons_result = host_logons_nb.run( value=host_txt.value, timespan=qry_prov.query_time.timespan, ) # In[ ]: def most_common_users(): if host_result.host_entity.OSFamily.name == "Windows": accounts = host_logons_result.logon_sessions["Account"].value_counts() accounts.drop(index="NT AUTHORITY\SYSTEM", inplace=True) computer_accounts = [row for row in accounts.index if row.endswith("$")] accounts.drop(index=computer_accounts, inplace=True) return [account.split("\\")[1] for account in accounts.index] users = most_common_users() user_name = user_name or users[0] user_param = widgets.Combobox( description="Select User Account To Focus On", value=user_name, options=list(users), ) user_param # In[ ]: user_name = user_param.value if isinstance(host_logons_result.logon_sessions, pd.DataFrame) and not host_logons_result.logon_sessions.empty: md(f"Logon sessions for {user_name}:", "bold") display(host_logons_result.logon_sessions[host_logons_result.logon_sessions["TargetUserName"].str.contains(user_name, case=False)]) else: md("No valid logon sessions found") # # Host Processes # The following is a process tree of all the processes executed on the host in the time window defined. # You can interact with the tree to see parent and child processes. # In[ ]: if isinstance(host_result.processes, pd.DataFrame) and not host_result.processes.empty: host_result.processes.mp_plot.process_tree() else: md("No process execution information found.") # The above process tree may be too large to find events of value, the following is a processes tree to processeses associated with the defined user (if no user is defined then the most commonly seen user is used). # In[ ]: if isinstance(host_result.processes, pd.DataFrame) and not host_result.processes.empty: if not host_result.processes[ host_result.processes["Account"].str.contains(user_name, case=False) ].empty: md(f"Processes executed by {user_name}", "bold") host_result.processes[ host_result.processes["Account"].str.contains(user_name, case=False) ].mp_plot.process_tree() else: md(f"No processes executed by {user_name}") else: md("No process execution information found.") # ## Extract IOCs related to these processes and look them up against Threat Intelligence # Process data often contains command line activity, we can extract IoCs from these command lines and look them up in Threat Intelligence sources to help narrow focus on interesting processes. # # To access the DataFrames output by this code call `host_result.processes` or `process_ti_results` to access to data. # In[ ]: from msticnb.nb.azsent.host.host_summary import _process_ti if isinstance(host_result.processes, pd.DataFrame) and not host_result.processes.empty: user_processes = host_result.processes[ host_result.processes["Account"].str.contains(user_name, case=False) ] if host_result.host_entity.OSFamily.name == "Windows": cmd_column = "CommandLine" else: cmd_column = "SyslogMessage" ti_prov = host_nb.ti_prov if hasattr(host_nb, "ti_prov") else TILookup() process_ti_results = _process_ti(user_processes, cmd_column, ti_prov) if isinstance(process_ti_results, pd.DataFrame) and not process_ti_results.empty: md(f"TI results for processes executed by {user_name}") display(process_ti_results) else: md("No TI results found.") else: md("No process execution information found.") # # Review Network Connections # A hosts network traffic can often help identify anomolous or suspicious patterns of activity.
# The cell below runs the Host Network Connections Notebooklet that summarizes network connections related to the host.
# Review the output of this cell to identify suspicious network connection patterns. # # # To access the DataFrames output by this code call `host_network_result.flows`, `host_network_result.flows_ti` to access to data. # In[ ]: host_network_nb = nb.nblts.azsent.host.HostNetworkSummary() md( "Note: Different result properties are populated depending on the account type", "large, bold", ) host_network_result = host_network_nb.run( value=host_result.host_entity, timespan=qry_prov.query_time.timespan, ) # --- # # ## Use other notebooklets and pivots functions to drill down on other entities # # You may want to drill down on other entities in the Host data. # You can use methods of the IpAddress or Account entities, for example, # to look at these in more detail. # # Run the ip_address_summary notebooklet pivot # ```python # IpAddress = entities.IpAddress # ip_result = IpAddress.nblt.ip_address_summary("157.56.162.53") # ``` # # View the TI results # ```python # ip_result.browse_ti_results() # ``` # --- # # # More information: # # ## Notebooklets and Pivots # [Notebooklets](https://msticnb.readthedocs.io/en/latest/) # # [Pivot functions](https://msticpy.readthedocs.io/en/latest/data_analysis/PivotFunctions.html) # # ## Notebook/MSTICPy configuration # [Getting Started](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/A%20Getting%20Started%20Guide%20For%20Azure%20Sentinel%20ML%20Notebooks.ipynb)
# [MSTICPy Configuration guide](https://msticpy.readthedocs.io/en/latest/getting_started/msticpyconfig.html) # # [ConfigureNotebookEnvironment notebook](https://github.com/Azure/Azure-Sentinel-Notebooks/blob/master/ConfiguringNotebookEnvironment.ipynb)