# Imports
import sys
MIN_REQ_PYTHON = (3,6)
if sys.version_info < MIN_REQ_PYTHON:
print('Check the Kernel->Change Kernel menu and ensure that Python 3.6')
print('or later is selected as the active kernel.')
sys.exit("Python %s.%s or later is required.\n" % MIN_REQ_PYTHON)
import numpy as np
from IPython import get_ipython
from IPython.display import display, HTML
import ipywidgets as widgets
import matplotlib.pyplot as plt
import seaborn as sns
import networkx as nx
sns.set()
import pandas as pd
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 50)
pd.set_option('display.max_colwidth', 100)
import os
module_path = os.path.abspath(os.path.join('../..'))
if module_path not in sys.path:
sys.path.append(module_path)
import msticpy.sectools as sectools
import msticpy.nbtools as asi
import msticpy.nbtools.kql as qry
import msticpy.nbtools.nbdisplay as nbdisp
# Load test data
process_tree = pd.read_csv('data/process_tree.csv')
process_tree[['CommandLine']].head()
CommandLine | |
---|---|
0 | .\ftp -s:C:\RECYCLER\xxppyy.exe |
1 | .\reg not /domain:everything that /sid:shines is /krbtgt:golden ! |
2 | cmd /c "systeminfo && systeminfo" |
3 | .\rundll32 /C 12345.exe |
4 | .\rundll32 /C c:\users\MSTICAdmin\12345.exe |
# get a commandline from our data set
cmdline = process_tree['CommandLine'].loc[78]
cmdline
'netsh start capture=yes IPv4.Address=1.2.3.4 tracefile=C:\\\\Users\\\\user\\\\AppData\\\\Local\\\\Temp\\\\bzzzzzz.txt'
# Instantiate an IoCExtract object
from msticpy.sectools import IoCExtract
ioc_extractor = IoCExtract()
# any IoCs in the string?
iocs_found = ioc_extractor.extract(cmdline)
if iocs_found:
print('\nPotential IoCs found in alert process:')
display(iocs_found)
Potential IoCs found in alert process:
defaultdict(set, {'ipv4': {'1.2.3.4'}, 'windows_path': {'C:\\\\Users\\\\user\\\\AppData\\\\Local\\\\Temp\\\\bzzzzzz.txt'}})
ioc_extractor = IoCExtract()
ioc_df = ioc_extractor.extract(data=process_tree, columns=['CommandLine'], os_family='Windows')
if len(ioc_df):
display(HTML("<h3>IoC patterns found in process tree.</h3>"))
display(ioc_df)
IoCType | Observable | SourceIndex | |
---|---|---|---|
0 | windows_path | C:\RECYCLER\xxppyy.exe | 0 |
1 | windows_path | .\ftp | 0 |
2 | windows_path | .\reg | 1 |
3 | windows_path | .\rundll32 | 3 |
4 | windows_path | c:\users\MSTICAdmin\12345.exe | 4 |
5 | windows_path | .\rundll32 | 4 |
6 | windows_path | .\rundll32 | 5 |
7 | windows_path | c:\users\MSTICAdmin\1234.exe | 6 |
8 | windows_path | .\rundll32 | 6 |
9 | windows_path | .\rundll32 | 7 |
10 | windows_path | .\reg.exe add \hkcu\software\microsoft\some\key\Run | 8 |
11 | dns | tsetup.1.0.14.exe | 9 |
12 | dns | tsetup.1.exe | 9 |
13 | dns | tsetup.1.0.14.tmp | 9 |
14 | windows_path | c:\Diagnostics\UserTmp\tsetup.1.exe | 9 |
15 | windows_path | C:\Users\MSTICAdmin\Downloads\tsetup.1.0.14.exe | 9 |
16 | windows_path | C:\Users\MSTICAdmin\AppData\Local\Temp\2\is-01DD7.tmp\tsetup.1.0.14.tmp | 9 |
17 | windows_path | .\rundll32.exe | 10 |
18 | windows_path | .\netsh.exe | 11 |
19 | windows_path | .\cmd | 12 |
20 | windows_path | C:\inetpub\wwwroot | 12 |
21 | windows_path | .\cmd | 13 |
22 | windows_path | C:\inetpub\wwwroot | 13 |
23 | windows_path | .\cmd | 14 |
24 | windows_path | C:\inetpub\wwwroot | 14 |
25 | windows_path | .\cmd | 15 |
26 | windows_path | \\[REDACTED]\c$\users\[REDACTED]\Documents | 15 |
27 | windows_path | C:\ProgramData | 15 |
28 | windows_path | .\cmd | 16 |
29 | windows_path | C:\inetpub\wwwroot | 16 |
30 | windows_path | c:\windows\system32\inetsrv\appcmd | 16 |
31 | windows_path | .\cmd | 17 |
32 | windows_path | C:\inetpub\wwwroot | 17 |
33 | windows_path | C:\inetpub\logs\logFiles\W3SVC1 | 17 |
34 | windows_path | c:\Diagnostics\UserTmp\perfc.dat | 18 |
35 | windows_path | c:\Diagnostics\UserTmp\sdopfjiowtbkjfnbeioruj.exe | 19 |
36 | dns | doubleextension.pdf.exe | 20 |
37 | windows_path | c:\Diagnostics\UserTmp\doubleextension.pdf.exe | 20 |
38 | windows_path | \Windows\system32\conhost.exe | 22 |
39 | windows_path | \C: | 22 |
40 | windows_path | c:\testshare | 26 |
41 | windows_path | \\MSTICAlertsWin1\TestShare | 27 |
42 | url | http://server/file.sct | 31 |
43 | dns | server | 31 |
44 | windows_path | .\regsvr32 | 31 |
45 | windows_path | .\suchost.exe | 32 |
46 | windows_path | .\evil.ps1; | 35 |
47 | windows_path | .\powershell.exe | 35 |
48 | windows_path | .\powershell | 36 |
49 | url | http://somedomain/best-kitten-names-1.jpg' | 37 |
50 | dns | somedomain | 37 |
51 | windows_path | \AppData\Local\Temp\kittens1.jpg'; | 37 |
52 | windows_path | C:\Users\$env:UserName | 37 |
53 | windows_path | .\pOWErS^H^ElL^.eX^e^ | 37 |
54 | windows_path | .\n^e^t | 38 |
55 | windows_path | .\powershell | 39 |
56 | md5_hash | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | 40 |
57 | md5_hash | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | 41 |
58 | md5_hash | 81ed03caf6901e444c72ac67d192fb9c | 44 |
59 | url | http://badguyserver/pwnme" | 46 |
60 | dns | badguyserver | 46 |
61 | url | http://badguyserver/pwnme" | 47 |
62 | dns | badguyserver | 47 |
63 | windows_path | .\powershell | 47 |
64 | windows_path | .\powershell | 48 |
65 | windows_path | .\powershell | 49 |
66 | windows_path | .\powershell | 50 |
67 | windows_path | .\rUnDlL32 | 58 |
68 | windows_path | .\reg query add mscfile\\\\open | 59 |
69 | windows_path | .\reg | 60 |
70 | windows_path | .\dubrute.exe | 61 |
71 | windows_path | .\nlbrute.exe | 62 |
72 | windows_path | \system\CurrentControlSet\Control\Terminal | 63 |
73 | windows_path | .\reg | 63 |
74 | windows_path | \system\CurrentControlSet\Control\Terminal | 64 |
75 | windows_path | .\reg | 64 |
76 | windows_path | \\tsclient\c | 65 |
77 | windows_path | \Microsoft\Windows\CurrentVersion Certificate).Certificate);.\powershell | 67 |
78 | windows_path | C:\Windows\System32\mshta.exe | 67 |
79 | windows_path | .\powershell.exe | 67 |
80 | windows_path | .\netsh | 68 |
81 | windows_path | c:\users\Bob\appdata\Roaming\RbtGskQ\RbtGskQ.exe | 68 |
82 | windows_path | .\reg add HKLM\KEY_LOCAL_MACHINE\...securityproviders\wdigest | 69 |
83 | windows_path | c:\Windows\System32\cmd.exe | 70 |
84 | windows_path | c:\Diagnostics\UserTmp\scrsave.scr | 71 |
85 | windows_path | c:\Diagnostics\UserTmp\svchost.exe | 72 |
86 | windows_path | c:\Diagnostics\UserTmp\smss.exe | 73 |
87 | windows_path | c:\Windows\System32\svchost.exe | 74 |
88 | dns | system.management.automation.amsiutils | 77 |
89 | dns | system.management.automation.amsiutils').getfield('amsiinitfailed','nonpublic,static').setvalue(... | 77 |
90 | url | http://system.management.automation.amsiutils').getfield('amsiinitfailed','nonpublic,static').se... | 77 |
91 | windows_path | .\powershell.exe | 77 |
92 | ipv4 | 1.2.3.4 | 78 |
93 | windows_path | C:\\Users\\user\\AppData\\Local\\Temp\\bzzzzzz.txt | 78 |
94 | windows_path | .\wuauclt.exe | 79 |
95 | windows_path | c:\windows\softwaredistribution\cscript.exe | 79 |
96 | windows_path | c:\windows\softwaredistribution\cscript.exe | 80 |
97 | windows_path | .\lsass.exe | 80 |
98 | windows_path | c:\windows\system32\wscript.exe | 82 |
99 | windows_path | C:\inetpub\wwwroot | 83 |
100 | windows_path | c:\windows\system32\inetsrv\appcmd | 83 |
101 | windows_path | c:\Diagnostics\UserTmp\2840.exe | 84 |
102 | windows_path | c:\Diagnostics\UserTmp\a_keygen.exe | 85 |
103 | windows_path | c:\Diagnostics\UserTmp\bittorrent.exe | 87 |
104 | windows_path | c:\Diagnostics\UserTmp\netsh.exe | 88 |
105 | windows_path | c:\Diagnostics\UserTmp\ransomware.exe | 90 |
106 | windows_path | \\server\payload.dll | 92 |
107 | windows_path | C:\Users\Administrator\AppData\Roaming\{RANDOM}.txt | 94 |
108 | ipv4 | 127.0.0.1 | 102 |
109 | url | http://127.0.0.1/ | 102 |
110 | windows_path | \SOFTWARE\Microsoft\Windows NT\CurrentVersion\Svchost\MyNastySvcHostConfig | 103 |
111 | windows_path | .\reg | 103 |
112 | windows_path | \SOFTWARE\Microsoft\Windows NT\CurrentVersion\Svchost\MyNastySvcHostConfig | 104 |
113 | windows_path | .\reg | 104 |
114 | windows_path | C:\Users\MSTICA~1\AppData\Local\Temp\hd.exe | 105 |
115 | windows_path | \\.\pipe\blahtest | 107 |
116 | windows_path | .\reg.exe | 108 |
117 | windows_path | \console | 108 |
118 | windows_path | c:\windows\fonts\csrss.exe | 109 |
119 | windows_path | c:\windows\fonts\conhost.exe | 110 |
120 | windows_path | .\mimikatz.exe | 111 |
121 | windows_path | .\rundll32.exe | 112 |
122 | windows_path | c:\windows\fonts\conhost.exe | 112 |
123 | windows_path | .\regsvr32 | 113 |
124 | windows_path | c:\windows\fonts\csrss.exe | 113 |
125 | windows_path | c:\Diagnostics\UserTmp | 115 |
126 | windows_path | c:\Diagnostics\WindowsSimulateDetections.bat | 115 |
127 | windows_path | C:\Windows\System32\win32calc.exe | 116 |
# IoCExtract docstring
IoCExtract?
extractor = IoCExtract()
for ioc_type, pattern in extractor.ioc_types.items():
display(HTML(f'<b>{ioc_type}</b>'))
display(HTML(f'<div style="margin-left:20px"><pre>{pattern.comp_regex}<pre></div>)'))
re.compile('(?P(?:[0-9]{1,3}\\.){3}[0-9]{1,3})', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile('(?
re.compile('((?=[a-z0-9-]{1,63}\\.)[a-z0-9]+(-[a-z0-9]+)*\\.){2,}[a-z]{2,63}', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile("\n (?P(https?|ftp|telnet|ldap|file)://)\n (?P ([a-z0-9-._~!$&\\'()*+,;=:]|%[0-9A-F]{2})*@)?\n (?P ([a-z0-9-._~!$&\\'()*+,;=]|%[0-9A-F]{2})*)\n, re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile('\n (?P[a-z]:|\\\\\\\\[a-z0-9_.$-]+||[.]+)\n (?P \\\\(?:[^\\/:*?"\\\'<>|\\r\\n]+\\\\)*)\n (?P [^\\\\/*?""<>|\\r\\n ]+)', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile('(?P/+||[.]+)\n (?P /(?:[^\\\\/:*?<>|\\r\\n]+/)*)\n (?P [^/\\0<>|\\r\\n ]+)', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile('(?:^|[^A-Fa-f0-9])(?P[A-Fa-f0-9]{32})(?:$|[^A-Fa-f0-9])', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile('(?:^|[^A-Fa-f0-9])(?P[A-Fa-f0-9]{40})(?:$|[^A-Fa-f0-9])', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
re.compile('(?:^|[^A-Fa-f0-9])(?P[A-Fa-f0-9]{64})(?:$|[^A-Fa-f0-9])', re.IGNORECASE|re.MULTILINE|re.VERBOSE)
Docstring:
Add an IoC type and regular expression to use to the built-in set.
Note: adding an ioc_type that exists in the internal set will overwrite that item
Regular expressions are compiled with re.I | re.X | re.M (Ignore case, Verbose
and MultiLine)
:param: ioc_type{str} - a unique name for the IoC type
:param: ioc_regex{str} - a regular expression used to search for the type
import re
rcomp = re.compile(r'(?P<pipe>\\\\\.\\pipe\\[^\s\\]+)')
extractor.add_ioc_type(ioc_type='win_named_pipe', ioc_regex=r'(?P<pipe>\\\\\.\\pipe\\[^\s\\]+)')
# Check that it added ok
print(extractor.ioc_types['win_named_pipe'])
# Use it in our data set
ioc_extractor.extract(data=process_tree, columns=['CommandLine'], os_family='Windows').query('IoCType == \'win_named_pipe\'')
IoCPattern(ioc_type='win_named_pipe', comp_regex=re.compile('(?P<pipe>\\\\\\\\\\.\\\\pipe\\\\[^\\s\\\\]+)', re.IGNORECASE|re.MULTILINE|re.VERBOSE), priority=0)
IoCType | Observable | SourceIndex | |
---|---|---|---|
116 | win_named_pipe | \\.\pipe\blahtest" | 107 |
Docstring:
Extract IoCs from either a string or pandas DataFrame.
:param data: input DataFrame from which to read source strings
:param columns: The list of columns to use as source strings,
if the data parameter is used.
:param src: source string in which to look for IoC patterns
:param os_family: 'Linux' or 'Windows'
Returns:
dict of found observables (if input is a string) or
DataFrame of observables
Extract takes either a string or a pandas DataFrame as input.
When using the string option as an input extract will
return a dictionary of results.
When using a DataFrame the results will be returned as a new
DataFrame with the following columns:
- IoCType: the mnemonic used to distinguish different IoC Types
- Observable: the actual value of the observable
- SourceIndex: the index of the row in the input DataFrame from
which the source for the IoC observable was extracted.
Note the os_family parameter is optional. If you are not interested in searching for Linux paths omit this or set to 'Windows'. Almost anything is a legal character in a Linux path name so this is a very loose regex (the built-in one is more restrictive than the possible path names, otherwise this will match too much to be useful).
# You can specify multiple columns
ioc_extractor.extract(data=process_tree.head(20), columns=['NewProcessName', 'CommandLine']).head(10)
IoCType | Observable | SourceIndex | |
---|---|---|---|
0 | windows_path | C:\Diagnostics\UserTmp\ftp.exe | 0 |
1 | windows_path | C:\RECYCLER\xxppyy.exe | 0 |
2 | windows_path | .\ftp | 0 |
3 | windows_path | C:\Diagnostics\UserTmp\reg.exe | 1 |
4 | windows_path | .\reg | 1 |
5 | windows_path | C:\Diagnostics\UserTmp\cmd.exe | 2 |
6 | windows_path | C:\Diagnostics\UserTmp\rundll32.exe | 3 |
7 | windows_path | .\rundll32 | 3 |
8 | windows_path | C:\Diagnostics\UserTmp\rundll32.exe | 4 |
9 | windows_path | c:\users\MSTICAdmin\12345.exe | 4 |
Where an input row has multiple IoC matches the output of this merge will result in duplicate rows from the input (one per IoC match). The previous index is preserved in the second column (and in the SourceIndex column).
Note: you will need to set the type of the SourceIndex column. In the example below case we are matching with the default numeric index so we force the type to be numeric. In cases where you are using an index of a different dtype you will need to convert the SourceIndex (dtype=object) to match the type of your index column.
input_df = data=process_tree.head(20)
output_df = ioc_extractor.extract(data=input_df, columns=['NewProcessName', 'CommandLine'])
# set the type of the SourceIndex column. In this case we are matching with the default numeric index.
output_df['SourceIndex'] = pd.to_numeric(output_df['SourceIndex'])
merged_df = pd.merge(left=input_df, right=output_df, how='outer', left_index=True, right_on='SourceIndex')
merged_df.head()
Unnamed: 0 | TenantId | Account | EventID | TimeGenerated | Computer | SubjectUserSid | SubjectUserName | SubjectDomainName | SubjectLogonId | NewProcessId | NewProcessName | TokenElevationType | ProcessId | CommandLine | ParentProcessName | TargetLogonId | SourceComputerId | TimeCreatedUtc | NodeRole | Level | ProcessId1 | NewProcessId1 | IoCType | Observable | SourceIndex | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 0 | 802d39e1-9d70-404d-832c-2de5e2478eda | MSTICAlertsWin1\MSTICAdmin | 4688 | 2019-01-15 05:15:15.677 | MSTICAlertsWin1 | S-1-5-21-996632719-2361334927-4038480536-500 | MSTICAdmin | MSTICAlertsWin1 | 0xfaac27 | 0x1580 | C:\Diagnostics\UserTmp\ftp.exe | %%1936 | 0xbc8 | .\ftp -s:C:\RECYCLER\xxppyy.exe | C:\Windows\System32\cmd.exe | 0x0 | 46fe7078-61bb-4bed-9430-7ac01d91c273 | 2019-01-15 05:15:15.677 | source | 0 | NaN | NaN | windows_path | C:\Diagnostics\UserTmp\ftp.exe | 0 |
1 | 0 | 802d39e1-9d70-404d-832c-2de5e2478eda | MSTICAlertsWin1\MSTICAdmin | 4688 | 2019-01-15 05:15:15.677 | MSTICAlertsWin1 | S-1-5-21-996632719-2361334927-4038480536-500 | MSTICAdmin | MSTICAlertsWin1 | 0xfaac27 | 0x1580 | C:\Diagnostics\UserTmp\ftp.exe | %%1936 | 0xbc8 | .\ftp -s:C:\RECYCLER\xxppyy.exe | C:\Windows\System32\cmd.exe | 0x0 | 46fe7078-61bb-4bed-9430-7ac01d91c273 | 2019-01-15 05:15:15.677 | source | 0 | NaN | NaN | windows_path | C:\RECYCLER\xxppyy.exe | 0 |
2 | 0 | 802d39e1-9d70-404d-832c-2de5e2478eda | MSTICAlertsWin1\MSTICAdmin | 4688 | 2019-01-15 05:15:15.677 | MSTICAlertsWin1 | S-1-5-21-996632719-2361334927-4038480536-500 | MSTICAdmin | MSTICAlertsWin1 | 0xfaac27 | 0x1580 | C:\Diagnostics\UserTmp\ftp.exe | %%1936 | 0xbc8 | .\ftp -s:C:\RECYCLER\xxppyy.exe | C:\Windows\System32\cmd.exe | 0x0 | 46fe7078-61bb-4bed-9430-7ac01d91c273 | 2019-01-15 05:15:15.677 | source | 0 | NaN | NaN | windows_path | .\ftp | 0 |
3 | 1 | 802d39e1-9d70-404d-832c-2de5e2478eda | MSTICAlertsWin1\MSTICAdmin | 4688 | 2019-01-15 05:15:16.167 | MSTICAlertsWin1 | S-1-5-21-996632719-2361334927-4038480536-500 | MSTICAdmin | MSTICAlertsWin1 | 0xfaac27 | 0x16fc | C:\Diagnostics\UserTmp\reg.exe | %%1936 | 0xbc8 | .\reg not /domain:everything that /sid:shines is /krbtgt:golden ! | C:\Windows\System32\cmd.exe | 0x0 | 46fe7078-61bb-4bed-9430-7ac01d91c273 | 2019-01-15 05:15:16.167 | sibling | 1 | NaN | NaN | windows_path | C:\Diagnostics\UserTmp\reg.exe | 1 |
4 | 1 | 802d39e1-9d70-404d-832c-2de5e2478eda | MSTICAlertsWin1\MSTICAdmin | 4688 | 2019-01-15 05:15:16.167 | MSTICAlertsWin1 | S-1-5-21-996632719-2361334927-4038480536-500 | MSTICAdmin | MSTICAlertsWin1 | 0xfaac27 | 0x16fc | C:\Diagnostics\UserTmp\reg.exe | %%1936 | 0xbc8 | .\reg not /domain:everything that /sid:shines is /krbtgt:golden ! | C:\Windows\System32\cmd.exe | 0x0 | 46fe7078-61bb-4bed-9430-7ac01d91c273 | 2019-01-15 05:15:16.167 | sibling | 1 | NaN | NaN | windows_path | .\reg | 1 |