diff options
Diffstat (limited to 'python-msticpy.spec')
| -rw-r--r-- | python-msticpy.spec | 711 |
1 files changed, 711 insertions, 0 deletions
diff --git a/python-msticpy.spec b/python-msticpy.spec new file mode 100644 index 0000000..64e7501 --- /dev/null +++ b/python-msticpy.spec @@ -0,0 +1,711 @@ +%global _empty_manifest_terminate_build 0 +Name: python-msticpy +Version: 2.4.0 +Release: 1 +Summary: MSTIC Security Tools +License: MIT License +URL: https://github.com/microsoft/msticpy +Source0: https://mirrors.nju.edu.cn/pypi/web/packages/a7/4a/9fcd9bfc0bd754b84043a15e343f1fffd4d8c4580458bc79394a21104f60/msticpy-2.4.0.tar.gz +BuildArch: noarch + +Requires: python3-attrs +Requires: python3-azure-common +Requires: python3-azure-core +Requires: python3-azure-identity +Requires: python3-azure-mgmt-subscription +Requires: python3-beautifulsoup4 +Requires: python3-bokeh +Requires: python3-cryptography +Requires: python3-deprecated +Requires: python3-dnspython +Requires: python3-folium +Requires: python3-geoip2 +Requires: python3-httpx +Requires: python3-html5lib +Requires: python3-ipywidgets +Requires: python3-KqlmagicCustom[auth_code_clipboard,jupyter-basic] +Requires: python3-lxml +Requires: python3-matplotlib +Requires: python3-msal +Requires: python3-msal-extensions +Requires: python3-msrest +Requires: python3-msrestazure +Requires: python3-nest-asyncio +Requires: python3-networkx +Requires: python3-numpy +Requires: python3-pandas +Requires: python3-pygments +Requires: python3-pyjwt +Requires: python3-dateutil +Requires: python3-pytz +Requires: python3-pyyaml +Requires: python3-setuptools +Requires: python3-tldextract +Requires: python3-tqdm +Requires: python3-typing-extensions +Requires: python3-urllib3 +Requires: python3-ipython +Requires: python3-ipython +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-storage-blob +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-KqlmagicCustom[jupyter-extended] +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-keyvault +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-azure-storage-blob +Requires: python3-keyring +Requires: python3-mo-sql-parsing +Requires: python3-nest-asyncio +Requires: python3-openpyxl +Requires: python3-passivetotal +Requires: python3-scikit-learn +Requires: python3-scipy +Requires: python3-splunk-sdk +Requires: python3-statsmodels +Requires: python3-sumologic-sdk +Requires: python3-vt-graph-api +Requires: python3-vt-py +Requires: python3-KqlmagicCustom[jupyter-extended] +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-keyvault +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-azure-storage-blob +Requires: python3-keyring +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-keyvault +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-azure-storage-blob +Requires: python3-keyring +Requires: python3-KqlmagicCustom[jupyter-extended] +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-keyvault +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-azure-storage-blob +Requires: python3-keyring +Requires: python3-aiohttp +Requires: python3-async-cache +Requires: python3-bandit +Requires: python3-beautifulsoup4 +Requires: python3-black +Requires: python3-coverage +Requires: python3-docutils +Requires: python3-filelock +Requires: python3-flake8 +Requires: python3-isort +Requires: python3-markdown +Requires: python3-mccabe +Requires: python3-mypy +Requires: python3-nbdime +Requires: python3-nbconvert +Requires: python3-pandas +Requires: python3-pep8-naming +Requires: python3-pep8 +Requires: python3-pipreqs +Requires: python3-pre-commit +Requires: python3-pycodestyle +Requires: python3-pydocstyle +Requires: python3-pyflakes +Requires: python3-pygeohash +Requires: python3-pylint +Requires: python3-pyroma +Requires: python3-pytest-check +Requires: python3-pytest-cov +Requires: python3-pytest-xdist +Requires: python3-pytest +Requires: python3-readthedocs-sphinx-ext +Requires: python3-responses +Requires: python3-respx +Requires: python3-sphinx-rtd-theme +Requires: python3-sphinx +Requires: python3-types-attrs +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-keyvault +Requires: python3-keyring +Requires: python3-KqlmagicCustom[jupyter-extended] +Requires: python3-scikit-learn +Requires: python3-scipy +Requires: python3-statsmodels +Requires: python3-passivetotal +Requires: python3-KqlmagicCustom[jupyter-extended] +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-keyvault +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-azure-storage-blob +Requires: python3-keyring +Requires: python3-splunk-sdk +Requires: python3-mo-sql-parsing +Requires: python3-sumologic-sdk +Requires: python3-openpyxl +Requires: python3-KqlmagicCustom[jupyter-extended] +Requires: python3-aiohttp +Requires: python3-async-cache +Requires: python3-azure-keyvault-secrets +Requires: python3-azure-mgmt-compute +Requires: python3-azure-mgmt-core +Requires: python3-azure-mgmt-keyvault +Requires: python3-azure-mgmt-monitor +Requires: python3-azure-mgmt-network +Requires: python3-azure-mgmt-resource +Requires: python3-azure-mgmt-resourcegraph +Requires: python3-azure-storage-blob +Requires: python3-bandit +Requires: python3-beautifulsoup4 +Requires: python3-black +Requires: python3-coverage +Requires: python3-docutils +Requires: python3-filelock +Requires: python3-flake8 +Requires: python3-isort +Requires: python3-keyring +Requires: python3-markdown +Requires: python3-mccabe +Requires: python3-mo-sql-parsing +Requires: python3-mypy +Requires: python3-nbconvert +Requires: python3-nbdime +Requires: python3-nest-asyncio +Requires: python3-openpyxl +Requires: python3-pandas +Requires: python3-passivetotal +Requires: python3-pep8-naming +Requires: python3-pep8 +Requires: python3-pipreqs +Requires: python3-pre-commit +Requires: python3-pycodestyle +Requires: python3-pydocstyle +Requires: python3-pyflakes +Requires: python3-pygeohash +Requires: python3-pylint +Requires: python3-pyroma +Requires: python3-pytest-check +Requires: python3-pytest-cov +Requires: python3-pytest-xdist +Requires: python3-pytest +Requires: python3-readthedocs-sphinx-ext +Requires: python3-responses +Requires: python3-respx +Requires: python3-scikit-learn +Requires: python3-scipy +Requires: python3-sphinx-rtd-theme +Requires: python3-sphinx +Requires: python3-splunk-sdk +Requires: python3-statsmodels +Requires: python3-sumologic-sdk +Requires: python3-types-attrs +Requires: python3-vt-graph-api +Requires: python3-vt-py +Requires: python3-vt-py +Requires: python3-vt-graph-api +Requires: python3-nest-asyncio + +%description +## Log Data Acquisition +QueryProvider is an extensible query library targeting Azure Sentinel/Log Analytics, +Splunk, OData +and other log data sources. It also has special support for +[Mordor](https://github.com/OTRF/mordor) data sets and using local data. +Built-in parameterized queries allow complex queries to be run +from a single function call. Add your own queries using a simple YAML +schema. +[Data Queries Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/Data_Queries.ipynb) +## Data Enrichment +### Threat Intelligence providers +The TILookup class can lookup IoCs across multiple TI providers. built-in +providers include AlienVault OTX, IBM XForce, VirusTotal and Azure Sentinel. +The input can be a single IoC observable or a pandas DataFrame containing +multiple observables. Depending on the provider, you may require an account +and an API key. Some providers also enforce throttling (especially for free +tiers), which might affect performing bulk lookups. +[TIProviders](https://msticpy.readthedocs.io/en/latest/data_acquisition/TIProviders.html) +and +[TILookup Usage Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/TIProviders.ipynb) +### GeoLocation Data +The GeoIP lookup classes allow you to match the geo-locations of IP addresses +using either: +- GeoLiteLookup - Maxmind Geolite (see <https://www.maxmind.com>) +- IPStackLookup - IPStack (see <https://ipstack.com>) +<img src="./docs/source/visualization/_static/folium_sf_zoom.png" + alt="Folium map" + title="Plotting Geo IP Location" height="200" /> +[GeoIP Lookup](https://msticpy.readthedocs.io/en/latest/data_acquisition/GeoIPLookups.html) +and +[GeoIP Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/GeoIPLookups.ipynb) +### Azure Resource Data, Storage and Azure Sentinel API +The AzureData module contains functionality for enriching data regarding Azure host +details with additional host details exposed via the Azure API. The AzureSentinel +module allows you to query incidents, retrieve detector and hunting +queries. AzureBlogStorage lets you read and write data from blob storage. +[Azure Resource APIs](https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureData.html), +[Azure Sentinel APIs](https://msticpy.readthedocs.io/en/latest/data_acquisition/Sentinel.html), +[Azure Storage](https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureBlobStorage.html) +## Security Analysis +This subpackage contains several modules helpful for working on security investigations and hunting: +### Anomalous Sequence Detection +Detect unusual sequences of events in your Office, Active Directory or other log data. +You can extract sessions (e.g. activity initiated by the same account) and identify and +visualize unusual sequences of activity. For example, detecting an attacker setting +a mail forwarding rule on someone's mailbox. +[Anomalous Sessions](https://msticpy.readthedocs.io/en/latest/data_analysis/AnomalousSequence.html) +and +[Anomalous Sequence Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/AnomalousSequence.ipynb) +### Time Series Analysis +Time series analysis allows you to identify unusual patterns in your log data +taking into account normal seasonal variations (e.g. the regular ebb and flow of +events over hours of the day, days of the week, etc.). Using both analysis and +visualization highlights unusual traffic flows or event activity for any data +set. +<img src="./docs/source/visualization/_static/TimeSeriesAnomalieswithRangeTool.png" +alt="Time Series anomalies" title="Time Series anomalies" height="300" /> +[Time Series](https://msticpy.readthedocs.io/en/latest/visualization/TimeSeriesAnomalies.html) +## Visualization +### Event Timelines +Display any log events on an interactive timeline. Using the +[Bokeh Visualization Library](https://bokeh.org/) the timeline control enables +you to visualize one or more event streams, interactively zoom into specific time +slots and view event details for plotted events. +<img src="./docs/source/visualization/_static/TimeLine-01.png" +alt="Timeline" title="Msticpy Timeline Control" height="300" /> +[Timeline](https://msticpy.readthedocs.io/en/latest/visualization/EventTimeline.html) +and +[Timeline Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/EventTimeline.ipynb) +### Process Trees +The process tree functionality has two main components: +- Process Tree creation - taking a process creation log from a host and building + the parent-child relationships between processes in the data set. +- Process Tree visualization - this takes the processed output displays an interactive process tree using Bokeh plots. +There are a set of utility functions to extract individual and partial trees from the processed data set. +<img src="./docs/source/visualization/_static/process_tree3.png" +alt="Process Tree" +title="Interactive Process Tree" height="400" /> +[Process Tree](https://msticpy.readthedocs.io/en/latest/visualization/ProcessTree.html) +and +[Process Tree Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/ProcessTree.ipynb) +## Data Manipulation and Utility functions +### Pivot Functions +Lets you use *MSTICPy* functionality in an "entity-centric" way. +All functions, queries and lookups that relate to a particular entity type +(e.g. Host, IpAddress, Url) are collected together as methods of that +entity class. So, if you want to do things with an IP address, just load +the IpAddress entity and browse its methods. +[Pivot Functions](https://msticpy.readthedocs.io/en/latest/data_analysis/PivotFunctions.html) +and +[Pivot Functions Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/PivotFunctions.ipynb) +### base64unpack +Base64 and archive (gz, zip, tar) extractor. It will try to identify any base64 encoded +strings and try decode them. If the result looks like one of the supported archive types it +will unpack the contents. The results of each decode/unpack are rechecked for further +base64 content and up to a specified depth. +[Base64 Decoding](https://msticpy.readthedocs.io/en/latest/data_analysis/Base64Unpack.html) +and +[Base64Unpack Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/Base64Unpack.ipynb) +### iocextract +Uses regular expressions to look for Indicator of Compromise (IoC) patterns - IP Addresses, URLs, +DNS domains, Hashes, file paths. +Input can be a single string or a pandas dataframe. +[IoC Extraction](https://msticpy.readthedocs.io/en/latest/data_analysis/IoCExtract.html) +and +[IoCExtract Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/IoCExtract.ipynb) +### eventcluster (experimental) +This module is intended to be used to summarize large numbers of +events into clusters of different patterns. High volume repeating +events can often make it difficult to see unique and interesting items. +<img src="./docs/source/data_analysis/_static/EventClustering_2a.png" + alt="Clustering" + title="Clustering based on command-line variability" height="400" /> +This is an unsupervised learning module implemented using SciKit Learn DBScan. +[Event Clustering](https://msticpy.readthedocs.io/en/latest/data_analysis/EventClustering.html) +and +[Event Clustering Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/EventClustering.ipynb) +### auditdextract +Module to load and decode Linux audit logs. It collapses messages sharing the same +message ID into single events, decodes hex-encoded data fields and performs some +event-specific formatting and normalization (e.g. for process start events it will +re-assemble the process command line arguments into a single string). +### syslog_utils +Module to support an investigation of a Linux host with only syslog logging enabled. +This includes functions for collating host data, clustering logon events and detecting +user sessions containing suspicious activity. +### cmd_line +A module to support the detection of known malicious command line activity or suspicious +patterns of command line activity. +### domain_utils +A module to support investigation of domain names and URLs with functions to +validate a domain name and screenshot a URL. +### Notebook widgets +These are built from the [Jupyter ipywidgets](https://ipywidgets.readthedocs.io/) collection +and group common functionality useful in InfoSec tasks such as list pickers, +query time boundary settings and event display into an easy-to-use format. +<img src="./docs/source/visualization/_static/Widgets1.png" + alt="Time span Widget" + title="Query time setter" height="100" /> +<img src="./docs/source/visualization/_static/Widgets4.png" + alt="Alert browser" + +%package -n python3-msticpy +Summary: MSTIC Security Tools +Provides: python-msticpy +BuildRequires: python3-devel +BuildRequires: python3-setuptools +BuildRequires: python3-pip +%description -n python3-msticpy +## Log Data Acquisition +QueryProvider is an extensible query library targeting Azure Sentinel/Log Analytics, +Splunk, OData +and other log data sources. It also has special support for +[Mordor](https://github.com/OTRF/mordor) data sets and using local data. +Built-in parameterized queries allow complex queries to be run +from a single function call. Add your own queries using a simple YAML +schema. +[Data Queries Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/Data_Queries.ipynb) +## Data Enrichment +### Threat Intelligence providers +The TILookup class can lookup IoCs across multiple TI providers. built-in +providers include AlienVault OTX, IBM XForce, VirusTotal and Azure Sentinel. +The input can be a single IoC observable or a pandas DataFrame containing +multiple observables. Depending on the provider, you may require an account +and an API key. Some providers also enforce throttling (especially for free +tiers), which might affect performing bulk lookups. +[TIProviders](https://msticpy.readthedocs.io/en/latest/data_acquisition/TIProviders.html) +and +[TILookup Usage Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/TIProviders.ipynb) +### GeoLocation Data +The GeoIP lookup classes allow you to match the geo-locations of IP addresses +using either: +- GeoLiteLookup - Maxmind Geolite (see <https://www.maxmind.com>) +- IPStackLookup - IPStack (see <https://ipstack.com>) +<img src="./docs/source/visualization/_static/folium_sf_zoom.png" + alt="Folium map" + title="Plotting Geo IP Location" height="200" /> +[GeoIP Lookup](https://msticpy.readthedocs.io/en/latest/data_acquisition/GeoIPLookups.html) +and +[GeoIP Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/GeoIPLookups.ipynb) +### Azure Resource Data, Storage and Azure Sentinel API +The AzureData module contains functionality for enriching data regarding Azure host +details with additional host details exposed via the Azure API. The AzureSentinel +module allows you to query incidents, retrieve detector and hunting +queries. AzureBlogStorage lets you read and write data from blob storage. +[Azure Resource APIs](https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureData.html), +[Azure Sentinel APIs](https://msticpy.readthedocs.io/en/latest/data_acquisition/Sentinel.html), +[Azure Storage](https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureBlobStorage.html) +## Security Analysis +This subpackage contains several modules helpful for working on security investigations and hunting: +### Anomalous Sequence Detection +Detect unusual sequences of events in your Office, Active Directory or other log data. +You can extract sessions (e.g. activity initiated by the same account) and identify and +visualize unusual sequences of activity. For example, detecting an attacker setting +a mail forwarding rule on someone's mailbox. +[Anomalous Sessions](https://msticpy.readthedocs.io/en/latest/data_analysis/AnomalousSequence.html) +and +[Anomalous Sequence Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/AnomalousSequence.ipynb) +### Time Series Analysis +Time series analysis allows you to identify unusual patterns in your log data +taking into account normal seasonal variations (e.g. the regular ebb and flow of +events over hours of the day, days of the week, etc.). Using both analysis and +visualization highlights unusual traffic flows or event activity for any data +set. +<img src="./docs/source/visualization/_static/TimeSeriesAnomalieswithRangeTool.png" +alt="Time Series anomalies" title="Time Series anomalies" height="300" /> +[Time Series](https://msticpy.readthedocs.io/en/latest/visualization/TimeSeriesAnomalies.html) +## Visualization +### Event Timelines +Display any log events on an interactive timeline. Using the +[Bokeh Visualization Library](https://bokeh.org/) the timeline control enables +you to visualize one or more event streams, interactively zoom into specific time +slots and view event details for plotted events. +<img src="./docs/source/visualization/_static/TimeLine-01.png" +alt="Timeline" title="Msticpy Timeline Control" height="300" /> +[Timeline](https://msticpy.readthedocs.io/en/latest/visualization/EventTimeline.html) +and +[Timeline Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/EventTimeline.ipynb) +### Process Trees +The process tree functionality has two main components: +- Process Tree creation - taking a process creation log from a host and building + the parent-child relationships between processes in the data set. +- Process Tree visualization - this takes the processed output displays an interactive process tree using Bokeh plots. +There are a set of utility functions to extract individual and partial trees from the processed data set. +<img src="./docs/source/visualization/_static/process_tree3.png" +alt="Process Tree" +title="Interactive Process Tree" height="400" /> +[Process Tree](https://msticpy.readthedocs.io/en/latest/visualization/ProcessTree.html) +and +[Process Tree Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/ProcessTree.ipynb) +## Data Manipulation and Utility functions +### Pivot Functions +Lets you use *MSTICPy* functionality in an "entity-centric" way. +All functions, queries and lookups that relate to a particular entity type +(e.g. Host, IpAddress, Url) are collected together as methods of that +entity class. So, if you want to do things with an IP address, just load +the IpAddress entity and browse its methods. +[Pivot Functions](https://msticpy.readthedocs.io/en/latest/data_analysis/PivotFunctions.html) +and +[Pivot Functions Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/PivotFunctions.ipynb) +### base64unpack +Base64 and archive (gz, zip, tar) extractor. It will try to identify any base64 encoded +strings and try decode them. If the result looks like one of the supported archive types it +will unpack the contents. The results of each decode/unpack are rechecked for further +base64 content and up to a specified depth. +[Base64 Decoding](https://msticpy.readthedocs.io/en/latest/data_analysis/Base64Unpack.html) +and +[Base64Unpack Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/Base64Unpack.ipynb) +### iocextract +Uses regular expressions to look for Indicator of Compromise (IoC) patterns - IP Addresses, URLs, +DNS domains, Hashes, file paths. +Input can be a single string or a pandas dataframe. +[IoC Extraction](https://msticpy.readthedocs.io/en/latest/data_analysis/IoCExtract.html) +and +[IoCExtract Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/IoCExtract.ipynb) +### eventcluster (experimental) +This module is intended to be used to summarize large numbers of +events into clusters of different patterns. High volume repeating +events can often make it difficult to see unique and interesting items. +<img src="./docs/source/data_analysis/_static/EventClustering_2a.png" + alt="Clustering" + title="Clustering based on command-line variability" height="400" /> +This is an unsupervised learning module implemented using SciKit Learn DBScan. +[Event Clustering](https://msticpy.readthedocs.io/en/latest/data_analysis/EventClustering.html) +and +[Event Clustering Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/EventClustering.ipynb) +### auditdextract +Module to load and decode Linux audit logs. It collapses messages sharing the same +message ID into single events, decodes hex-encoded data fields and performs some +event-specific formatting and normalization (e.g. for process start events it will +re-assemble the process command line arguments into a single string). +### syslog_utils +Module to support an investigation of a Linux host with only syslog logging enabled. +This includes functions for collating host data, clustering logon events and detecting +user sessions containing suspicious activity. +### cmd_line +A module to support the detection of known malicious command line activity or suspicious +patterns of command line activity. +### domain_utils +A module to support investigation of domain names and URLs with functions to +validate a domain name and screenshot a URL. +### Notebook widgets +These are built from the [Jupyter ipywidgets](https://ipywidgets.readthedocs.io/) collection +and group common functionality useful in InfoSec tasks such as list pickers, +query time boundary settings and event display into an easy-to-use format. +<img src="./docs/source/visualization/_static/Widgets1.png" + alt="Time span Widget" + title="Query time setter" height="100" /> +<img src="./docs/source/visualization/_static/Widgets4.png" + alt="Alert browser" + +%package help +Summary: Development documents and examples for msticpy +Provides: python3-msticpy-doc +%description help +## Log Data Acquisition +QueryProvider is an extensible query library targeting Azure Sentinel/Log Analytics, +Splunk, OData +and other log data sources. It also has special support for +[Mordor](https://github.com/OTRF/mordor) data sets and using local data. +Built-in parameterized queries allow complex queries to be run +from a single function call. Add your own queries using a simple YAML +schema. +[Data Queries Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/Data_Queries.ipynb) +## Data Enrichment +### Threat Intelligence providers +The TILookup class can lookup IoCs across multiple TI providers. built-in +providers include AlienVault OTX, IBM XForce, VirusTotal and Azure Sentinel. +The input can be a single IoC observable or a pandas DataFrame containing +multiple observables. Depending on the provider, you may require an account +and an API key. Some providers also enforce throttling (especially for free +tiers), which might affect performing bulk lookups. +[TIProviders](https://msticpy.readthedocs.io/en/latest/data_acquisition/TIProviders.html) +and +[TILookup Usage Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/TIProviders.ipynb) +### GeoLocation Data +The GeoIP lookup classes allow you to match the geo-locations of IP addresses +using either: +- GeoLiteLookup - Maxmind Geolite (see <https://www.maxmind.com>) +- IPStackLookup - IPStack (see <https://ipstack.com>) +<img src="./docs/source/visualization/_static/folium_sf_zoom.png" + alt="Folium map" + title="Plotting Geo IP Location" height="200" /> +[GeoIP Lookup](https://msticpy.readthedocs.io/en/latest/data_acquisition/GeoIPLookups.html) +and +[GeoIP Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/GeoIPLookups.ipynb) +### Azure Resource Data, Storage and Azure Sentinel API +The AzureData module contains functionality for enriching data regarding Azure host +details with additional host details exposed via the Azure API. The AzureSentinel +module allows you to query incidents, retrieve detector and hunting +queries. AzureBlogStorage lets you read and write data from blob storage. +[Azure Resource APIs](https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureData.html), +[Azure Sentinel APIs](https://msticpy.readthedocs.io/en/latest/data_acquisition/Sentinel.html), +[Azure Storage](https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureBlobStorage.html) +## Security Analysis +This subpackage contains several modules helpful for working on security investigations and hunting: +### Anomalous Sequence Detection +Detect unusual sequences of events in your Office, Active Directory or other log data. +You can extract sessions (e.g. activity initiated by the same account) and identify and +visualize unusual sequences of activity. For example, detecting an attacker setting +a mail forwarding rule on someone's mailbox. +[Anomalous Sessions](https://msticpy.readthedocs.io/en/latest/data_analysis/AnomalousSequence.html) +and +[Anomalous Sequence Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/AnomalousSequence.ipynb) +### Time Series Analysis +Time series analysis allows you to identify unusual patterns in your log data +taking into account normal seasonal variations (e.g. the regular ebb and flow of +events over hours of the day, days of the week, etc.). Using both analysis and +visualization highlights unusual traffic flows or event activity for any data +set. +<img src="./docs/source/visualization/_static/TimeSeriesAnomalieswithRangeTool.png" +alt="Time Series anomalies" title="Time Series anomalies" height="300" /> +[Time Series](https://msticpy.readthedocs.io/en/latest/visualization/TimeSeriesAnomalies.html) +## Visualization +### Event Timelines +Display any log events on an interactive timeline. Using the +[Bokeh Visualization Library](https://bokeh.org/) the timeline control enables +you to visualize one or more event streams, interactively zoom into specific time +slots and view event details for plotted events. +<img src="./docs/source/visualization/_static/TimeLine-01.png" +alt="Timeline" title="Msticpy Timeline Control" height="300" /> +[Timeline](https://msticpy.readthedocs.io/en/latest/visualization/EventTimeline.html) +and +[Timeline Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/EventTimeline.ipynb) +### Process Trees +The process tree functionality has two main components: +- Process Tree creation - taking a process creation log from a host and building + the parent-child relationships between processes in the data set. +- Process Tree visualization - this takes the processed output displays an interactive process tree using Bokeh plots. +There are a set of utility functions to extract individual and partial trees from the processed data set. +<img src="./docs/source/visualization/_static/process_tree3.png" +alt="Process Tree" +title="Interactive Process Tree" height="400" /> +[Process Tree](https://msticpy.readthedocs.io/en/latest/visualization/ProcessTree.html) +and +[Process Tree Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/ProcessTree.ipynb) +## Data Manipulation and Utility functions +### Pivot Functions +Lets you use *MSTICPy* functionality in an "entity-centric" way. +All functions, queries and lookups that relate to a particular entity type +(e.g. Host, IpAddress, Url) are collected together as methods of that +entity class. So, if you want to do things with an IP address, just load +the IpAddress entity and browse its methods. +[Pivot Functions](https://msticpy.readthedocs.io/en/latest/data_analysis/PivotFunctions.html) +and +[Pivot Functions Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/PivotFunctions.ipynb) +### base64unpack +Base64 and archive (gz, zip, tar) extractor. It will try to identify any base64 encoded +strings and try decode them. If the result looks like one of the supported archive types it +will unpack the contents. The results of each decode/unpack are rechecked for further +base64 content and up to a specified depth. +[Base64 Decoding](https://msticpy.readthedocs.io/en/latest/data_analysis/Base64Unpack.html) +and +[Base64Unpack Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/Base64Unpack.ipynb) +### iocextract +Uses regular expressions to look for Indicator of Compromise (IoC) patterns - IP Addresses, URLs, +DNS domains, Hashes, file paths. +Input can be a single string or a pandas dataframe. +[IoC Extraction](https://msticpy.readthedocs.io/en/latest/data_analysis/IoCExtract.html) +and +[IoCExtract Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/IoCExtract.ipynb) +### eventcluster (experimental) +This module is intended to be used to summarize large numbers of +events into clusters of different patterns. High volume repeating +events can often make it difficult to see unique and interesting items. +<img src="./docs/source/data_analysis/_static/EventClustering_2a.png" + alt="Clustering" + title="Clustering based on command-line variability" height="400" /> +This is an unsupervised learning module implemented using SciKit Learn DBScan. +[Event Clustering](https://msticpy.readthedocs.io/en/latest/data_analysis/EventClustering.html) +and +[Event Clustering Notebook](https://github.com/microsoft/msticpy/blob/master/docs/notebooks/EventClustering.ipynb) +### auditdextract +Module to load and decode Linux audit logs. It collapses messages sharing the same +message ID into single events, decodes hex-encoded data fields and performs some +event-specific formatting and normalization (e.g. for process start events it will +re-assemble the process command line arguments into a single string). +### syslog_utils +Module to support an investigation of a Linux host with only syslog logging enabled. +This includes functions for collating host data, clustering logon events and detecting +user sessions containing suspicious activity. +### cmd_line +A module to support the detection of known malicious command line activity or suspicious +patterns of command line activity. +### domain_utils +A module to support investigation of domain names and URLs with functions to +validate a domain name and screenshot a URL. +### Notebook widgets +These are built from the [Jupyter ipywidgets](https://ipywidgets.readthedocs.io/) collection +and group common functionality useful in InfoSec tasks such as list pickers, +query time boundary settings and event display into an easy-to-use format. +<img src="./docs/source/visualization/_static/Widgets1.png" + alt="Time span Widget" + title="Query time setter" height="100" /> +<img src="./docs/source/visualization/_static/Widgets4.png" + alt="Alert browser" + +%prep +%autosetup -n msticpy-2.4.0 + +%build +%py3_build + +%install +%py3_install +install -d -m755 %{buildroot}/%{_pkgdocdir} +if [ -d doc ]; then cp -arf doc %{buildroot}/%{_pkgdocdir}; fi +if [ -d docs ]; then cp -arf docs %{buildroot}/%{_pkgdocdir}; fi +if [ -d example ]; then cp -arf example %{buildroot}/%{_pkgdocdir}; fi +if [ -d examples ]; then cp -arf examples %{buildroot}/%{_pkgdocdir}; fi +pushd %{buildroot} +if [ -d usr/lib ]; then + find usr/lib -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/lib64 ]; then + find usr/lib64 -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/bin ]; then + find usr/bin -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/sbin ]; then + find usr/sbin -type f -printf "/%h/%f\n" >> filelist.lst +fi +touch doclist.lst +if [ -d usr/share/man ]; then + find usr/share/man -type f -printf "/%h/%f.gz\n" >> doclist.lst +fi +popd +mv %{buildroot}/filelist.lst . +mv %{buildroot}/doclist.lst . + +%files -n python3-msticpy -f filelist.lst +%dir %{python3_sitelib}/* + +%files help -f doclist.lst +%{_docdir}/* + +%changelog +* Fri May 05 2023 Python_Bot <Python_Bot@openeuler.org> - 2.4.0-1 +- Package Spec generated |