Commit d1f01c8c authored by Nong Hoang Tu's avatar Nong Hoang Tu
Browse files

New upstream version 2.7.0

parents
Pipeline #5980 failed with stages
name: lint_python
on: [pull_request, push]
jobs:
lint_python:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- run: pip install bandit black codespell flake8 isort mypy pytest pyupgrade
- run: bandit --recursive --skip B101 . || true # B101 is assert statements
- run: black --check . || true
- run: codespell || true # --ignore-words-list="" --skip="
- run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --show-source --statistics
- run: isort --check-only --profile black . || true
- run: pip install -r requirements.txt || true
- run: mypy --install-types --non-interactive . || true
- run: pytest . || true
- run: pytest --doctest-modules . || true
- run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
# General
.DS_Store
.AppleDouble
.LSOverride
# Icon must end with two \r
Icon
# Thumbnails
._*
# Files that might appear in the root of a volume
.DocumentRevisions-V100
.fseventsd
.Spotlight-V100
.TemporaryItems
.Trashes
.VolumeIcon.icns
.com.apple.timemachine.donotpresent
# Directories potentially created on remote AFP share
.AppleDB
.AppleDesktop
Network Trash Folder
Temporary Items
.apdisk
TODO
detected_events.json
tmp
*.log
*.evtx
*.evtx_data
*.tar
*.tar.sha256
# Since `evtx_dump` precompiled binaries are not shipped with musl support, we need to use the
# Debian-based Python image instead of the Alpine-based image, which increases the size of the
# final image (~70 MB overhead).
#
ARG PYTHON_VERSION="3.9-slim"
FROM "python:${PYTHON_VERSION}" as stage
ARG ZIRCOLITE_INSTALL_PREFIX="/opt"
ARG ZIRCOLITE_REPOSITORY_URI="https://github.com/wagga40/Zircolite.git"
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive \
apt-get install --yes --no-install-recommends \
git && \
apt-get autoremove --purge --yes && \
rm -rf /var/lib/apt/lists/*
WORKDIR "${ZIRCOLITE_INSTALL_PREFIX}"
RUN git clone \
"${ZIRCOLITE_REPOSITORY_URI}" \
zircolite
WORKDIR "${ZIRCOLITE_INSTALL_PREFIX}/zircolite"
RUN chmod 0755 \
zircolite.py
FROM "python:${PYTHON_VERSION}"
LABEL author="wagga40"
LABEL description="A standalone SIGMA-based detection tool for EVTX."
LABEL maintainer="wagga40"
ARG ZIRCOLITE_INSTALL_PREFIX="/opt"
WORKDIR "${ZIRCOLITE_INSTALL_PREFIX}"
COPY --chown=root:root --from=stage \
"${ZIRCOLITE_INSTALL_PREFIX}/zircolite" \
zircolite
WORKDIR "${ZIRCOLITE_INSTALL_PREFIX}/zircolite"
RUN python3 -m pip install \
--requirement requirements.txt
ENTRYPOINT [ "python3", "zircolite.py" ]
CMD [ "--help" ]
#!make
DOCKER?=docker
DOCKER_BUILD_FLAGS?=
DOCKER_REGISTRY?=docker.io
DOCKER_TAG?=2.0.0
GIT?=git
PY3?=python3
DATE=$(shell date +%s)
TMP_GIT=tmp-$(shell date +%s)
define HELP_MENU
Usage: make [<env>] <target> [<target> ...]
Main targets:
all (default) call the default target(s)
build build the Docker image
clean remove all default artifacts
help show this help
save save the Docker image to an archive
rulesets update default rulesets (files will be created in current directory)
Refer to the documentation for use cases and examples.
endef
.PHONY: all build clean help save
all: clean
build:
ifndef DOCKER
$(error Docker (https://docs.docker.com/install/) is required. Please install it first)
endif
$(DOCKER) image build \
--rm \
--tag $(DOCKER_REGISTRY)/wagga40/zircolite:$(DOCKER_TAG) \
$(DOCKER_BUILD_FLAGS) \
.
help:
$(info $(HELP_MENU))
clean:
rm -rf "detected_events.json"
rm -rf ./tmp-*
rm -f zircolite.log
rm -f fields.json
rm -f zircolite.tar
save:
ifndef DOCKER
$(error Docker (https://docs.docker.com/install/) is required. Please install it first)
endif
$(DOCKER) image save \
--output zircolite.tar \
$(DOCKER_REGISTRY)/wagga40/zircolite:$(DOCKER_TAG)
rulesets:
$(info Please check docs to generate rulesets directly with sigmatools and sigmac : https://github.com/wagga40/Zircolite/blob/master/docs/Usage.md#with-sigmatools)
# <p align="center">![](pics/zircolite_400.png)</p>
## Standalone SIGMA-based detection tool for EVTX, Auditd, Sysmon for linux or JSONL/NDJSON Logs
![](pics/Zircolite.svg)
[![python](https://img.shields.io/badge/python-3.8-blue)](https://www.python.org/)
![version](https://img.shields.io/badge/Platform-Win-green)
![version](https://img.shields.io/badge/Platform-Lin-green)
![version](https://img.shields.io/badge/Platform-Mac-green)
![version](https://img.shields.io/badge/Architecture-64bit-red)
**Zircolite is a standalone tool written in Python 3. It allows to use SIGMA rules on MS Windows EVTX (EVTX and JSONL format), Auditd logs and Sysmon for Linux logs**
- **Zircolite** can be used directly on the investigated endpoint (use [releases](https://github.com/wagga40/Zircolite/releases)) or in your forensic/detection lab
- **Zircolite** is fast and can parse large datasets in just seconds (check [benchmarks](docs/Internals.md#benchmarks))
**Zircolite can be used directly in Python or you can use the binaries provided in [releases](https://github.com/wagga40/Zircolite/releases) (Microsoft Windows and Linux only).**
**Documentation is [here](docs).**
## Requirements / Installation
You can install dependencies with : `pip3 install -r requirements.txt`
The use of [evtx_dump](https://github.com/omerbenamram/evtx) is **optional but required by default (because it is for now much faster)**, If you do not want to use it you have to use the `--noexternal` option. The tool is provided if you clone the Zircolite repository (the official repository is [here](https://github.com/omerbenamram/evtx)).
## Quick start
#### EVTX files :
Help is available with `zircolite.py -h`. If your EVTX files have the extension ".evtx" :
```shell
python3 zircolite.py --evtx <EVTX_FOLDER/EVTX_FILE> --ruleset <Converted Sigma rules>
python3 zircolite.py --evtx sysmon.evtx --ruleset rules/rules_windows_sysmon.json
```
The SYSMON ruleset used here is a default one and it is for logs coming from endpoints where SYSMON installed. A generic ruleset is available too.
#### Auditd logs :
```shell
python3 zircolite.py --evtx <EVTX_FOLDER/EVTX_FILE> --ruleset <Converted Sigma rules> --auditd
python3 zircolite.py --evtx auditd.log --ruleset rules/rules_linux.json --auditd
```
#### Sysmon for Linux logs :
```shell
python3 zircolite.py --evtx <EVTX_FOLDER/EVTX_FILE> --ruleset <Converted Sigma rules> --sysmon4linux
python3 zircolite.py --evtx auditd.log --ruleset rules/rules_linux.json --sysmon4linux
```
#### JSONL/NDJSON files :
```shell
python3 zircolite.py --evtx <JSON_FOLDER/JSON_FILE> --ruleset rules/rules_windows_sysmon.json --jsononly
```
:information_source: If you want to try the tool you can test with these samples :
- [EVTX-ATTACK-SAMPLES](https://github.com/sbousseaden/EVTX-ATTACK-SAMPLES) (EVTX Files)
- [MORDOR - APT29](https://github.com/OTRF/Security-Datasets/tree/master/datasets/compound/apt29) (JSONL Files)
- [MORDOR - APT3](https://github.com/OTRF/Security-Datasets/tree/master/datasets/compound/windows/apt3) (JSONL Files)
## Docs
Everything is [here](docs).
## Tutorials, references and related projects
### Tutorials
- [Russ McRee](https://holisticinfosec.io) has published a pretty good [tutorial](https://holisticinfosec.io/post/2021-09-28-zircolite/) on SIGMA and **Zircolite** in his [blog](https://holisticinfosec.io/post/2021-09-28-zircolite/)
- **César Marín** has published a tutorial in **spanish** [here](https://derechodelared.com/zircolite-ejecucion-de-reglas-sigma-en-ficheros-evtx/)
### EU ATT&CK Workshop October 2021
[Florian Roth](https://github.com/Neo23x0/) cited **Zircolite** in his [**SIGMA Hall of fame**](https://github.com/Neo23x0/Talks/blob/master/Sigma_Hall_of_Fame_20211022.pdf) in its talk dugin the October 2021 EU ATT&CK Workshop.
### Related projects
[Michel de CREVOISIER](https://github.com/mdecrevoisier) is doing an amazing work with SIGMA, MITRE Att&ck (c) and other projects. Check [his work on mapping EVTX on the MITRE Att&ck (c) framework](https://github.com/mdecrevoisier/EVTX-to-MITRE-Attack).
## Mini-Gui
![](pics/gui.jpg)
The Mini-GUI can be used totally offline, it allows the user to display and search results. To know how to use the Mini-GUI, check docs [here](docs).
## Battle-tested
Zircolite has been used to perform cold-analysis (in Lab) on EVTX in multiple "real-life" situations.
However, even if Zircolite has been used many times to perform analysis directly on a Microsoft Windows endpoint, there is not yet a pipeline to thoroughly test every release.
## License
- All the **code** of the project is licensed under the [GNU Lesser General Public License](https://www.gnu.org/licenses/lgpl-3.0.en.html)
- `evtx_dump` is under the MIT license
- The rules are released under the [Detection Rule License (DRL) 1.0](https://github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md)
{
"exclusions" : ["xmlns"],
"useless" : [null, ""],
"mappings" :
{
"Event.EventData.UserData" : "UserData",
"Event.System.Provider.#attributes.Guid" : "Guid",
"Event.EventData.ContextInfo": "ContextInfo",
"Event.System.Execution.#attributes.ProcessID": "ProcessID",
"Event.System.Execution.#attributes.ThreadID": "ThreadID",
"Event.System.EventID" : "EventID",
"Event.System.EventID.#text" : "EventID",
"Event.System.Channel":"Channel",
"Event.System.Computer":"Computer",
"Event.System.Correlation":"Correlation",
"Event.System.Correlation.#attributes.ActivityID":"ActivityID",
"Event.System.EventID.#attributes.Qualifiers":"Qualifiers",
"Event.System.EventRecordID":"EventRecordID",
"Event.System.Keywords":"Keywords",
"Event.System.Level":"Level",
"Event.System.Opcode":"Opcode",
"Event.System.Provider.#attributes.EventSourceName":"EventSourceName",
"Event.System.Provider.#attributes.Name" : "Provider_Name",
"Event.System.Security":"Security",
"Event.System.Security.#attributes.UserID":"UserID",
"Event.System.Task":"Task",
"Event.System.TimeCreated.#attributes.SystemTime":"SystemTime",
"Event.System.Version":"Version",
"Event.EventData.AccessList":"AccessList",
"Event.EventData.AccessMask":"AccessMask",
"Event.EventData.Accesses":"Accesses",
"Event.EventData.AccountDomain":"AccountDomain",
"Event.EventData.AccountExpires":"AccountExpires",
"Event.EventData.AccountName":"AccountName",
"Event.EventData.AddonName":"AddonName",
"Event.EventData.Address":"Address",
"Event.EventData.AddressLength":"AddressLength",
"Event.EventData.AllowedToDelegateTo":"AllowedToDelegateTo",
"Event.EventData.Application":"Application",
"Event.EventData.AttributeLDAPDisplayName":"AttributeLDAPDisplayName",
"Event.EventData.AttributeValue":"AttributeValue",
"Event.EventData.AuditPolicyChanges":"AuditPolicyChanges",
"Event.EventData.AuditSourceName":"AuditSourceName",
"Event.EventData.AuthenticationPackageName":"AuthenticationPackageName",
"Event.EventData.Binary":"Binary",
"Event.EventData.BootMode":"BootMode",
"Event.EventData.BuildVersion":"BuildVersion",
"Event.EventData.CallingProcessName":"CallingProcessName",
"Event.EventData.CallTrace":"CallTrace",
"Event.EventData.CommandLine":"CommandLine",
"Event.EventData.Company":"Company",
"Event.EventData.Context":"Context",
"Event.EventData.CreationUtcTime":"CreationUtcTime",
"Event.EventData.CurrentDirectory":"CurrentDirectory",
"Event.EventData.DCName":"DCName",
"Event.EventData.Description":"Description",
"Event.EventData.DestinationAddress":"DestinationAddress",
"Event.EventData.DestinationHostname":"DestinationHostname",
"Event.EventData.DestinationIp":"DestinationIp",
"Event.EventData.DestinationIsIpv6":"DestinationIsIpv6",
"Event.EventData.DestinationPort":"DestinationPort",
"Event.EventData.DestinationPortName":"DestinationPortName",
"Event.EventData.DestPort":"DestPort",
"Event.EventData.Detail":"Detail",
"Event.EventData.Details":"Details",
"Event.EventData.DetectionSource":"DetectionSource",
"Event.EventData.DeviceClassName":"DeviceClassName",
"Event.EventData.DeviceDescription":"DeviceDescription",
"Event.EventData.DeviceName":"DeviceName",
"Event.EventData.DeviceNameLength":"DeviceNameLength",
"Event.EventData.DeviceTime":"DeviceTime",
"Event.EventData.DeviceVersionMajor":"DeviceVersionMajor",
"Event.EventData.DeviceVersionMinor":"DeviceVersionMinor",
"Event.EventData.DisplayName":"DisplayName",
"Event.EventData.EngineVersion":"EngineVersion",
"Event.EventData.ErrorCode":"ErrorCode",
"Event.EventData.ErrorDescription":"ErrorDescription",
"Event.EventData.ErrorMessage":"ErrorMessage",
"Event.EventData.EventSourceId":"EventSourceId",
"Event.EventData.EventType":"EventType",
"Event.EventData.ExtensionId":"ExtensionId",
"Event.EventData.ExtensionName":"ExtensionName",
"Event.EventData.ExtraInfo":"ExtraInfo",
"Event.EventData.FailureCode":"FailureCode",
"Event.EventData.FailureReason":"FailureReason",
"Event.EventData.FileVersion":"FileVersion",
"Event.EventData.FilterHostProcessID":"FilterHostProcessID",
"Event.EventData.FinalStatus":"FinalStatus",
"Event.EventData.GrantedAccess":"GrantedAccess",
"Event.EventData.Group":"Group",
"Event.EventData.GroupDomain":"GroupDomain",
"Event.EventData.GroupName":"GroupName",
"Event.EventData.GroupSid":"GroupSid",
"Event.EventData.HandleId":"HandleId",
"Event.EventData.Hash":"Hash",
"Event.EventData.Hashes":"Hashes",
"Event.EventData.HiveName":"HiveName",
"Event.EventData.HomeDirectory":"HomeDirectory",
"Event.EventData.HomePath":"HomePath",
"Event.EventData.HostApplication":"HostApplication",
"Event.EventData.HostName":"HostName",
"Event.EventData.HostVersion":"HostVersion",
"Event.EventData.IdleStateCount":"IdleStateCount",
"Event.EventData.Image":"Image",
"Event.EventData.ImageLoaded":"ImageLoaded",
"Event.EventData.ImagePath":"ImagePath",
"Event.EventData.Initiated":"Initiated",
"Event.EventData.IntegrityLevel":"IntegrityLevel",
"Event.EventData.IpAddress":"IpAddress",
"Event.EventData.IpPort":"IpPort",
"Event.EventData.KeyLength":"KeyLength",
"Event.EventData.LayerRTID":"LayerRTID",
"Event.EventData.LDAPDisplayName":"LDAPDisplayName",
"Event.EventData.LmPackageName":"LmPackageName",
"Event.EventData.LogonGuid":"LogonGuid",
"Event.EventData.LogonHours":"LogonHours",
"Event.EventData.LogonId":"LogonId",
"Event.EventData.LogonProcessName":"LogonProcessName",
"Event.EventData.LogonType":"LogonType",
"Event.EventData.MajorVersion":"MajorVersion",
"Event.EventData.Data.#text":"Message",
"Event.EventData.MinorVersion":"MinorVersion",
"Event.EventData.NewName":"NewName",
"Event.EventData.NewProcessId":"NewProcessId",
"Event.EventData.NewProcessName":"NewProcessName",
"Event.EventData.NewState":"NewState",
"Event.EventData.NewThreadId":"NewThreadId",
"Event.EventData.NewTime":"NewTime",
"Event.EventData.NewUacValue":"NewUacValue",
"Event.EventData.NewValue":"NewValue",
"Event.EventData.NotificationPackageName":"NotificationPackageName",
"Event.EventData.Number":"Number",
"Event.EventData.NumberOfGroupPolicyObjects":"NumberOfGroupPolicyObjects",
"Event.EventData.ObjectClass":"ObjectClass",
"Event.EventData.ObjectName":"ObjectName",
"Event.EventData.ObjectServer":"ObjectServer",
"Event.EventData.ObjectType":"ObjectType",
"Event.EventData.ObjectValueName":"ObjectValueName",
"Event.EventData.OldTime":"OldTime",
"Event.EventData.OldUacValue":"OldUacValue",
"Event.EventData.OperationType":"OperationType",
"Event.EventData.OriginalFileName":"OriginalFileName",
"Event.EventData.PackageName":"PackageName",
"Event.EventData.ParentCommandLine":"ParentCommandLine",
"Event.EventData.ParentImage":"ParentImage",
"Event.EventData.ParentProcessGuid":"ParentProcessGuid",
"Event.EventData.ParentProcessId":"ParentProcessId",
"Event.EventData.PasswordLastSet":"PasswordLastSet",
"Event.EventData.Payload":"Payload",
"Event.EventData.PerfStateCount":"PerfStateCount",
"Event.EventData.PipeName":"PipeName",
"Event.EventData.PreviousTime":"PreviousTime",
"Event.EventData.PrimaryGroupId":"PrimaryGroupId",
"Event.EventData.PrivilegeList":"PrivilegeList",
"Event.EventData.ProcessCommandLine":"ProcessCommandLine",
"Event.EventData.ProcessGuid":"ProcessGuid",
"Event.EventData.ProcessId":"ProcessId",
"Event.EventData.ProcessName":"ProcessName",
"Event.EventData.ProcessingMode":"ProcessingMode",
"Event.EventData.ProcessingTimeInMilliseconds":"ProcessingTimeInMilliseconds",
"Event.EventData.Product":"Product",
"Event.EventData.ProfilePath":"ProfilePath",
"Event.EventData.Properties":"Properties",
"Event.EventData.Protocol":"Protocol",
"Event.EventData.ProtocolHostProcessID":"ProtocolHostProcessID",
"Event.EventData.PuaCount":"PuaCount",
"Event.EventData.PuaPolicyId":"PuaPolicyId",
"Event.EventData.Publisher":"Publisher",
"Event.EventData.QfeVersion":"QfeVersion",
"Event.EventData.QueryName":"QueryName",
"Event.EventData.QueryResults":"QueryResults",
"Event.EventData.QueryStatus":"QueryStatus",
"Event.EventData.RelativeTargetName":"RelativeTargetName",
"Event.EventData.ResourceManager":"ResourceManager",
"Event.EventData.RetryMinutes":"RetryMinutes",
"Event.EventData.RuleName":"RuleName",
"Event.EventData.SamAccountName":"SAMAccountName",
"Event.EventData.SchemaVersion":"SchemaVersion",
"Event.EventData.ScriptPath":"ScriptPath",
"Event.EventData.ScriptBlockText":"ScriptBlockText",
"Event.EventData.SecurityPackageName":"SecurityPackageName",
"Event.EventData.ServerID":"ServerID",
"Event.EventData.ServerURL":"ServerURL",
"Event.EventData.Service":"Service",
"Event.EventData.ServiceName":"ServiceName",
"Event.EventData.ServicePrincipalNames":"ServicePrincipalNames",
"Event.EventData.ServiceType":"ServiceType",
"Event.EventData.ServiceVersion":"ServiceVersion",
"Event.EventData.ShareLocalPath":"ShareLocalPath",
"Event.EventData.ShareName":"ShareName",
"Event.EventData.ShutdownActionType":"ShutdownActionType",
"Event.EventData.ShutdownEventCode":"ShutdownEventCode",
"Event.EventData.ShutdownReason":"ShutdownReason",
"Event.EventData.SidHistory":"SidHistory",
"Event.EventData.Signature":"Signature",
"Event.EventData.SignatureStatus":"SignatureStatus",
"Event.EventData.Signed":"Signed",
"Event.EventData.SourceAddress":"SourceAddress",
"Event.EventData.SourceHostname":"SourceHostname",
"Event.EventData.SourceImage":"SourceImage",
"Event.EventData.SourceIp":"SourceIp",
"Event.EventData.SourceNetworkAddress":"SourceNetworkAddress",
"Event.EventData.SourceIsIpv6":"SourceIsIpv6",
"Event.EventData.SourcePort":"SourcePort",
"Event.EventData.SourcePortName":"SourcePortName",
"Event.EventData.SourceProcessGuid":"SourceProcessGuid",
"Event.EventData.SourceProcessId":"SourceProcessId",
"Event.EventData.StartAddress":"StartAddress",
"Event.EventData.StartFunction":"StartFunction",
"Event.EventData.StartModule":"StartModule",
"Event.EventData.StartTime":"StartTime",
"Event.EventData.StartType":"StartType",
"Event.EventData.State":"State",
"Event.EventData.Status":"Status",
"Event.EventData.StopTime":"StopTime",
"Event.EventData.SubStatus":"SubStatus",
"Event.EventData.SubjectDomainName":"SubjectDomainName",
"Event.EventData.SubjectLogonId":"SubjectLogonId",
"Event.EventData.SubjectUserName":"SubjectUserName",
"Event.EventData.SubjectUserSid":"SubjectUserSid",
"Event.EventData.TSId":"TSId",
"Event.EventData.TargetDomainName":"TargetDomainName",
"Event.EventData.TargetFilename":"TargetFileName",
"Event.EventData.TargetImage":"TargetImage",
"Event.EventData.TargetInfo":"TargetInfo",
"Event.EventData.TargetLogonGuid":"TargetLogonGuid",
"Event.EventData.TargetLogonId":"TargetLogonId",
"Event.EventData.TargetObject":"TargetObject",
"Event.EventData.TargetProcessAddress":"TargetProcessAddress",
"Event.EventData.TargetProcessGuid":"TargetProcessGuid",
"Event.EventData.TargetProcessId":"TargetProcessId",
"Event.EventData.TargetServerName":"TargetServerName",
"Event.EventData.TargetSid":"TargetSid",
"Event.EventData.TargetUserName":"TargetUserName",
"Event.EventData.TargetUserSid":"TargetUserSid",
"Event.EventData.TaskContent":"TaskContent",
"Event.EventData.TaskContentNew":"TaskContentNew",
"Event.EventData.TaskName":"TaskName",
"Event.EventData.TerminalSessionId":"TerminalSessionId",
"Event.EventData.ThrottleStateCount":"ThrottleStateCount",
"Event.EventData.TicketEncryptionType":"TicketEncryptionType",
"Event.EventData.TicketOptions":"TicketOptions",
"Event.EventData.TimeSource":"TimeSource",
"Event.EventData.TokenElevationType":"TokenElevationType",
"Event.EventData.TransactionId":"TransactionId",
"Event.EventData.TransmittedServices":"TransmittedServices",
"Event.EventData.User":"User",
"Event.EventData.UserAccountControl":"UserAccountControl",
"Event.EventData.UserParameters":"UserParameters",
"Event.EventData.UserPrincipalName":"UserPrincipalName",
"Event.EventData.UserSid":"UserSid",
"Event.EventData.UserWorkstations":"UserWorkstations",
"Event.EventData.UtcTime":"UtcTime",
"Event.EventData.Version":"Version",
"Event.EventData.Workstation":"Workstation",
"Event.EventData.WorkstationName":"WorkstationName",
"Event.EventData.updateGuid":"updateGuid",
"Event.EventData.updateRevisionNumber":"updateRevisionNumber",
"Event.EventData.updateTitle":"updateTitle",
"Event.EventData.ParentIntegrityLevel":"ParentIntegrityLevel",
"Event.EventData.ParentUser":"ParentUser"
}
}
\ No newline at end of file
# Zircolite documentation
## Advanced use
* [Working with large datasets](#working-with-large-datasets)
* [Using GNU Parallel](#using-gnu-parallel)
* [Using Zircolite MP](#using-zircolite-mp)
* [Filtering](#filtering)
* [File filters](#file-filters)
* [Time filters](#time-filters)
* [Rule filters](#rule-filters)
* [Forwarding detected events](#forwarding-detected-events)
* [Templating and Formatting](#templating-and-formatting)
* [Mini GUI](#mini-gui)
* [Packaging Zircolite](#packaging-zircolite)
---
### Working with large datasets
Zircolite tries to be as fast as possible so a lot of data is stored in memory. So :
- **Zircolite memory use oscillate between 2 or 3 times the size of the logs**
- It is not a good idea to use it on very big EVTX files or a large number of EVTX **as is**
The tool has been created to be used on very big datasets and there are a lot of ways to speed up Zircolite :
- Using as much CPU core as possible : see below "[Using GNU Parallel](using-gnu-parallel)"
- Using [Filtering](#filtering)
#### Using GNU Parallel
Except when `evtx_dump` is used, Zircolite only use one core. So if you have a lot of EVTX files and their total size is big, it is recommanded that you use a script to launch multiple Zircolite instances. On Linux or MacOS The easiest way is to use **GNU Parallel**.
:information_source: on MacOS, please use GNU find (`brew install find` will install `gfind`)
- **"DFIR Case mode" : One directory per computer/endpoint**
This mode is very useful when you have a case where all your evidences is stored per computer (one directory per computer containing all EVTX for this computer). It will create one result file per computer in the current directory.
```shell
find <CASE_DIRECTORY> -maxdepth 1 -mindepth 1 -type d | \
parallel --bar python3 zircolite.py --evtx {}
--ruleset rules/rules_windows_sysmon.json --outfile {/.}.json
```
One downside of this mode is that if you have less computer evidences than CPU Cores, they all will not be used.
- **"WEF/WEC mode" : One zircolite instance per EVTX**
You can use this mode when you have a lot of aggregated EVTX coming from multiple computers. It is generaly the case when you use WEF/WEC and you recover the EVTX files from the collector. This mode will create one result file per EVTX.
```shell
find <CASE_DIRECTORY> -type f -name "*.| \
parallel -j -1 --progress python3 zircolite.py --evtx {} \
--ruleset rules/rules_windows_sysmon.json --outfile {/.}.json
```
In this example the `-j -1` is for using all cores but one. You can adjust the number of used cores with this arguments.
#### Using Zircolite MP
***deprecated***
---
### Filtering
Zircolite has a lot of filtering options to speed up the detection process. Don't overlook these options because they can save you a lot of time.
#### File filters
Some EVTX files are not used by SIGMA rules but can become quite large (a good example is `Microsoft-Windows-SystemDataArchiver%4Diagnostic.evtx`), if you use Zircolite with a directory as input argument, all EVTX files will be converted, saved and matched against the SIGMA Rules.
To speed up the detection process, you may want to use Zircolite on files matching or not matching a specific pattern. For that you can use **filters** provided by the two command line arguments :
- `-s` or `--select` : select files partly matching the provided a string (case insensitive)
- `-a` or `--avoid` : exclude files partly matching the provided a string (case insensitive)